Merge branch 'master' of github.com:vkuznet/WMCore into iam-token

CHANGES

@@ -1,3 +1,10 @@
+2.0.3.pre5 to 2.0.3.pre6:
+  - Add buildAllDep option && Fix package verstion estimation for ParseSpecCmsswdist. (Todor Ivanov) #11132
+  - Bring dbs3-client version to 4.0.8 (Alan Malta Rodrigues) #11138
+  - speed up listDatasetFileDetails API (Valentin Kuznetsov) #11099
+  - Remove profile scope (Valentin Kuznetsov) #11134
+
+
 2.0.3.pre4 to 2.0.3.pre5:
   - fix unit tests (Alan Malta Rodrigues) #11119
   - Properly handle OpenRunningTimeout in WorkQueue (Alan Malta Rodrigues) #11119

bin/adhoc-scripts/ParseSpecCmsswdist.py

@@ -110,19 +110,23 @@
 import os
 import logging
 # import pprint
+from pprint import pformat
+
+
+DEPS_SPEC = set()
 
 
 def getDepsSpec(specdir, specfile):
     """
     - Opens a spec files
     - Gets the list of Requirements
-    - Gets the list of Requirements that start with `py2-`
+    - Gets the list of Requirements that start with `py2-` or `py3-`
     """
     specsPy = []  # p5- is for cpan, perl5 dependencies
     specsAll = []
     specfilePath = os.path.join(specdir, specfile)
     logging.info(specfilePath)
-    with open(specfilePath) as specFile:
+    with open(specfilePath, encoding='utf-8') as specFile:
         for line in specFile.readlines():
             requiresPattern = "Requires:"
             if line.strip().startswith(requiresPattern):
@@ -139,25 +143,38 @@ def getDepsSpec(specdir, specfile):
     return specsPy, specsAll
 
 
-DEPS_SPEC = []
-
-
-def getDepsRecursive(specdir, specfile):
+def getDepsRecursive(specdir, specfile, buildAllDep=False, recursionDepth=-1):
     """
     - calls getDepsSpec()
     - add all the dependencies that start with `py2-` or `py3-` to deps_spec
     - call recursively getDepsSpec() on all the dependencies
+    :param specdir: The directory holding all the spec files to iterate through
+    :param specfile: The initial spec file.
+    :param buildAllDep: A Bool flag to signal building of all dependencies not only python related
+    :param recursionDepth: An integer to set the maximum recursion depth.
     """
+
+    global DEPS_SPEC
     # first time that we call this, no specfile is saved because
     # is it the parent specfile
+    if recursionDepth == 0:
+        return
+    elif recursionDepth < 0:
+        # if recursionDepth is negative means indefinite.
+        pass
+    else:
+        recursionDepth -= 1
+
     specsPy, specsAll = getDepsSpec(specdir, specfile)
-    logging.info("%s %s", specfile, specsAll)
-    logging.info("%s %s", specfile, specsPy)
-    for spec in specsPy:
-        if spec not in DEPS_SPEC:
-            DEPS_SPEC.append(spec)
+    logging.debug("%s: specsAll: \n%s", specfile, pformat(specsAll))
+    logging.debug("%s: sepcsPy: \n%s", specfile, pformat(specsPy))
+    if buildAllDep:
+        DEPS_SPEC |= set(specsAll)
+    else:
+        DEPS_SPEC |= set(specsPy)
+
     for spec in specsAll:
-        getDepsRecursive(specdir, spec)
+        getDepsRecursive(specdir, spec, buildAllDep=buildAllDep, recursionDepth=recursionDepth)
 
 
 def buildWithPip(lines):
@@ -222,12 +239,20 @@ def getPipVersion(specdir, specfile):
     spec files only
     """
     specpath = os.path.join(specdir, specfile)
-    with open(specpath) as specFile:
+    with open(specpath, encoding='utf-8') as specFile:
         lines = specFile.readlines()
         line0 = lines[0].strip()
         nameAndVer = []
         if "### RPM external" in line0:
-            nameAndVer = line0[line0.find("-") + 1:].split(" ")
+            # nameAndVer = line0[line0.find("-") + 1:].split(" ")
+            line0List = line0.split(" ")
+            name=line0List[-2]
+            # getting rid of py{2,3}- prefix in the package name.
+            for patternStr in ("py2-", "py3-"):
+                name = name.replace(patternStr, "")
+
+            version=line0List[-1]
+            nameAndVer = [name, version]
             logging.debug("%s %s", specfile, nameAndVer)
             if buildWithPip(lines):
                 name = getNameBuiltwithpip(lines)
@@ -244,22 +269,27 @@ def writeRequirements(specdir, depsspec, requirementsFilename):
     """Writes requirements.txt file"""
     requirementLines = []
     for spec in depsspec:
+        # logging.info("finding pip version for spec: %s", spec)
         line = getPipVersion(specdir, spec)
         if line:
             line += "\n"
             requirementLines.append(line)
-    with open(requirementsFilename, "w") as requirementsFile:
+    with open(requirementsFilename, "w", encoding='utf-8') as requirementsFile:
         requirementsFile.writelines(requirementLines)
 
 
 def main():
     """
-    :param spec_dir: Path to base direcotry that contains the spec files
-    :type spec_dir: str, required
-    :param spec_file: filename of the spec file
-    :type spec_file: str, required
+    All parameters taken through argument parsing:
+    :param --spec-dir: Path to base direcotry that contains the spec files
+    :type spec-dir: str, required
+    :param spec-file: filename of the spec file
+    :type spec-file: str, required
+    :param --level: Recursion level.
+    :param --all: A Bool flag to signal for building all dependencies not python related.
+    :param --recursive: A Bool flag to signal recursive iteration through spec files.
     """
-    logging.basicConfig(level=logging.INFO)
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
 
     parser = argparse.ArgumentParser()
     parser.add_argument("-d", "--spec-dir",
@@ -271,21 +301,35 @@ def main():
                         help="spec file filename (e.g. `t0.spec`)",
                         type=str,
                         required=True)
+    parser.add_argument("-a", '--all', dest='buildAllDep', action='store_true', default=False)
+    parser.add_argument("-l", '--level',
+                        help="Recursion level. If --recursive is True and --level \
+                              is specified recursion will stop at depth=level. Negative value means indefinite - This is the Default.",
+                        dest='recursionDepth',
+                        type=int,
+                        default=-1)
+
     featureParser = parser.add_mutually_exclusive_group(required=False)
-    featureParser.add_argument('--recursive', dest='recursive', action='store_true')
-    featureParser.add_argument('--no-recursive', dest='recursive', action='store_false')
+    featureParser.add_argument('--recursive', dest='recursive',
+                               action='store_true',
+                               help="Iterate recursively trough all spec files at SPEC_DIR.")
+    featureParser.add_argument('--no-recursive', dest='recursive',
+                               action='store_false',
+                               help="Parse only the current spec file - This is the Default.")
     parser.set_defaults(feature=False)
     args = parser.parse_args()
 
+
     if args.recursive:
-        getDepsRecursive(args.spec_dir, args.spec_file)
+        getDepsRecursive(args.spec_dir, args.spec_file,
+                         buildAllDep=args.buildAllDep,
+                         recursionDepth=args.recursionDepth)
     else:
-        specsPy, _ = getDepsSpec(args.spec_dir, args.spec_file)
-        for spec in specsPy:
-            if spec not in DEPS_SPEC:
-                DEPS_SPEC.append(spec)
-    # pprint.pprint(DEPS_SPEC)
-    logging.info(len(DEPS_SPEC))
+        getDepsRecursive(args.spec_dir, args.spec_file,
+                         buildAllDep=args.buildAllDep,
+                         recursionDepth=1)
+
+    logging.info("DEPS_SPEC: len: %s: \n%s", len(DEPS_SPEC), pformat(DEPS_SPEC))
 
     writeRequirements(args.spec_dir, DEPS_SPEC,
                       "requirements_" + args.spec_file + "_auto.txt")

bin/create-iam-token.sh

@@ -45,7 +45,6 @@ if [ -n "IAM_TOKEN" ]; then
     # grant_type=client_credentials key=value pair is required by IAM provider
     # to specify that request contains clients credentials
     curl -s -k -d grant_type=client_credentials \
-            -d scope="profile" \
             -u ${client_id}:${client_secret} \
             https://cms-auth.web.cern.ch/token | jq -r '.access_token' > $IAM_TOKEN
     echo "New IAM token generated and can be found at $IAM_TOKEN"

requirements.txt

@@ -12,33 +12,33 @@
 # for more details please refer to official Python documentation, see
 # https://www.python.org/dev/peps/pep-0440/#version-specifiers
 
-Cheetah3~=3.2.6.post2         # wmagent,reqmgr2,reqmon
-CherryPy~=17.4.0              # wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms
-CMSCouchapp~=1.3.4            # wmagent
-CMSMonitoring~=0.3.4          # wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms
-coverage~=5.4                 # wmagent,wmagent-devtools
-cx-Oracle~=7.3.0              # wmagent
-dbs3-client~=4.0.7            # wmagent,reqmgr2,reqmon,global-workqueue
-future~=0.18.2                # wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
-gfal2-python~=1.11.0.post3    # reqmgr2ms
-httplib2~=0.19.0              # wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
-htcondor~=8.9.7               # wmagent
-Jinja2~=3.0.1                 # wmagent
-memory-profiler~=0.58.0       # wmagent-devtools
-mock~=4.0.3                   # wmagent,wmagent-devtools
-mox3~=1.1.0                   # wmagent-devtools
-mysqlclient~=2.0.3            # wmagent
-nose~=1.3.7                   # wmagent-devtools
-nose2~=0.10.0                 # wmagent-devtools
-pep8~=1.7.1                   # wmagent-devtools
-psutil~=5.8.0                 # wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue
-pycurl~=7.43.0.6              # wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms
-pylint~=2.7.0                 # wmagent-devtools
-pymongo~=4.0.1                # wmagent-devtools,reqmgr2ms
-pyOpenSSL~=18.0.0             # wmagent
-pyzmq~=19.0.2                 # wmagent
-retry~=0.9.2                  # wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms
-rucio-clients~=1.25.5         # wmagent,global-workqueue,reqmgr2ms
-Sphinx~=4.0.3                 # wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
-SQLAlchemy~=1.3.3             # wmagent
+Cheetah3~=3.2.6.post1         # wmcore,wmagent,reqmgr2,reqmon
+CherryPy~=17.4.0              # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms
+CMSCouchapp~=1.3.4            # wmcore,wmagent
+CMSMonitoring~=0.3.4          # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms
+coverage~=5.4                 # wmcore,wmagent,wmagent-devtools
+cx-Oracle~=7.3.0              # wmcore,wmagent
+dbs3-client~=4.0.8            # wmcore,wmagent,reqmgr2,reqmon,global-workqueue
+future~=0.18.2                # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
+gfal2-python~=1.11.0.post3    # wmcore,reqmgr2ms
+httplib2~=0.19.0              # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
+htcondor~=8.9.7               # wmcore,wmagent
+Jinja2~=3.0.1                 # wmcore,wmagent
+memory-profiler~=0.58.0       # wmcore,wmagent-devtools
+mock~=4.0.3                   # wmcore,wmagent,wmagent-devtools
+mox3~=1.1.0                   # wmcore,wmagent-devtools
+mysqlclient~=2.0.3            # wmcore,wmagent
+nose~=1.3.7                   # wmcore,wmagent-devtools
+nose2~=0.10.0                 # wmcore,wmagent-devtools
+pycodestyle~=2.8.0            # wmcore,wmagent-devtools
+psutil~=5.8.0                 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue
+pycurl~=7.43.0.6              # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms
+pylint~=2.13.5                # wmcore,wmagent-devtools
+pymongo~=4.0.1                # wmcore,wmagent-devtools,reqmgr2ms
+pyOpenSSL~=18.0.0             # wmcore,wmagent
+pyzmq~=19.0.2                 # wmcore,wmagent
+retry~=0.9.2                  # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms
+rucio-clients~=1.25.5         # wmcore,wmagent,global-workqueue,reqmgr2ms
+Sphinx~=4.0.3                 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
+SQLAlchemy~=1.3.3             # wmcore,wmagent
 pyjwt~=2.3.0                  # wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms,wmcore