Test setdatasetfilesstatus (#5246)

* add setdataset.py for #5204

* refactor and add setfiles

* add Content-type arg to HTTPRequests

* setdataset to use contentType

* rename commands to setdatasetstatus setfilestatus

* add autocomplete

* list of LFNs not supported yet

* some pylint and pep8

* add logging for setfilestatus

* more HTTPRequest,CRABRest,getDBSRest to new RestInterfaces.py

* do not pass version to REST clients, it is set in HTTPRequests

* removed one import __version__ too many !

* fix use of version and UserAgent

* fix use of version and UserAgent

* simpley make userAgent=CRABClient/__version__ the default

* cleanup use of Content-type

* cleanup use of Content-type

* add comment

src/python/CRABClient/CRABOptParser.py

@@ -1,7 +1,6 @@
 from optparse import OptionParser
 
 from CRABClient import __version__ as client_version
-from CRABClient.ClientUtilities import getAvailCommands
 from ServerUtilities import SERVICE_INSTANCES
 
 

src/python/CRABClient/ClientUtilities.py

@@ -230,8 +230,7 @@ def uploadlogfile(logger, proxyfilename, taskname=None, logfilename=None, logpat
         # so it needs its own REST server instantiation
         restClass = CRABClient.Emulator.getEmulator('rest')
         crabserver = restClass(hostname=serverurl, localcert=proxyfilename, localkey=proxyfilename,
-                               retry=2, logger=logger, verbose=False, version=__version__,
-                               userAgent='CRABClient')
+                               retry=2, logger=logger, verbose=False)
         crabserver.setDbInstance(instance)
         cacheurl = server_info(crabserver=crabserver, subresource='backendurls')['cacheSSL']
 
@@ -646,7 +645,6 @@ def validateSubmitOptions(options, args):
 #Since server_info class needs SubCommand, and SubCommand needs server_info for
 #delegating the proxy then we are screwed
 #If anyone has a better solution please go on, otherwise live with that one :) :)
-from CRABClient import __version__
 
 def server_info(crabserver=None, subresource=None):
     """

src/python/CRABClient/Commands/SubCommand.py

@@ -362,18 +362,16 @@ def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
         # this is usually the first time that a call to the server is made, so where Emulator('rest') is initialized
         # arguments to Emulator('rest') call must match those for HTTPRequest.__init__ in RESTInteractions.py
         #server = CRABClient.Emulator.getEmulator('rest')(url=serverurl, localcert=proxyfilename, localkey=proxyfilename,
-        #          version=__version__, retry=2, logger=logger)
+        #          retry=2, logger=logger)
         if self.cmdconf['requiresREST']:
             crabRest = CRABClient.Emulator.getEmulator('rest')
             self.crabserver = crabRest(hostname=self.serverurl, localcert=self.proxyfilename, localkey=self.proxyfilename,
-                                        retry=2, logger=self.logger, verbose=False, version=__version__,
-                                        userAgent='CRABClient')
+                                        retry=2, logger=self.logger, verbose=False)
             self.crabserver.setDbInstance(self.instance)
             # prepare also a test crabserver instance which will send tarballs to S3
             self.s3tester = crabRest(hostname='cmsweb-testbed.cern.ch',
                                       localcert=self.proxyfilename, localkey=self.proxyfilename,
-                                      retry=0, logger=self.logger, verbose=False, version=__version__,
-                                      userAgent='CRABClient')
+                                      retry=0, logger=self.logger, verbose=False)
             self.s3tester.setDbInstance('preprod')
             self.handleMyProxy()
 

src/python/CRABClient/Commands/setdatasetstatus.py

@@ -9,60 +9,13 @@
 from CRABClient.Commands.SubCommand import SubCommand
 from CRABClient.ClientExceptions import MissingOptionException, ConfigurationException, CommandFailedException
 from CRABClient.ClientUtilities import colors
-from CRABClient.CrabRestInterface import HTTPRequests
+from CRABClient.RestInterfaces import getDbsREST
 
 if sys.version_info >= (3, 0):
     from urllib.parse import urlencode  # pylint: disable=E0611
 if sys.version_info < (3, 0):
     from urllib import urlencode
 
-try:
-    from CRABClient import __version__
-except:  # pylint: disable=bare-except
-    __version__ = '0.0.0'
-
-
-def getDbsREST(instance=None, logger=None, cert=None, key=None, version=None):
-    """
-    given a DBS istance (e.g. prod/phys03) returns a DBSReader and DBSWriter
-    client instances which communicate with DBS REST via curl
-    Arguments:
-    logger: a logger
-    cert, key : name of files, can use the path to X509_USER_PROXY for both
-    version: the CRAB Client version to put in the User Agent field of the query
-    """
-    # if user supplied a simple prod/phys03 like instance, these two lines will do
-    # note that our HTTPRequests will add https://
-    dbsReadUrl = "cmsweb.cern.ch:8443/dbs/" + instance + "/DBSReader/"
-    dbsWriteUrl = "cmsweb.cern.ch:8443/dbs/" + instance + "/DBSWriter/"
-    # a possible use case e.g. for testing is to use int instance of DBS. requires testbed CMSWEB
-    if instance.startswith('int'):
-        dbsReadUrl = dbsReadUrl.replace('cmsweb', 'cmsweb-testbed')
-        dbsWriteUrl = dbsWriteUrl.replace('cmsweb', 'cmsweb-testbed')
-    # if user knoww better and provided a full URL, we'll take and adapt
-    # to have both Reader and Writer,
-    if instance.startswith("https://"):
-        url = instance.lstrip("https://")  # will be added back in HTTPRequests
-        if "DBSReader" in url:
-            dbsReadUrl = url
-            dbsWriteUrl = url.replace('DBSReader', 'DBSWriter')
-        elif 'DBSWriter' in url:
-            dbsWriteUrl = url
-            dbsReadUrl = url.replace('DBSWriter', 'DBSReader')
-        else:
-            raise ConfigurationException("bad instance value %s" % instance)
-
-    logger.debug('Read Url  = %s' % dbsReadUrl)
-    logger.debug('Write Url = %s' % dbsWriteUrl)
-
-    dbsReader = HTTPRequests(hostname=dbsReadUrl, localcert=cert, localkey=key,
-                             retry=2, logger=logger, verbose=False, contentType='application/json',
-                             userAgent='CRABClient', version=version)
-
-    dbsWriter = HTTPRequests(hostname=dbsWriteUrl, localcert=cert, localkey=key,
-                             retry=2, logger=logger, verbose=False, contentType='application/json',
-                             userAgent='CRABClient', version=version)
-    return dbsReader, dbsWriter
 
 
 class setdatasetstatus(SubCommand):
@@ -95,8 +48,7 @@ def __call__(self):
 
         # from DBS instance, to DBS REST services
         dbsReader, dbsWriter = getDbsREST(instance=instance, logger=self.logger,
-                                          cert=self.proxyfilename, key=self.proxyfilename,
-                                          version=__version__)
+                                          cert=self.proxyfilename, key=self.proxyfilename)
 
         self.logger.info("looking up Dataset %s in DBS %s" % (dataset, instance))
         datasetStatusQuery = {'dataset': dataset, 'dataset_access_type': '*', 'detail': True}

src/python/CRABClient/Commands/setfilestatus.py

@@ -8,12 +8,8 @@
 from CRABClient.Commands.SubCommand import SubCommand
 from CRABClient.ClientExceptions import MissingOptionException, ConfigurationException, CommandFailedException
 from CRABClient.ClientUtilities import colors
-from CRABClient.Commands.setdatasetstatus import getDbsREST
 
-try:
-    from CRABClient import __version__
-except:  # pylint: disable=bare-except
-    __version__ = '0.0.0'
+from CRABClient.RestInterfaces import getDbsREST
 
 
 class setfilestatus(SubCommand):
@@ -63,8 +59,7 @@ def __call__(self):
 
         # from DBS instance, to DBS REST services
         dbsReader, dbsWriter = getDbsREST(instance=instance, logger=self.logger,
-                                          cert=self.proxyfilename, key=self.proxyfilename,
-                                          version=__version__)
+                                          cert=self.proxyfilename, key=self.proxyfilename)
         # we will need the dataset name
         if dataset:
             datasetName = dataset

src/python/CRABClient/Emulator.py

@@ -24,6 +24,6 @@ def setEmulator(name, value):
     overrideDict[name] = value
 
 def getDefaults():
-    import CRABClient.CrabRestInterface
-    return {'rest' : CRABClient.CrabRestInterface.CRABRest,
+    import CRABClient.RestInterfaces
+    return {'rest' : CRABClient.RestInterfaces.CRABRest,
             'ufc' : 'dummy_ufc'}

src/python/CRABClient/CrabRestInterface.py → src/python/CRABClient/RestInterfaces.py

@@ -20,15 +20,12 @@
 
 from CRABClient.ClientUtilities import execute_command
 from ServerUtilities import encodeRequest
-from CRABClient.ClientExceptions import RESTInterfaceException
+from CRABClient.ClientExceptions import RESTInterfaceException, ConfigurationException
 
 try:
-    from TaskWorker import __version__
+    from CRABClient import __version__
 except:  # pylint: disable=bare-except
-    try:
-        from CRABClient import __version__
-    except:  # pylint: disable=bare-except
-        __version__ = '0.0.0'
+    __version__ = '0.0.0'
 
 
 EnvironmentException = Exception
@@ -89,7 +86,7 @@ class HTTPRequests(dict):
     """
 
     def __init__(self, hostname='localhost', localcert=None, localkey=None, contentType=None,
-                 retry=0, logger=None, version=__version__, verbose=False, userAgent='CRAB?'):
+                 retry=0, logger=None, version=__version__, verbose=False, userAgent=None):
         """
         Initialise an HTTP handler
         """
@@ -107,9 +104,10 @@ def __init__(self, hostname='localhost', localcert=None, localkey=None, contentT
             else:
                 # add port 8443
                 self['host'] = self['host'].replace(".cern.ch", ".cern.ch:8443", 1)
+        if not userAgent:
+            userAgent = 'CRABClient/%s' % __version__
         self.setdefault("cert", localcert)
         self.setdefault("key", localkey)
-        self.setdefault("version", version)
         self.setdefault("retry", retry)
         self.setdefault("verbose", verbose)
         self.setdefault("userAgent", userAgent)
@@ -162,7 +160,7 @@ def makeRequest(self, uri=None, data=None, verb='GET'):
         caCertPath = self.getCACertPath()
         url = 'https://' + self['host'] + uri
 
-        # if it is a dictionary, we need to encode it to string
+        # if it is a dictionary, we need to encode it to string (will not affect JSON)
         if isinstance(data, dict):
             data = encodeRequest(data)
         self.logger.debug("Encoded data for curl request: %s", data)
@@ -182,20 +180,18 @@ def makeRequest(self, uri=None, data=None, verb='GET'):
         # Same variable is also used inside CRABServer, we should keep name changes (if any) synchronized
         if os.getenv('CRAB_useGoCurl'):
             command += '/cvmfs/cms.cern.ch/cmsmon/gocurl -verbose 2 -method {0}'.format(verb)
-            command += ' -header "User-Agent: %s/%s"' % (self['userAgent'], self['version'])
+            command += ' -header "User-Agent: %s"' % self['userAgent']
             command += ' -header "Accept: */*"'
             if self['Content-type']:
                 command += ' -header "Content-type: %s"' % self['Content-type']
-            if verb in ['POST', 'PUT']:
-                command += ' -header "Content-Type: application/x-www-form-urlencoded"'
             command += ' -data "@%s"' % path
             command += ' -cert "%s"' % self['cert']
             command += ' -key "%s"' % self['key']
             command += ' -capath "%s"' % caCertPath
             command += ' -url "%s" | tee /dev/stderr ' % url
         else:
             command += 'curl -v -X {0}'.format(verb)
-            command += ' -H "User-Agent: %s/%s"' % (self['userAgent'], self['version'])
+            command += ' -H "User-Agent: %s"' % self['userAgent']
             command += ' -H "Accept: */*"'
             if self['Content-type']:
                 command += ' -H "Content-type: %s"' % self['Content-type']
@@ -265,10 +261,9 @@ class CRABRest:
     Add two methods to set and get the DB instance
     """
 
-    def __init__(self, hostname='localhost', localcert=None, localkey=None, version=__version__,
-                 retry=0, logger=None, verbose=False, userAgent='CRAB?'):
+    def __init__(self, hostname='localhost', localcert=None, localkey=None,
+                 retry=0, logger=None, verbose=False, userAgent=None):
         self.server = HTTPRequests(hostname=hostname, localcert=localcert, localkey=localkey,
-                                   version=version,
                                    retry=retry, logger=logger, verbose=verbose, userAgent=userAgent)
         instance = 'prod'
         self.uriNoApi = '/crabserver/' + instance + '/'
@@ -294,3 +289,48 @@ def put(self, api=None, data=None):
     def delete(self, api=None, data=None):
         uri = self.uriNoApi + api
         return self.server.delete(uri, data)
+
+
+def getDbsREST(instance=None, logger=None, cert=None, key=None, userAgent=None):
+    """
+    given a DBS istance (e.g. prod/phys03) returns a DBSReader and DBSWriter
+    HTTP client instances which can communicate with DBS REST via curl
+    Arguments:
+    instance: a DBS instance in the form prod/global or prod/phys03 or similar
+            or a full DBS URL like https:cmsweb.cern.ch/dbs/dev/phys01/DBSReader
+            in the latter care the corresponding DBSWriter will also be created, or
+            viceversa, the caller can indicate a DBSWriter and will get back HTTPRequest
+            objects for both Reader and Writer
+    logger: a logger
+    cert, key : name of files, can use the path to X509_USER_PROXY for both
+    """
+    # if user supplied a simple prod/phys03 like instance, these two lines will do
+    # note that our HTTPRequests will add https://
+    dbsReadUrl = "cmsweb.cern.ch:8443/dbs/" + instance + "/DBSReader/"
+    dbsWriteUrl = "cmsweb.cern.ch:8443/dbs/" + instance + "/DBSWriter/"
+    # a possible use case e.g. for testing is to use int instance of DBS. requires testbed CMSWEB
+    if instance.startswith('int'):
+        dbsReadUrl = dbsReadUrl.replace('cmsweb', 'cmsweb-testbed')
+        dbsWriteUrl = dbsWriteUrl.replace('cmsweb', 'cmsweb-testbed')
+    # if user knoww better and provided a full URL, we'll take and adapt
+    # to have both Reader and Writer,
+    if instance.startswith("https://"):
+        url = instance.lstrip("https://")  # will be added back in HTTPRequests
+        if "DBSReader" in url:
+            dbsReadUrl = url
+            dbsWriteUrl = url.replace('DBSReader', 'DBSWriter')
+        elif 'DBSWriter' in url:
+            dbsWriteUrl = url
+            dbsReadUrl = url.replace('DBSWriter', 'DBSReader')
+        else:
+            raise ConfigurationException("bad instance value %s" % instance)
+
+    logger.debug('Read Url  = %s' % dbsReadUrl)
+    logger.debug('Write Url = %s' % dbsWriteUrl)
+
+    dbsReader = HTTPRequests(hostname=dbsReadUrl, localcert=cert, localkey=key,
+                             retry=2, logger=logger, verbose=False, userAgent=userAgent)
+
+    dbsWriter = HTTPRequests(hostname=dbsWriteUrl, localcert=cert, localkey=key,
+                             retry=2, logger=logger, verbose=False, userAgent=userAgent)
+    return dbsReader, dbsWriter