From 9204ce04c329fdc0ccd017fdff83c19e3cc2eb31 Mon Sep 17 00:00:00 2001 From: Todor Ivanov Date: Thu, 14 Jan 2021 14:54:03 +0100 Subject: [PATCH 1/2] Include logic for cleaning input data rules. Fetch globalLocks from wmstatsServer and reqmgr. Add more fileds to MSRuleCleanerWflow && Recursively parsing the workflow description. Filter repeated values for str objects in wfParser. Include ParentDataset field && Convert back to lists && Extend getRucioRules logic to include input datasets. Resolve parents && Execute plineMSTrBlock Replace closure with class in MSRuleclanerWorkflow. Code review changes: Private methods for globalLocks && changes in data structures at MSRulecleanerWorkflow Clean FIXME comments. Code review changes. Suggested Rucioacct to contList map. Suggested Rucioacct to contList map 2. Introduce Resolve Parent Exception. Handle Resolve Parent Exception. Code review changes. Code review changes. Code review changes. Handle None value returned per dataset from findParents(). --- .../DataStructs/MSRuleCleanerWflow.py | 159 +++++++++++++++++- .../MicroService/Unified/MSRuleCleaner.py | 128 +++++++++++--- src/python/WMCore/Services/ReqMgr/ReqMgr.py | 12 ++ .../Services/WMStatsServer/WMStatsServer.py | 10 ++ 4 files changed, 279 insertions(+), 30 deletions(-) diff --git a/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py b/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py index b56c32ffdf..10af68f094 100644 --- a/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py +++ b/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py @@ -7,6 +7,126 @@ from __future__ import division, print_function from copy import deepcopy +from Utils.IteratorTools import flattenList + + +class WfParser(object): + """ + Workflow description parser class. + """ + def __init__(self, docSchema): + """ + The init method for the Workflow parser class. + :param docSchema: Document template in the form of a list of tuples as follows: + [('KeyName', DefaultValue, type), + ('KeyName', DefaultValue, type), + ...] + To be used for identifying the fields to be searched for + in the workflow description + + """ + self.extDoc = {} + for tup in docSchema: + self.extDoc[tup[0]] = {'keyName': tup[0], + 'values': list(), + 'default': tup[1], + 'type': tup[2]} + + def __call__(self, wfDescr): + """ + The Call method for the Workflow parser class. + """ + self._paramFinder(wfDescr) + self._wfParse() + return self.extDoc + + def _paramFinder(self, wfObj): + """ + Private method used to recursively traverse a workflow description + and search for all the keyNames defined in the extDoc auxiliary data + structure. If a 'keyName' happens to be present in several nested levels, + or in several similar objects from the same level (like {'Task1': {}, + 'Task2': {} ...), all the values found are accumulated in the respective + (flat) list at extDoc[keyName]['values'], which is later to be converted + to the originally expected type for the given field as described in the + Document Template + :param wfObj: Dictionary containing the workflow description + + """ + if isinstance(wfObj, (list, set, tuple)): + for value in wfObj: + self._paramFinder(value) + if isinstance(wfObj, dict): + for key, value in wfObj.items(): + self._paramFinder(value) + for key in self.extDoc.keys(): + if key in wfObj.keys(): + self.extDoc[key]['values'].append(deepcopy(wfObj[key])) + + def _wfParse(self): + """ + Workflow description parser. Given a document template representing all the + keyNames to be searched and a workflow description to search in recursively, + returns all the fields that it can find aggregated according to the rules bellow: + * if the number of found key instances is 0 - sets the default value from + the template. + * if the number of found key instances is 1 - sets the so found value from the + workflow description and converts it back to the form expected and described + in the template (removes the outermost list used for value aggregation) + * if the number of found key instances is > 1 - the values are aggregated + according to the expected types and data structure defined in the + template as follows: + * bool: sets it to True if any of the values found was set to True + * list: chains/flattens all the sub lists into a single list containing + all the values found + * dict: aggregates/flattens all the key-value pairs from all the + dictionaries found into one big dictionary + WARNING: (if an inner keyName happens to be found in multiple + dictionaries from the aggregated list of dictionaries + it will be overwritten with the values from the last + one to be merged into the finally constructed dictionary)! + * str: will be accumulated in a list containing all the values found + WARNING: (will change the expected structure of the field from + a single string to a list of strings)! + + :param wfDescr: Dictionary with the workflow description + :param docTemplate: Document template in the form of a list of tuples as follows: + [('KeyName', DefaultValue, type), + ('KeyName', DefaultValue, type), + ...] + To be used for identifying the fields to be searched for + in the workflow description + """ + + # Convert back the so aggregated extDoc to the original structure: + for keyName, data in self.extDoc.items(): + if len(data['values']) == 0: + self.extDoc[keyName] = deepcopy(data['default']) + elif len(data['values']) == 1: + self.extDoc[keyName] = deepcopy(data['values'][0]) + elif len(data['values']) > 1: + if data['type'] is bool: + self.extDoc[keyName] = any(data['values']) + elif data['type'] is list: + self.extDoc[keyName] = list(set(flattenList(data['values']))) + # WARNING: If it happens this list to be constructed out of elements + # which are instances of unhashable types (e.g. dict, list) + # the set() call will produce an ERR, but this is unlikely + # to happen, see [1] - All the fields we fetch from the + # so nested structure of Task/Step Chain dictionary are + # of hashable types. + # [1] https://github.com/dmwm/WMCore/blob/ed40d33069bdddcd98ed5b8430d5ca6662e5941f/src/python/WMCore/WMSpec/StdSpecs/StdBase.py#L1189 + elif data['type'] is dict: + self.extDoc[keyName] = {} + for item in data['values']: + self.extDoc[keyName].update(item) + elif (isinstance(data['type'], tuple) and (str in data['type'] or unicode in data['type'])) or \ + (data['type'] is str or data['type'] is unicode): + data['values'] = list(set(data['values'])) + if len(data['values']) == 1: + self.extDoc[keyName] = deepcopy(data['values'][0]) + else: + self.extDoc[keyName] = deepcopy(data['values']) class MSRuleCleanerWflow(dict): @@ -15,16 +135,24 @@ class MSRuleCleanerWflow(dict): of the MSRuleCleaner Micro Service. """ - def __init__(self, doc, **kwargs): + def __init__(self, wfDescr, **kwargs): super(MSRuleCleanerWflow, self).__init__(**kwargs) # Search for all the keys we need from the ReqManager workflow description - myDoc = {} - for tup in self.docSchema(): - if tup[0] in doc: - myDoc[tup[0]] = deepcopy(doc[tup[0]]) - else: - myDoc.update({tup[0]: tup[1]}) + wfParser = WfParser(self.docSchema()) + myDoc = wfParser(wfDescr) + + # Convert some fields to lists explicitly: + # NOTE: Those are fields defined as strings in the original workflow + # representation, but may turn into lists during the recursive + # search and we will use them as lists for the rest of the code. + for key in ['DataPileup', 'MCPileup', 'ParentDataset']: + if not isinstance(myDoc[key], list): + if myDoc[key] is None: + myDoc[key] = [] + else: + myDoc[key] = [myDoc[key]] + self.update(myDoc) def docSchema(self): @@ -54,7 +182,15 @@ def docSchema(self): 'ParentageResolved': Bool, 'PlineMarkers': None, 'IsClean': False - 'ForceArchive', False] + 'IsLogDBClean': False, + 'IsArchivalDelayExpired': False, + 'ForceArchive': False, + 'RequestTransition': [], + 'IncludeParents': False + 'DataPileup': [], + 'MCPileup': [], + 'InputDataset': None, + 'ParentDataset': [] } :return: a list of tuples """ @@ -73,7 +209,12 @@ def docSchema(self): ('IsLogDBClean', False, bool), ('IsArchivalDelayExpired', False, bool), ('ForceArchive', False, bool), - ('RequestTransition', [], list)] + ('RequestTransition', [], list), + ('IncludeParents', False, bool), + ('DataPileup', None, (str, unicode)), + ('MCPileup', None, (str, unicode)), + ('InputDataset', None, (str, unicode)), + ('ParentDataset', None, (str, unicode))] # NOTE: ParentageResolved is set by default to True it will be False only if: # - RequestType is StepChain diff --git a/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py b/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py index dd16c72725..4cbcd22c80 100644 --- a/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py +++ b/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py @@ -34,6 +34,17 @@ from Utils.Pipeline import Pipeline, Functor from WMCore.WMException import WMException from WMCore.Services.LogDB.LogDB import LogDB +from WMCore.Services.WMStatsServer.WMStatsServer import WMStatsServer +from WMCore.MicroService.Unified.Common import findParent + + +class MSRuleCleanerResolveParentError(WMException): + """ + WMCore exception class for the MSRuleCleaner module, in WMCore MicroServices, + used to signal if an error occurred during parent dataset resolution step. + """ + def __init__(self, message): + super(MSRuleCleanerResolveParentError, self).__init__(message) class MSRuleCleanerArchivalError(WMException): @@ -81,15 +92,20 @@ def __init__(self, msConfig, logger=None): self.logDB = LogDB(self.msConfig["logDBUrl"], self.msConfig["logDBReporter"], logger=self.logger) + self.wmstatsSvc = WMStatsServer(self.msConfig['wmstatsUrl'], logger=self.logger) # Building all the Pipelines: pName = 'plineMSTrCont' self.plineMSTrCont = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), + Functor(self.setParentDatasets), + Functor(self.getRucioRules, 'container', self.msConfig['rucioMStrAccount']), Functor(self.cleanRucioRules)]) pName = 'plineMSTrBlock' self.plineMSTrBlock = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), + Functor(self.setParentDatasets), + Functor(self.getRucioRules, 'block', self.msConfig['rucioMStrAccount']), Functor(self.cleanRucioRules)]) pName = 'plineAgentCont' self.plineAgentCont = Pipeline(name=pName, @@ -130,6 +146,32 @@ def __init__(self, msConfig, logger=None): self.wfCounters = {'cleaned': {}, 'archived': {'normalArchived': 0, 'forceArchived': 0}} + self.globalLocks = set() + + def getGlobalLocks(self): + """ + Fetches the list of 'globalLocks' from wmstats server and the list of + 'parentLocks' from request manager. Stores/updates the unified set in + the 'globalLocks' instance variable. Returns the resultant unified set. + :return: A union set of the 'globalLocks' and the 'parentLocks' lists + """ + self.logger.info("Fetching globalLocks list from wmstats server.") + try: + globalLocks = set(self.wmstatsSvc.getGlobalLocks()) + except Exception as ex: + msg = "Failed to refresh global locks list for the current polling cycle. Error: %s " + msg += "Skipping this polling cycle." + self.logger.error(msg, str(ex)) + raise ex + self.logger.info("Fetching parentLocks list from reqmgr2 server.") + try: + parentLocks = set(self.reqmgr2.getParentLocks()) + except Exception as ex: + msg = "Failed to refresh parent locks list for the current poling cycle. Error: %s " + msg += "Skipping this polling cycle." + self.logger.error(msg, str(ex)) + raise ex + self.globalLocks = globalLocks | parentLocks def resetCounters(self): """ @@ -152,7 +194,6 @@ def execute(self, reqStatus): self.currThreadIdent = self.currThread.name self.updateReportDict(summary, "thread_id", self.currThreadIdent) self.resetCounters() - self.logger.info("MSRuleCleaner is running in mode: %s.", self.mode) # Build the list of workflows to work on: @@ -167,6 +208,7 @@ def execute(self, reqStatus): # Call _execute() and feed the relevant pipeline with the objects popped from requestRecords try: + self.getGlobalLocks() totalNumRequests, cleanNumRequests, normalArchivedNumRequests, forceArchivedNumRequests = self._execute(requestRecords) msg = "\nNumber of processed workflows: %s." msg += "\nNumber of properly cleaned workflows: %s." @@ -280,6 +322,11 @@ def _dispatchWflow(self, wflow): for pline in self.cleanuplines: try: pline.run(wflow) + except MSRuleCleanerResolveParentError as ex: + msg = "%s: Parentage Resolve Error: %s. " + msg += "Will retry again in the next cycle." + self.logger.error(msg, pline.name, ex.message()) + continue except Exception as ex: msg = "%s: General error from pipeline. Workflow: %s. Error: \n%s. " msg += "\nWill retry again in the next cycle." @@ -528,30 +575,75 @@ def getMSOutputTransferInfo(self, wflow): wflow['TransferDone'] = True return wflow + def setParentDatasets(self, wflow): + """ + Used to resolve parent datasets for a workflow. + :param wflow: A MSRuleCleaner workflow representation + :return: The workflow object + """ + if wflow['InputDataset'] and wflow['IncludeParents']: + childDataset = wflow['InputDataset'] + parentDataset = findParent([childDataset], self.msConfig['dbsUrl']) + # NOTE: If findParent() returned None then the DBS service failed to + # resolve the request (it is considered an ERROR outside WMCore) + if parentDataset.get(childDataset, None) is None: + msg = "Failed to resolve parent dataset for: %s in workflow: %s" % (childDataset, wflow['RequestName']) + raise MSRuleCleanerResolveParentError(msg) + elif parentDataset: + wflow['ParentDataset'] = [parentDataset[childDataset]] + msg = "Found parent %s for input dataset %s in workflow: %s " + self.logger.info(msg, parentDataset, wflow['InputDataset'], wflow['RequestName']) + else: + msg = "Could not find parent for input dataset: %s in workflows: %s" + self.logger.error(msg, wflow['InputDataset'], wflow['RequestName']) + return wflow + def getRucioRules(self, wflow, gran, rucioAcct): """ Queries Rucio and builds the relevant list of blocklevel rules for the given workflow :param wflow: A MSRuleCleaner workflow representation :param gran: Data granularity to search for Rucio rules. Possible values: - 'block' || 'container' + 'block' or 'container' :return: The workflow object """ currPline = wflow['PlineMarkers'][-1] - # Find all the output placement rules created by the agents - for dataCont in wflow['OutputDatasets']: - if gran == 'container': - for rule in self.rucio.listDataRules(dataCont, account=rucioAcct): - wflow['RulesToClean'][currPline].append(rule['id']) - elif gran == 'block': - try: - blocks = self.rucio.getBlocksInContainer(dataCont) - for block in blocks: - for rule in self.rucio.listDataRules(block, account=rucioAcct): - wflow['RulesToClean'][currPline].append(rule['id']) - except WMRucioDIDNotFoundException: - msg = "Container: %s not found in Rucio for workflow: %s." + + # Create the container list to the rucio account map and set the checkGlobalLocks flag. + mapRuleType = {self.msConfig['rucioWmaAccount']: ["OutputDatasets"], + self.msConfig['rucioMStrAccount']: ["InputDataset", "MCPileup", + "DataPileup", "ParentDataset"]} + if rucioAcct == self.msConfig['rucioMStrAccount']: + checkGlobalLocks = True + else: + checkGlobalLocks = False + + # Find all the data placement rules created by the components: + for dataType in mapRuleType[rucioAcct]: + dataList = wflow[dataType] if isinstance(wflow[dataType], list) else [wflow[dataType]] + for dataCont in dataList: + self.logger.debug("getRucioRules: dataCont: %s", pformat(dataCont)) + if checkGlobalLocks and dataCont in self.globalLocks: + msg = "Found dataset: %s in GlobalLocks. NOT considering it for filling the " + msg += "RulesToClean list for both container and block level Rules for workflow: %s!" self.logger.info(msg, dataCont, wflow['RequestName']) + continue + if gran == 'container': + for rule in self.rucio.listDataRules(dataCont, account=rucioAcct): + wflow['RulesToClean'][currPline].append(rule['id']) + msg = "Found %s container-level rule to be deleted for container %s" + self.logger.info(msg, rule['id'], dataCont) + elif gran == 'block': + try: + blocks = self.rucio.getBlocksInContainer(dataCont) + for block in blocks: + for rule in self.rucio.listDataRules(block, account=rucioAcct): + wflow['RulesToClean'][currPline].append(rule['id']) + msg = "Found %s block-level rule to be deleted for container %s" + self.logger.info(msg, rule['id'], dataCont) + except WMRucioDIDNotFoundException: + msg = "Container: %s not found in Rucio for workflow: %s." + self.logger.info(msg, dataCont, wflow['RequestName']) return wflow def cleanRucioRules(self, wflow): @@ -581,12 +673,6 @@ def cleanRucioRules(self, wflow): # Set the cleanup flag: wflow['CleanupStatus'][currPline] = all(delResults) - # ---------------------------------------------------------------------- - # FIXME : To be removed once the plineMSTrBlock && plineMSTrCont are - # developed - if wflow['CleanupStatus'][currPline] in ['plineMSTrBlock', 'plineMSTrCont']: - wflow['CleanupStatus'][currPline] = True - # ---------------------------------------------------------------------- return wflow def getRequestRecords(self, reqStatus): diff --git a/src/python/WMCore/Services/ReqMgr/ReqMgr.py b/src/python/WMCore/Services/ReqMgr/ReqMgr.py index ca636871c7..0f46722e5f 100644 --- a/src/python/WMCore/Services/ReqMgr/ReqMgr.py +++ b/src/python/WMCore/Services/ReqMgr/ReqMgr.py @@ -72,6 +72,18 @@ def _createQuery(self, queryDict): return args.rstrip('&') + def getParentLocks(self): + """ + _getParentLocks_ + + A public method to return the parent locks from ReqMgr. + :returns: A list of datasets + + """ + callname = 'parentlocks' + result = self._getResult(callname, clearCache=True, verb="GET") + return result[0][callname] + def getRequestByNames(self, names): """ diff --git a/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py b/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py index 1d9f4e375c..0f4bc18d63 100644 --- a/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py +++ b/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py @@ -65,6 +65,16 @@ def _createQuery(self, queryDict): return args.rstrip('&') + def getGlobalLocks(self): + """ + _getGlobalLocks_ + + A public method to return the global locks from WMStatServer. + :returns: A list of datasets + """ + callname = 'globallocks' + return self._getResult(callname, clearCache=True, verb="GET") + def getActiveData(self): """ From 362f5760cb0fd99fa833e6724f6220f3cd04d14c Mon Sep 17 00:00:00 2001 From: Todor Ivanov Date: Thu, 11 Feb 2021 16:52:53 +0100 Subject: [PATCH 2/2] Unit tests Unit tests Unit tests --- .../Static/IncludeParentsRequestDump.json | 198 +++++++++++++++ .../requests/Static/MultiPURequestDump.json | 227 +++++++++++++++++ .../requests/Static/ReRecoRequestDump.json | 234 ++++++++++++++++++ .../DataStructs_t/MSRuleCleanerWflow_t.py | 180 +++++++++++--- .../Unified_t/MSRuleCleaner_t.py | 231 +++++++++-------- 5 files changed, 937 insertions(+), 133 deletions(-) create mode 100644 test/data/ReqMgr/requests/Static/IncludeParentsRequestDump.json create mode 100644 test/data/ReqMgr/requests/Static/MultiPURequestDump.json create mode 100644 test/data/ReqMgr/requests/Static/ReRecoRequestDump.json diff --git a/test/data/ReqMgr/requests/Static/IncludeParentsRequestDump.json b/test/data/ReqMgr/requests/Static/IncludeParentsRequestDump.json new file mode 100644 index 0000000000..f5d27508a9 --- /dev/null +++ b/test/data/ReqMgr/requests/Static/IncludeParentsRequestDump.json @@ -0,0 +1,198 @@ +{ + "PrepID": "PrepID-test2inwf-1510737328-RunHLTPhy2017B_RAWAOD", + "MaxMergeEvents": 100000000, + "Comments": { + "CheckList": [ + "StepChain with IncludeParents" + ], + "WorkFlowDesc": [ + "Single step; IncludeParents True and 3 runs whitelisted; auto-splitting" + ] + }, + "Requestor": "amaltaro", + "Step1": { + "InputDataset": "/Cosmics/Commissioning2015-PromptReco-v1/RECO", + "IncludeParents": true, + "GlobalTag": "GR_P_V49", + "RunWhitelist": [ + 234297, + 234321, + 234332 + ], + "StepName": "DQMHLTonRAWAOD_2017", + "ConfigCacheID": "029a0b63329659025f308af53c7e092b", + "EventsPerJob": 14400, + "EventsPerLumi": 14400 + }, + "TotalInputFiles": 21, + "ChainParentageMap": { + "Step1": { + "ParentDset": "/Cosmics/Commissioning2015-PromptReco-v1/RECO", + "ChildDsets": [ + "/Cosmics/Integ_Test-CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11/RAW-RECO" + ] + } + }, + "CustodialGroup": "DataOps", + "MaxMergeSize": 4294967296, + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "SubRequestType": "", + "TrustPUSitelists": false, + "HardTimeout": 129900, + "OpenRunningTimeout": 0, + "LumiList": {}, + "DashboardHost": "cms-jobmon.cern.ch", + "EnableHarvesting": false, + "BlockCloseMaxWaitTime": 66400, + "SiteWhitelist": [ + "T1_US_FNAL", + "T2_CH_CERN" + ], + "BlockCloseMaxSize": 5000000000000, + "DQMConfigCacheID": null, + "DeterministicPileup": true, + "ValidStatus": "PRODUCTION", + "AcquisitionEra": "Integ_Test", + "Campaign": "April2020_Val", + "GlobalTag": "GR_P_V49", + "ScramArch": [ + "slc6_amd64_gcc491" + ], + "CouchDBName": "reqmgr_config_cache", + "NonCustodialGroup": "DataOps", + "DeleteFromSource": false, + "TotalInputEvents": 418808, + "SizePerEvent": 512, + "ConfigCacheID": null, + "StepChain": 1, + "CustodialSubType": "Replica", + "GracePeriod": 300, + "MergedLFNBase": "/store/backfill/1", + "ParentageResolved": false, + "EnableNewStageout": false, + "SubscriptionPriority": "Low", + "RobustMerge": true, + "Team": "testbed-vocms0193", + "AllowOpportunistic": false, + "DQMHarvestUnit": "byRun", + "AutoApproveSubscriptionSites": [], + "RequestWorkflow": "https://cmsweb-testbed.cern.ch/couchdb/reqmgr_workload_cache/amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81/spec", + "BlockCloseMaxFiles": 500, + "VoGroup": "unknown", + "RequestType": "StepChain", + "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader/", + "SiteBlacklist": [], + "RequestStatus": "completed", + "SoftTimeout": 129600, + "Group": "DATAOPS", + "BlockCloseMaxEvents": 25000000, + "CustodialSites": [], + "ProcessingString": "StepChain_InclParents_HG2004_Val_Privv12", + "Multicore": 1, + "TrustSitelists": false, + "UnmergedLFNBase": "/store/unmerged", + "DashboardPort": 8884, + "TimePerEvent": 2, + "DQMSequences": [], + "OutputModulesLFNBases": [ + "/store/unmerged/Integ_Test/Cosmics/RAW-RECO/CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11" + ], + "OutputDatasets": [ + "/Cosmics/Integ_Test-CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11/RAW-RECO" + ], + "PrimaryDataset": null, + "NonCustodialSubType": "Replica", + "RunNumber": 0, + "OverrideCatalog": null, + "PeriodicHarvestInterval": 0, + "TotalEstimatedJobs": 31, + "MaxWaitTime": 86400, + "RequestString": "StepChain_InclParents_April2020_Val", + "RequestorDN": "", + "RequestDate": [ + 2020, + 4, + 14, + 10, + 7, + 13 + ], + "InitialPriority": 420000, + "EventStreams": null, + "Memory": 2300, + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, + "IncludeParents": false, + "VoRole": "unknown", + "PriorityTransition": [ + { + "Priority": 420000, + "DN": "", + "UpdateTime": 1586858833 + } + ], + "CMSSWVersion": "CMSSW_7_3_2", + "NonCustodialSites": [], + "GlobalTagConnect": null, + "MinMergeSize": 2147483648, + "RequestPriority": 420000, + "ProcessingVersion": 11, + "TotalInputLumis": 149, + "FirstEvent": 1, + "RequestName": "amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81", + "CouchURL": "https://cmsweb-testbed.cern.ch/couchdb", + "CouchWorkloadDBName": "reqmgr_workload_cache", + "RequestTransition": [ + { + "Status": "new", + "DN": "", + "UpdateTime": 1586858833 + }, + { + "DN": "", + "Status": "assignment-approved", + "UpdateTime": 1586858834 + }, + { + "DN": "", + "Status": "assigned", + "UpdateTime": 1586858835 + }, + { + "DN": "", + "Status": "staging", + "UpdateTime": 1586859358 + }, + { + "DN": "", + "Status": "staged", + "UpdateTime": 1586859733 + }, + { + "DN": "", + "Status": "acquired", + "UpdateTime": 1586860322 + }, + { + "DN": "", + "Status": "running-open", + "UpdateTime": 1586860927 + }, + { + "DN": "", + "Status": "running-closed", + "UpdateTime": 1586861535 + }, + { + "DN": "", + "Status": "completed", + "UpdateTime": 1586863942 + } + ], + "FirstLumi": 1, + "DQMUploadProxy": null, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "_id": "amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81", + "Dashboard": "integration" +} diff --git a/test/data/ReqMgr/requests/Static/MultiPURequestDump.json b/test/data/ReqMgr/requests/Static/MultiPURequestDump.json new file mode 100644 index 0000000000..b46e6a9191 --- /dev/null +++ b/test/data/ReqMgr/requests/Static/MultiPURequestDump.json @@ -0,0 +1,227 @@ +{ + "PrepID": "TEST-TEST-HIG-RunIISummer15wmLHEGS-00744", + "MaxMergeEvents": 100000000, + "Comments": { + "CheckList": [ + "StepChain: with multiple PU datasets", + "StepChain with TrustPUSitelists enabled" + ], + "WorkFlowDesc": [ + "Two steps, PU for Step1 without input dataset; Step2 using a different PU;", + "Given 100EpL, tweak EpJ to 200 instead of 205; TrustPU enabled for the MinBias" + ] + }, + "Requestor": "tivanov", + "Step2": { + "MCPileup": "/Neutrino_E-10_gun/RunIISpring15PrePremix-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v2-v2/GEN-SIM-DIGI-RAW", + "PrepID": "TEST-Step2-RunIISummer15wmLHEGS-00744", + "GlobalTag": "80X_mcRun2_asymptotic_2016_TrancheIV_v6", + "InputFromOutputModule": "RAWSIMoutput", + "InputStep": "GENSIM", + "SplittingAlgo": "EventAwareLumiBased", + "ScramArch": [ + "slc6_amd64_gcc530" + ], + "ConfigCacheID": "f7e311f2c6b5a0884faea133990edcbf", + "StepName": "DIGI", + "CMSSWVersion": "CMSSW_8_0_21" + }, + "Step1": { + "MCPileup": "/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIIWinter15GS-MCRUN2_71_V1-v1/GEN-SIM", + "PrepID": "TEST-Step1-RunIISummer15wmLHEGS-00744", + "GlobalTag": "MCRUN2_71_V1::All", + "Seeding": "AutomaticSeeding", + "StepName": "GENSIM", + "SplittingAlgo": "EventBased", + "ScramArch": [ + "slc6_amd64_gcc481" + ], + "EventsPerLumi": 100, + "ConfigCacheID": "526ca0745cd309b7cfef9f23b3d43acb", + "EventsPerJob": 200, + "RequestNumEvents": 4000, + "CMSSWVersion": "CMSSW_7_1_25_patch2" + }, + "TotalInputFiles": 0, + "ChainParentageMap": { + "Step2": { + "ParentDset": "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM", + "ChildDsets": [ + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM-RAW" + ] + }, + "Step1": { + "ParentDset": null, + "ChildDsets": [ + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM", + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/LHE" + ] + } + }, + "CustodialGroup": "DataOps", + "MaxMergeSize": 4294967296, + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "SubRequestType": "", + "TrustPUSitelists": true, + "HardTimeout": 129900, + "OpenRunningTimeout": 0, + "LumiList": {}, + "DashboardHost": "cms-jobmon.cern.ch", + "EnableHarvesting": false, + "BlockCloseMaxWaitTime": 66400, + "SiteWhitelist": [ + "T1_US_FNAL", + "T2_CH_CERN" + ], + "GlobalTagConnect": null, + "DQMConfigCacheID": null, + "DeterministicPileup": false, + "ValidStatus": "PRODUCTION", + "AcquisitionEra": "DMWM_Test", + "GlobalTag": "80X_mcRun2_asymptotic_2016_TrancheIV_v6", + "Campaign": "HG2002_Val", + "Multicore": 1, + "CouchDBName": "reqmgr_config_cache", + "NonCustodialGroup": "DataOps", + "DeleteFromSource": false, + "TotalInputEvents": 4000, + "SizePerEvent": 250, + "ConfigCacheID": null, + "StepChain": 2, + "CustodialSubType": "Replica", + "GracePeriod": 300, + "MergedLFNBase": "/store/backfill/1", + "ParentageResolved": false, + "EnableNewStageout": false, + "SubscriptionPriority": "Low", + "RobustMerge": true, + "Team": "testbed-vocms0193", + "AllowOpportunistic": false, + "DQMHarvestUnit": "byRun", + "AutoApproveSubscriptionSites": [], + "RequestWorkflow": "https://cmsweb-testbed.cern.ch/couchdb/reqmgr_workload_cache/tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925/spec", + "BlockCloseMaxFiles": 500, + "VoGroup": "unknown", + "RequestType": "StepChain", + "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/", + "SiteBlacklist": [], + "RequestStatus": "completed", + "SoftTimeout": 129600, + "Group": "DATAOPS", + "BlockCloseMaxEvents": 25000000, + "CustodialSites": [], + "ProcessingString": "SC_MultiPU_HG2002_Val_Todor_v13", + "ScramArch": [ + "slc6_amd64_gcc530" + ], + "TrustSitelists": false, + "UnmergedLFNBase": "/store/unmerged", + "DashboardPort": 8884, + "TimePerEvent": 140, + "DQMSequences": [], + "OutputModulesLFNBases": [ + "/store/unmerged/DMWM_Test/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/GEN-SIM/SC_MultiPU_HG2002_Val_Todor_v13-v20", + "/store/unmerged/DMWM_Test/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/GEN-SIM-RAW/SC_MultiPU_HG2002_Val_Todor_v13-v20", + "/store/unmerged/DMWM_Test/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/LHE/SC_MultiPU_HG2002_Val_Todor_v13-v20" + ], + "OutputDatasets": [ + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM", + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/LHE", + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM-RAW" + ], + "PrimaryDataset": "DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8", + "NonCustodialSubType": "Replica", + "RunNumber": 0, + "OverrideCatalog": null, + "PeriodicHarvestInterval": 0, + "TotalEstimatedJobs": 20, + "MaxWaitTime": 86400, + "RequestString": "SC_MultiPU_HG2002_Val", + "RequestorDN": "", + "RequestDate": [ + 2020, + 1, + 21, + 3, + 36, + 59 + ], + "InitialPriority": 316000, + "EventStreams": null, + "Memory": 2300, + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, + "IncludeParents": false, + "VoRole": "unknown", + "PriorityTransition": [ + { + "Priority": 316000, + "DN": "", + "UpdateTime": 1579577819 + } + ], + "CMSSWVersion": "CMSSW_8_0_21", + "NonCustodialSites": [], + "BlockCloseMaxSize": 5000000000000, + "MinMergeSize": 2147483648, + "RequestPriority": 316000, + "ProcessingVersion": 20, + "TotalInputLumis": 40, + "FirstEvent": 1, + "RequestName": "tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925", + "CouchURL": "https://cmsweb-testbed.cern.ch/couchdb", + "CouchWorkloadDBName": "reqmgr_workload_cache", + "RequestTransition": [ + { + "Status": "new", + "DN": "", + "UpdateTime": 1579577819 + }, + { + "DN": "", + "Status": "assignment-approved", + "UpdateTime": 1579577822 + }, + { + "DN": "", + "Status": "assigned", + "UpdateTime": 1579577822 + }, + { + "DN": "", + "Status": "staging", + "UpdateTime": 1579577895 + }, + { + "DN": "", + "Status": "staged", + "UpdateTime": 1579578050 + }, + { + "DN": "", + "Status": "acquired", + "UpdateTime": 1579578770 + }, + { + "DN": "", + "Status": "running-open", + "UpdateTime": 1579578770 + }, + { + "DN": "", + "Status": "running-closed", + "UpdateTime": 1579579378 + }, + { + "DN": "", + "Status": "completed", + "UpdateTime": 1579587203 + } + ], + "FirstLumi": 1, + "DQMUploadProxy": null, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "_id": "tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925", + "Dashboard": "integration" +} diff --git a/test/data/ReqMgr/requests/Static/ReRecoRequestDump.json b/test/data/ReqMgr/requests/Static/ReRecoRequestDump.json new file mode 100644 index 0000000000..8aba557cc1 --- /dev/null +++ b/test/data/ReqMgr/requests/Static/ReRecoRequestDump.json @@ -0,0 +1,234 @@ +{ + "PrepID": "ReReco-Run2017F-SingleElectron-09Aug2019_UL2017_EcalRecovery-00001", + "MaxMergeEvents": 20000000, + "Comments": "", + "Requestor": "pdmvserv", + "RunBlacklist": [], + "TotalInputFiles": 2729, + "CustodialGroup": "DataOps", + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "SubRequestType": "", + "TrustPUSitelists": false, + "HardTimeout": 159900, + "OpenRunningTimeout": 0, + "LumiList": {}, + "DashboardHost": "cms-jobmon.cern.ch", + "EnableHarvesting": false, + "Scenario": "pp", + "BlockCloseMaxWaitTime": 172800, + "SiteWhitelist": [ + "T1_IT_CNAF", + "T1_ES_PIC", + "T2_CH_CSCS", + "T2_UA_KIPT", + "T2_US_Purdue", + "T2_TW_NCHC", + "T2_UK_SGrid_RALPP", + "T2_FR_GRIF_LLR", + "T2_BE_UCL", + "T2_ES_IFCA", + "T2_BE_IIHE", + "T2_FR_IPHC", + "T2_DE_DESY", + "T1_UK_RAL", + "T1_US_FNAL", + "T2_US_Caltech", + "T2_UK_London_Brunel", + "T2_RU_JINR", + "T2_IT_Pisa", + "T2_US_Vanderbilt", + "T2_IN_TIFR", + "T2_FR_CCIN2P3", + "T1_FR_CCIN2P3", + "T2_US_Florida", + "T2_FR_GRIF_IRFU", + "T2_UK_London_IC", + "T2_IT_Bari", + "T2_US_Nebraska", + "T1_DE_KIT", + "T2_US_UCSD", + "T2_ES_CIEMAT", + "T1_RU_JINR", + "T1_RU_JINR", + "T2_PL_Warsaw", + "T2_US_Wisconsin", + "T2_US_MIT", + "T2_DE_RWTH", + "T2_CH_CERN", + "T2_PT_NCG_Lisbon", + "T2_BR_SPRACE" + ], + "VoRole": "unknown", + "DQMConfigCacheID": "335e35b42a8778432d1e8783300459e1", + "ValidStatus": "PRODUCTION", + "PriorityTransition": [ + { + "Priority": 500000, + "DN": "", + "UpdateTime": 1588759495 + } + ], + "GlobalTag": "106X_dataRun2_v20", + "Campaign": "UltraLegacy2017", + "ScramArch": [ + "slc7_amd64_gcc700" + ], + "CouchDBName": "reqmgr_config_cache", + "NonCustodialGroup": "DataOps", + "EventsPerJob": 9600, + "InputDataset": "/SingleElectron/Run2017F-v1/RAW", + "DeleteFromSource": false, + "TotalInputEvents": 15066632, + "SizePerEvent": 300, + "ConfigCacheID": "335e35b42a8778432d1e878330036b02", + "CustodialSubType": "Replica", + "GracePeriod": 300, + "MergedLFNBase": "/store/data", + "EnableNewStageout": false, + "TransientOutputModules": [], + "SubscriptionPriority": "Low", + "RobustMerge": true, + "SplittingAlgo": "EventAwareLumiBased", + "Team": "production", + "AllowOpportunistic": false, + "DQMHarvestUnit": "byRun", + "AutoApproveSubscriptionSites": [ + "T1_IT_CNAF_Disk" + ], + "RequestWorkflow": "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146/spec", + "BlockBlacklist": [], + "BlockCloseMaxFiles": 500, + "VoGroup": "unknown", + "RequestType": "ReReco", + "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "SiteBlacklist": [], + "RequestStatus": "completed", + "SoftTimeout": 159600, + "Group": "PPD", + "BlockCloseMaxEvents": 200000000, + "CustodialSites": [], + "ProcessingString": "09Aug2019_UL2017_EcalRecovery", + "Multicore": 8, + "TrustSitelists": false, + "UnmergedLFNBase": "/store/unmerged", + "DashboardPort": 8884, + "TimePerEvent": 3, + "DQMSequences": [], + "OutputModulesLFNBases": [ + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/EcalESAlign-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/AOD/09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/HcalCalIterativePhiSym-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/MINIAOD/09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/EcalUncalZElectron-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/EcalUncalWElectron-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/DQMIO/09Aug2019_UL2017_EcalRecovery-v1" + ], + "OutputDatasets": [ + "/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/MINIAOD", + "/SingleElectron/Run2017F-EcalUncalWElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO", + "/SingleElectron/Run2017F-EcalUncalZElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO", + "/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/AOD", + "/SingleElectron/Run2017F-EcalESAlign-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO", + "/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/DQMIO", + "/SingleElectron/Run2017F-HcalCalIterativePhiSym-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO" + ], + "PrimaryDataset": null, + "NonCustodialSubType": "Replica", + "BlockCloseMaxSize": 5000000000000, + "MaxMergeSize": 4294967296, + "OverrideCatalog": null, + "PeriodicHarvestInterval": 0, + "TotalEstimatedJobs": 1575, + "MaxWaitTime": 86400, + "RequestString": "Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery", + "RequestorDN": "", + "RunWhitelist": [ + 306459, + 306456, + 305064, + 306125 + ], + "RequestDate": [ + 2020, + 5, + 6, + 10, + 4, + 55 + ], + "InitialPriority": 500000, + "RunNumber": 0, + "EventStreams": null, + "Memory": 15900, + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/PRODUCTION" + }, + "FilesPerJob": 1, + "IncludeParents": false, + "BlockWhitelist": [], + "LumisPerJob": 8, + "AcquisitionEra": "Run2017F", + "CMSSWVersion": "CMSSW_10_6_11", + "NonCustodialSites": [ + "T1_IT_CNAF_Disk" + ], + "GlobalTagConnect": null, + "MinMergeSize": 2147483648, + "RequestPriority": 500000, + "ProcessingVersion": 1, + "TotalInputLumis": 8110, + "RequestName": "pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146", + "CouchURL": "https://cmsweb.cern.ch/couchdb", + "CouchWorkloadDBName": "reqmgr_workload_cache", + "RequestTransition": [ + { + "Status": "new", + "DN": "", + "UpdateTime": 1588759495 + }, + { + "DN": "", + "Status": "assignment-approved", + "UpdateTime": 1588759500 + }, + { + "DN": "", + "Status": "assigned", + "UpdateTime": 1588760963 + }, + { + "DN": "", + "Status": "staging", + "UpdateTime": 1588761258 + }, + { + "DN": "", + "Status": "staged", + "UpdateTime": 1588761688 + }, + { + "DN": "", + "Status": "acquired", + "UpdateTime": 1588762166 + }, + { + "DN": "", + "Status": "running-open", + "UpdateTime": 1588763098 + }, + { + "DN": "", + "Status": "running-closed", + "UpdateTime": 1588775354 + }, + { + "DN": "", + "Status": "completed", + "UpdateTime": 1589041121 + } + ], + "DQMUploadProxy": null, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "_id": "pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146", + "Dashboard": "reprocessing" + } diff --git a/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py b/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py index c1b3dc58f7..dc0c32a151 100644 --- a/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py +++ b/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py @@ -26,6 +26,9 @@ def setUp(self): self.maxDiff = None self.taskChainFile = getTestFile('data/ReqMgr/requests/Static/TaskChainRequestDump.json') self.stepChainFile = getTestFile('data/ReqMgr/requests/Static/StepChainRequestDump.json') + self.reRecoFile = getTestFile('data/ReqMgr/requests/Static/ReRecoRequestDump.json') + self.includeParentsFile = getTestFile('data/ReqMgr/requests/Static/IncludeParentsRequestDump.json') + self.multiPUFile = getTestFile('data/ReqMgr/requests/Static/MultiPURequestDump.json') self.reqRecordsFile = getTestFile('data/ReqMgr/requests/Static/BatchRequestsDump.json') with open(self.reqRecordsFile) as fd: self.reqRecords = json.load(fd) @@ -33,7 +36,12 @@ def setUp(self): self.taskChainReq = json.load(fd) with open(self.stepChainFile) as fd: self.stepChainReq = json.load(fd) - + with open(self.reRecoFile) as fd: + self.reRecoReq = json.load(fd) + with open(self.includeParentsFile) as fd: + self.includeParentsReq = json.load(fd) + with open(self.multiPUFile) as fd: + self.multiPUReq = json.load(fd) super(MSRuleCleanerWflowTest, self).setUp() def testTaskChainDefaults(self): @@ -56,63 +64,167 @@ def testTaskChainDefaults(self): self.assertEqual(wflow["IsArchivalDelayExpired"], False) self.assertEqual(wflow["ForceArchive"], False) self.assertEqual(wflow["RequestTransition"], []) + self.assertEqual(wflow['IncludeParents'], False) + self.assertEqual(wflow['DataPileup'], []) + self.assertEqual(wflow['MCPileup'], []) + self.assertEqual(wflow['InputDataset'], None) + self.assertEqual(wflow['ParentDataset'], []) def testTaskChain(self): # Test Taskchain: wflow = MSRuleCleanerWflow(self.taskChainReq) expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': False, 'IsClean': False, 'IsLogDBClean': False, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': None, 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - 'RequestTransition': [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {}, 'TargetStatus': None, 'TransferDone': False} self.assertDictEqual(wflow, expectedWflow) + def testReReco(self): + # Test ReReco workflow: + wflow = MSRuleCleanerWflow(self.reRecoReq) + expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/SingleElectron/Run2017F-v1/RAW', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/MINIAOD', + u'/SingleElectron/Run2017F-EcalUncalWElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO', + u'/SingleElectron/Run2017F-EcalUncalZElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO', + u'/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/AOD', + u'/SingleElectron/Run2017F-EcalESAlign-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO', + u'/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/DQMIO', + u'/SingleElectron/Run2017F-HcalCalIterativePhiSym-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO'], + 'ParentDataset': [], + 'ParentageResolved': True, + 'PlineMarkers': None, + 'RequestName': u'pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146', + 'RequestStatus': u'completed', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1588759495}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1588759500}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1588760963}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1588761258}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1588761688}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1588762166}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1588763098}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1588775354}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1589041121}], + 'RequestType': u'ReReco', + 'RulesToClean': {}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + + def testIncludeParents(self): + # Test include parents:: + wflow = MSRuleCleanerWflow(self.includeParentsReq) + expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': True, + 'InputDataset': u'/Cosmics/Commissioning2015-PromptReco-v1/RECO', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/Cosmics/Integ_Test-CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11/RAW-RECO'], + 'ParentDataset': [], + 'ParentageResolved': False, + 'PlineMarkers': None, + 'RequestName': u'amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81', + 'RequestStatus': u'completed', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1586858833}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1586858834}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1586858835}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1586859358}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1586859733}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1586860322}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1586860927}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1586861535}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1586863942}], + 'RequestType': u'StepChain', + 'RulesToClean': {}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + + def testMultiPU(self): + # Test workflow with multiple pileups:: + wflow = MSRuleCleanerWflow(self.multiPUReq) + expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': None, + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [u'/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIIWinter15GS-MCRUN2_71_V1-v1/GEN-SIM', + u'/Neutrino_E-10_gun/RunIISpring15PrePremix-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v2-v2/GEN-SIM-DIGI-RAW'], + 'OutputDatasets': [u'/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM', + u'/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/LHE', + u'/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM-RAW'], + 'ParentDataset': [], + 'ParentageResolved': False, + 'PlineMarkers': None, + 'RequestName': u'tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925', + 'RequestStatus': u'completed', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1579577819}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1579577822}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1579577822}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1579577895}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1579578050}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1579578770}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1579578770}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1579579378}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1579587203}], + 'RequestType': u'StepChain', + 'RulesToClean': {}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + if __name__ == '__main__': unittest.main() diff --git a/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py b/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py index 56704539d4..5f4462e035 100644 --- a/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py +++ b/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py @@ -85,52 +85,35 @@ def testPipelineAgentBlock(self): wflow = MSRuleCleanerWflow(self.taskChainReq) self.msRuleCleaner.plineAgentBlock.run(wflow) expectedWflow = {'CleanupStatus': {'plineAgentBlock': True}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': False, 'IsClean': False, 'IsLogDBClean': False, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': ['plineAgentBlock'], 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - "RequestTransition": [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', u'Status': u'new', u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {'plineAgentBlock': []}, 'TargetStatus': None, @@ -142,58 +125,125 @@ def testPipelineAgentCont(self): wflow = MSRuleCleanerWflow(self.taskChainReq) self.msRuleCleaner.plineAgentCont.run(wflow) expectedWflow = {'CleanupStatus': {'plineAgentCont': True}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': False, 'IsClean': False, 'IsLogDBClean': False, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': ['plineAgentCont'], 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - "RequestTransition": [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', u'Status': u'new', u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {'plineAgentCont': []}, 'TargetStatus': None, 'TransferDone': False} self.assertDictEqual(wflow, expectedWflow) + def testPipelineMSTrBlock(self): + # Test plineAgentCont + wflow = MSRuleCleanerWflow(self.taskChainReq) + self.msRuleCleaner.plineMSTrBlock.run(wflow) + expectedWflow = {'CleanupStatus': {'plineMSTrBlock': True}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', + u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', + u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], + 'ParentageResolved': True, + 'PlineMarkers': ['plineMSTrBlock'], + 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', + 'RequestStatus': u'announced', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], + 'RequestType': u'TaskChain', + 'RulesToClean': {'plineMSTrBlock': []}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + + def testPipelineMSTrCont(self): + # Test plineAgentCont + wflow = MSRuleCleanerWflow(self.taskChainReq) + self.msRuleCleaner.plineMSTrCont.run(wflow) + expectedWflow = {'CleanupStatus': {'plineMSTrCont': True}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', + u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', + u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], + 'ParentageResolved': True, + 'PlineMarkers': ['plineMSTrCont'], + 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', + 'RequestStatus': u'announced', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], + 'RequestType': u'TaskChain', + 'RulesToClean': {'plineMSTrCont': []}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + def testPipelineArchive(self): # Test plineAgentCont wflow = MSRuleCleanerWflow(self.taskChainReq) @@ -210,15 +260,20 @@ def testPipelineArchive(self): with self.assertRaises(MSRuleCleanerArchivalSkip): self.msRuleCleaner.plineArchive.run(wflow) expectedWflow = {'CleanupStatus': {'plineAgentBlock': True, 'plineAgentCont': True}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': True, 'IsClean': True, 'IsLogDBClean': True, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': ['plineArchive', 'plineAgentBlock', @@ -226,39 +281,17 @@ def testPipelineArchive(self): 'plineArchive'], 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - "RequestTransition": [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', u'Status': u'new', u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {'plineAgentBlock': [], 'plineAgentCont': []}, 'TargetStatus': 'normal-archived',