diff --git a/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py b/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py index b56c32ffdf..10af68f094 100644 --- a/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py +++ b/src/python/WMCore/MicroService/DataStructs/MSRuleCleanerWflow.py @@ -7,6 +7,126 @@ from __future__ import division, print_function from copy import deepcopy +from Utils.IteratorTools import flattenList + + +class WfParser(object): + """ + Workflow description parser class. + """ + def __init__(self, docSchema): + """ + The init method for the Workflow parser class. + :param docSchema: Document template in the form of a list of tuples as follows: + [('KeyName', DefaultValue, type), + ('KeyName', DefaultValue, type), + ...] + To be used for identifying the fields to be searched for + in the workflow description + + """ + self.extDoc = {} + for tup in docSchema: + self.extDoc[tup[0]] = {'keyName': tup[0], + 'values': list(), + 'default': tup[1], + 'type': tup[2]} + + def __call__(self, wfDescr): + """ + The Call method for the Workflow parser class. + """ + self._paramFinder(wfDescr) + self._wfParse() + return self.extDoc + + def _paramFinder(self, wfObj): + """ + Private method used to recursively traverse a workflow description + and search for all the keyNames defined in the extDoc auxiliary data + structure. If a 'keyName' happens to be present in several nested levels, + or in several similar objects from the same level (like {'Task1': {}, + 'Task2': {} ...), all the values found are accumulated in the respective + (flat) list at extDoc[keyName]['values'], which is later to be converted + to the originally expected type for the given field as described in the + Document Template + :param wfObj: Dictionary containing the workflow description + + """ + if isinstance(wfObj, (list, set, tuple)): + for value in wfObj: + self._paramFinder(value) + if isinstance(wfObj, dict): + for key, value in wfObj.items(): + self._paramFinder(value) + for key in self.extDoc.keys(): + if key in wfObj.keys(): + self.extDoc[key]['values'].append(deepcopy(wfObj[key])) + + def _wfParse(self): + """ + Workflow description parser. Given a document template representing all the + keyNames to be searched and a workflow description to search in recursively, + returns all the fields that it can find aggregated according to the rules bellow: + * if the number of found key instances is 0 - sets the default value from + the template. + * if the number of found key instances is 1 - sets the so found value from the + workflow description and converts it back to the form expected and described + in the template (removes the outermost list used for value aggregation) + * if the number of found key instances is > 1 - the values are aggregated + according to the expected types and data structure defined in the + template as follows: + * bool: sets it to True if any of the values found was set to True + * list: chains/flattens all the sub lists into a single list containing + all the values found + * dict: aggregates/flattens all the key-value pairs from all the + dictionaries found into one big dictionary + WARNING: (if an inner keyName happens to be found in multiple + dictionaries from the aggregated list of dictionaries + it will be overwritten with the values from the last + one to be merged into the finally constructed dictionary)! + * str: will be accumulated in a list containing all the values found + WARNING: (will change the expected structure of the field from + a single string to a list of strings)! + + :param wfDescr: Dictionary with the workflow description + :param docTemplate: Document template in the form of a list of tuples as follows: + [('KeyName', DefaultValue, type), + ('KeyName', DefaultValue, type), + ...] + To be used for identifying the fields to be searched for + in the workflow description + """ + + # Convert back the so aggregated extDoc to the original structure: + for keyName, data in self.extDoc.items(): + if len(data['values']) == 0: + self.extDoc[keyName] = deepcopy(data['default']) + elif len(data['values']) == 1: + self.extDoc[keyName] = deepcopy(data['values'][0]) + elif len(data['values']) > 1: + if data['type'] is bool: + self.extDoc[keyName] = any(data['values']) + elif data['type'] is list: + self.extDoc[keyName] = list(set(flattenList(data['values']))) + # WARNING: If it happens this list to be constructed out of elements + # which are instances of unhashable types (e.g. dict, list) + # the set() call will produce an ERR, but this is unlikely + # to happen, see [1] - All the fields we fetch from the + # so nested structure of Task/Step Chain dictionary are + # of hashable types. + # [1] https://github.com/dmwm/WMCore/blob/ed40d33069bdddcd98ed5b8430d5ca6662e5941f/src/python/WMCore/WMSpec/StdSpecs/StdBase.py#L1189 + elif data['type'] is dict: + self.extDoc[keyName] = {} + for item in data['values']: + self.extDoc[keyName].update(item) + elif (isinstance(data['type'], tuple) and (str in data['type'] or unicode in data['type'])) or \ + (data['type'] is str or data['type'] is unicode): + data['values'] = list(set(data['values'])) + if len(data['values']) == 1: + self.extDoc[keyName] = deepcopy(data['values'][0]) + else: + self.extDoc[keyName] = deepcopy(data['values']) class MSRuleCleanerWflow(dict): @@ -15,16 +135,24 @@ class MSRuleCleanerWflow(dict): of the MSRuleCleaner Micro Service. """ - def __init__(self, doc, **kwargs): + def __init__(self, wfDescr, **kwargs): super(MSRuleCleanerWflow, self).__init__(**kwargs) # Search for all the keys we need from the ReqManager workflow description - myDoc = {} - for tup in self.docSchema(): - if tup[0] in doc: - myDoc[tup[0]] = deepcopy(doc[tup[0]]) - else: - myDoc.update({tup[0]: tup[1]}) + wfParser = WfParser(self.docSchema()) + myDoc = wfParser(wfDescr) + + # Convert some fields to lists explicitly: + # NOTE: Those are fields defined as strings in the original workflow + # representation, but may turn into lists during the recursive + # search and we will use them as lists for the rest of the code. + for key in ['DataPileup', 'MCPileup', 'ParentDataset']: + if not isinstance(myDoc[key], list): + if myDoc[key] is None: + myDoc[key] = [] + else: + myDoc[key] = [myDoc[key]] + self.update(myDoc) def docSchema(self): @@ -54,7 +182,15 @@ def docSchema(self): 'ParentageResolved': Bool, 'PlineMarkers': None, 'IsClean': False - 'ForceArchive', False] + 'IsLogDBClean': False, + 'IsArchivalDelayExpired': False, + 'ForceArchive': False, + 'RequestTransition': [], + 'IncludeParents': False + 'DataPileup': [], + 'MCPileup': [], + 'InputDataset': None, + 'ParentDataset': [] } :return: a list of tuples """ @@ -73,7 +209,12 @@ def docSchema(self): ('IsLogDBClean', False, bool), ('IsArchivalDelayExpired', False, bool), ('ForceArchive', False, bool), - ('RequestTransition', [], list)] + ('RequestTransition', [], list), + ('IncludeParents', False, bool), + ('DataPileup', None, (str, unicode)), + ('MCPileup', None, (str, unicode)), + ('InputDataset', None, (str, unicode)), + ('ParentDataset', None, (str, unicode))] # NOTE: ParentageResolved is set by default to True it will be False only if: # - RequestType is StepChain diff --git a/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py b/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py index dd16c72725..4cbcd22c80 100644 --- a/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py +++ b/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py @@ -34,6 +34,17 @@ from Utils.Pipeline import Pipeline, Functor from WMCore.WMException import WMException from WMCore.Services.LogDB.LogDB import LogDB +from WMCore.Services.WMStatsServer.WMStatsServer import WMStatsServer +from WMCore.MicroService.Unified.Common import findParent + + +class MSRuleCleanerResolveParentError(WMException): + """ + WMCore exception class for the MSRuleCleaner module, in WMCore MicroServices, + used to signal if an error occurred during parent dataset resolution step. + """ + def __init__(self, message): + super(MSRuleCleanerResolveParentError, self).__init__(message) class MSRuleCleanerArchivalError(WMException): @@ -81,15 +92,20 @@ def __init__(self, msConfig, logger=None): self.logDB = LogDB(self.msConfig["logDBUrl"], self.msConfig["logDBReporter"], logger=self.logger) + self.wmstatsSvc = WMStatsServer(self.msConfig['wmstatsUrl'], logger=self.logger) # Building all the Pipelines: pName = 'plineMSTrCont' self.plineMSTrCont = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), + Functor(self.setParentDatasets), + Functor(self.getRucioRules, 'container', self.msConfig['rucioMStrAccount']), Functor(self.cleanRucioRules)]) pName = 'plineMSTrBlock' self.plineMSTrBlock = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), + Functor(self.setParentDatasets), + Functor(self.getRucioRules, 'block', self.msConfig['rucioMStrAccount']), Functor(self.cleanRucioRules)]) pName = 'plineAgentCont' self.plineAgentCont = Pipeline(name=pName, @@ -130,6 +146,32 @@ def __init__(self, msConfig, logger=None): self.wfCounters = {'cleaned': {}, 'archived': {'normalArchived': 0, 'forceArchived': 0}} + self.globalLocks = set() + + def getGlobalLocks(self): + """ + Fetches the list of 'globalLocks' from wmstats server and the list of + 'parentLocks' from request manager. Stores/updates the unified set in + the 'globalLocks' instance variable. Returns the resultant unified set. + :return: A union set of the 'globalLocks' and the 'parentLocks' lists + """ + self.logger.info("Fetching globalLocks list from wmstats server.") + try: + globalLocks = set(self.wmstatsSvc.getGlobalLocks()) + except Exception as ex: + msg = "Failed to refresh global locks list for the current polling cycle. Error: %s " + msg += "Skipping this polling cycle." + self.logger.error(msg, str(ex)) + raise ex + self.logger.info("Fetching parentLocks list from reqmgr2 server.") + try: + parentLocks = set(self.reqmgr2.getParentLocks()) + except Exception as ex: + msg = "Failed to refresh parent locks list for the current poling cycle. Error: %s " + msg += "Skipping this polling cycle." + self.logger.error(msg, str(ex)) + raise ex + self.globalLocks = globalLocks | parentLocks def resetCounters(self): """ @@ -152,7 +194,6 @@ def execute(self, reqStatus): self.currThreadIdent = self.currThread.name self.updateReportDict(summary, "thread_id", self.currThreadIdent) self.resetCounters() - self.logger.info("MSRuleCleaner is running in mode: %s.", self.mode) # Build the list of workflows to work on: @@ -167,6 +208,7 @@ def execute(self, reqStatus): # Call _execute() and feed the relevant pipeline with the objects popped from requestRecords try: + self.getGlobalLocks() totalNumRequests, cleanNumRequests, normalArchivedNumRequests, forceArchivedNumRequests = self._execute(requestRecords) msg = "\nNumber of processed workflows: %s." msg += "\nNumber of properly cleaned workflows: %s." @@ -280,6 +322,11 @@ def _dispatchWflow(self, wflow): for pline in self.cleanuplines: try: pline.run(wflow) + except MSRuleCleanerResolveParentError as ex: + msg = "%s: Parentage Resolve Error: %s. " + msg += "Will retry again in the next cycle." + self.logger.error(msg, pline.name, ex.message()) + continue except Exception as ex: msg = "%s: General error from pipeline. Workflow: %s. Error: \n%s. " msg += "\nWill retry again in the next cycle." @@ -528,30 +575,75 @@ def getMSOutputTransferInfo(self, wflow): wflow['TransferDone'] = True return wflow + def setParentDatasets(self, wflow): + """ + Used to resolve parent datasets for a workflow. + :param wflow: A MSRuleCleaner workflow representation + :return: The workflow object + """ + if wflow['InputDataset'] and wflow['IncludeParents']: + childDataset = wflow['InputDataset'] + parentDataset = findParent([childDataset], self.msConfig['dbsUrl']) + # NOTE: If findParent() returned None then the DBS service failed to + # resolve the request (it is considered an ERROR outside WMCore) + if parentDataset.get(childDataset, None) is None: + msg = "Failed to resolve parent dataset for: %s in workflow: %s" % (childDataset, wflow['RequestName']) + raise MSRuleCleanerResolveParentError(msg) + elif parentDataset: + wflow['ParentDataset'] = [parentDataset[childDataset]] + msg = "Found parent %s for input dataset %s in workflow: %s " + self.logger.info(msg, parentDataset, wflow['InputDataset'], wflow['RequestName']) + else: + msg = "Could not find parent for input dataset: %s in workflows: %s" + self.logger.error(msg, wflow['InputDataset'], wflow['RequestName']) + return wflow + def getRucioRules(self, wflow, gran, rucioAcct): """ Queries Rucio and builds the relevant list of blocklevel rules for the given workflow :param wflow: A MSRuleCleaner workflow representation :param gran: Data granularity to search for Rucio rules. Possible values: - 'block' || 'container' + 'block' or 'container' :return: The workflow object """ currPline = wflow['PlineMarkers'][-1] - # Find all the output placement rules created by the agents - for dataCont in wflow['OutputDatasets']: - if gran == 'container': - for rule in self.rucio.listDataRules(dataCont, account=rucioAcct): - wflow['RulesToClean'][currPline].append(rule['id']) - elif gran == 'block': - try: - blocks = self.rucio.getBlocksInContainer(dataCont) - for block in blocks: - for rule in self.rucio.listDataRules(block, account=rucioAcct): - wflow['RulesToClean'][currPline].append(rule['id']) - except WMRucioDIDNotFoundException: - msg = "Container: %s not found in Rucio for workflow: %s." + + # Create the container list to the rucio account map and set the checkGlobalLocks flag. + mapRuleType = {self.msConfig['rucioWmaAccount']: ["OutputDatasets"], + self.msConfig['rucioMStrAccount']: ["InputDataset", "MCPileup", + "DataPileup", "ParentDataset"]} + if rucioAcct == self.msConfig['rucioMStrAccount']: + checkGlobalLocks = True + else: + checkGlobalLocks = False + + # Find all the data placement rules created by the components: + for dataType in mapRuleType[rucioAcct]: + dataList = wflow[dataType] if isinstance(wflow[dataType], list) else [wflow[dataType]] + for dataCont in dataList: + self.logger.debug("getRucioRules: dataCont: %s", pformat(dataCont)) + if checkGlobalLocks and dataCont in self.globalLocks: + msg = "Found dataset: %s in GlobalLocks. NOT considering it for filling the " + msg += "RulesToClean list for both container and block level Rules for workflow: %s!" self.logger.info(msg, dataCont, wflow['RequestName']) + continue + if gran == 'container': + for rule in self.rucio.listDataRules(dataCont, account=rucioAcct): + wflow['RulesToClean'][currPline].append(rule['id']) + msg = "Found %s container-level rule to be deleted for container %s" + self.logger.info(msg, rule['id'], dataCont) + elif gran == 'block': + try: + blocks = self.rucio.getBlocksInContainer(dataCont) + for block in blocks: + for rule in self.rucio.listDataRules(block, account=rucioAcct): + wflow['RulesToClean'][currPline].append(rule['id']) + msg = "Found %s block-level rule to be deleted for container %s" + self.logger.info(msg, rule['id'], dataCont) + except WMRucioDIDNotFoundException: + msg = "Container: %s not found in Rucio for workflow: %s." + self.logger.info(msg, dataCont, wflow['RequestName']) return wflow def cleanRucioRules(self, wflow): @@ -581,12 +673,6 @@ def cleanRucioRules(self, wflow): # Set the cleanup flag: wflow['CleanupStatus'][currPline] = all(delResults) - # ---------------------------------------------------------------------- - # FIXME : To be removed once the plineMSTrBlock && plineMSTrCont are - # developed - if wflow['CleanupStatus'][currPline] in ['plineMSTrBlock', 'plineMSTrCont']: - wflow['CleanupStatus'][currPline] = True - # ---------------------------------------------------------------------- return wflow def getRequestRecords(self, reqStatus): diff --git a/src/python/WMCore/Services/ReqMgr/ReqMgr.py b/src/python/WMCore/Services/ReqMgr/ReqMgr.py index ca636871c7..0f46722e5f 100644 --- a/src/python/WMCore/Services/ReqMgr/ReqMgr.py +++ b/src/python/WMCore/Services/ReqMgr/ReqMgr.py @@ -72,6 +72,18 @@ def _createQuery(self, queryDict): return args.rstrip('&') + def getParentLocks(self): + """ + _getParentLocks_ + + A public method to return the parent locks from ReqMgr. + :returns: A list of datasets + + """ + callname = 'parentlocks' + result = self._getResult(callname, clearCache=True, verb="GET") + return result[0][callname] + def getRequestByNames(self, names): """ diff --git a/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py b/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py index 1d9f4e375c..0f4bc18d63 100644 --- a/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py +++ b/src/python/WMCore/Services/WMStatsServer/WMStatsServer.py @@ -65,6 +65,16 @@ def _createQuery(self, queryDict): return args.rstrip('&') + def getGlobalLocks(self): + """ + _getGlobalLocks_ + + A public method to return the global locks from WMStatServer. + :returns: A list of datasets + """ + callname = 'globallocks' + return self._getResult(callname, clearCache=True, verb="GET") + def getActiveData(self): """ diff --git a/test/data/ReqMgr/requests/Static/IncludeParentsRequestDump.json b/test/data/ReqMgr/requests/Static/IncludeParentsRequestDump.json new file mode 100644 index 0000000000..f5d27508a9 --- /dev/null +++ b/test/data/ReqMgr/requests/Static/IncludeParentsRequestDump.json @@ -0,0 +1,198 @@ +{ + "PrepID": "PrepID-test2inwf-1510737328-RunHLTPhy2017B_RAWAOD", + "MaxMergeEvents": 100000000, + "Comments": { + "CheckList": [ + "StepChain with IncludeParents" + ], + "WorkFlowDesc": [ + "Single step; IncludeParents True and 3 runs whitelisted; auto-splitting" + ] + }, + "Requestor": "amaltaro", + "Step1": { + "InputDataset": "/Cosmics/Commissioning2015-PromptReco-v1/RECO", + "IncludeParents": true, + "GlobalTag": "GR_P_V49", + "RunWhitelist": [ + 234297, + 234321, + 234332 + ], + "StepName": "DQMHLTonRAWAOD_2017", + "ConfigCacheID": "029a0b63329659025f308af53c7e092b", + "EventsPerJob": 14400, + "EventsPerLumi": 14400 + }, + "TotalInputFiles": 21, + "ChainParentageMap": { + "Step1": { + "ParentDset": "/Cosmics/Commissioning2015-PromptReco-v1/RECO", + "ChildDsets": [ + "/Cosmics/Integ_Test-CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11/RAW-RECO" + ] + } + }, + "CustodialGroup": "DataOps", + "MaxMergeSize": 4294967296, + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "SubRequestType": "", + "TrustPUSitelists": false, + "HardTimeout": 129900, + "OpenRunningTimeout": 0, + "LumiList": {}, + "DashboardHost": "cms-jobmon.cern.ch", + "EnableHarvesting": false, + "BlockCloseMaxWaitTime": 66400, + "SiteWhitelist": [ + "T1_US_FNAL", + "T2_CH_CERN" + ], + "BlockCloseMaxSize": 5000000000000, + "DQMConfigCacheID": null, + "DeterministicPileup": true, + "ValidStatus": "PRODUCTION", + "AcquisitionEra": "Integ_Test", + "Campaign": "April2020_Val", + "GlobalTag": "GR_P_V49", + "ScramArch": [ + "slc6_amd64_gcc491" + ], + "CouchDBName": "reqmgr_config_cache", + "NonCustodialGroup": "DataOps", + "DeleteFromSource": false, + "TotalInputEvents": 418808, + "SizePerEvent": 512, + "ConfigCacheID": null, + "StepChain": 1, + "CustodialSubType": "Replica", + "GracePeriod": 300, + "MergedLFNBase": "/store/backfill/1", + "ParentageResolved": false, + "EnableNewStageout": false, + "SubscriptionPriority": "Low", + "RobustMerge": true, + "Team": "testbed-vocms0193", + "AllowOpportunistic": false, + "DQMHarvestUnit": "byRun", + "AutoApproveSubscriptionSites": [], + "RequestWorkflow": "https://cmsweb-testbed.cern.ch/couchdb/reqmgr_workload_cache/amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81/spec", + "BlockCloseMaxFiles": 500, + "VoGroup": "unknown", + "RequestType": "StepChain", + "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader/", + "SiteBlacklist": [], + "RequestStatus": "completed", + "SoftTimeout": 129600, + "Group": "DATAOPS", + "BlockCloseMaxEvents": 25000000, + "CustodialSites": [], + "ProcessingString": "StepChain_InclParents_HG2004_Val_Privv12", + "Multicore": 1, + "TrustSitelists": false, + "UnmergedLFNBase": "/store/unmerged", + "DashboardPort": 8884, + "TimePerEvent": 2, + "DQMSequences": [], + "OutputModulesLFNBases": [ + "/store/unmerged/Integ_Test/Cosmics/RAW-RECO/CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11" + ], + "OutputDatasets": [ + "/Cosmics/Integ_Test-CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11/RAW-RECO" + ], + "PrimaryDataset": null, + "NonCustodialSubType": "Replica", + "RunNumber": 0, + "OverrideCatalog": null, + "PeriodicHarvestInterval": 0, + "TotalEstimatedJobs": 31, + "MaxWaitTime": 86400, + "RequestString": "StepChain_InclParents_April2020_Val", + "RequestorDN": "", + "RequestDate": [ + 2020, + 4, + 14, + 10, + 7, + 13 + ], + "InitialPriority": 420000, + "EventStreams": null, + "Memory": 2300, + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, + "IncludeParents": false, + "VoRole": "unknown", + "PriorityTransition": [ + { + "Priority": 420000, + "DN": "", + "UpdateTime": 1586858833 + } + ], + "CMSSWVersion": "CMSSW_7_3_2", + "NonCustodialSites": [], + "GlobalTagConnect": null, + "MinMergeSize": 2147483648, + "RequestPriority": 420000, + "ProcessingVersion": 11, + "TotalInputLumis": 149, + "FirstEvent": 1, + "RequestName": "amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81", + "CouchURL": "https://cmsweb-testbed.cern.ch/couchdb", + "CouchWorkloadDBName": "reqmgr_workload_cache", + "RequestTransition": [ + { + "Status": "new", + "DN": "", + "UpdateTime": 1586858833 + }, + { + "DN": "", + "Status": "assignment-approved", + "UpdateTime": 1586858834 + }, + { + "DN": "", + "Status": "assigned", + "UpdateTime": 1586858835 + }, + { + "DN": "", + "Status": "staging", + "UpdateTime": 1586859358 + }, + { + "DN": "", + "Status": "staged", + "UpdateTime": 1586859733 + }, + { + "DN": "", + "Status": "acquired", + "UpdateTime": 1586860322 + }, + { + "DN": "", + "Status": "running-open", + "UpdateTime": 1586860927 + }, + { + "DN": "", + "Status": "running-closed", + "UpdateTime": 1586861535 + }, + { + "DN": "", + "Status": "completed", + "UpdateTime": 1586863942 + } + ], + "FirstLumi": 1, + "DQMUploadProxy": null, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "_id": "amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81", + "Dashboard": "integration" +} diff --git a/test/data/ReqMgr/requests/Static/MultiPURequestDump.json b/test/data/ReqMgr/requests/Static/MultiPURequestDump.json new file mode 100644 index 0000000000..b46e6a9191 --- /dev/null +++ b/test/data/ReqMgr/requests/Static/MultiPURequestDump.json @@ -0,0 +1,227 @@ +{ + "PrepID": "TEST-TEST-HIG-RunIISummer15wmLHEGS-00744", + "MaxMergeEvents": 100000000, + "Comments": { + "CheckList": [ + "StepChain: with multiple PU datasets", + "StepChain with TrustPUSitelists enabled" + ], + "WorkFlowDesc": [ + "Two steps, PU for Step1 without input dataset; Step2 using a different PU;", + "Given 100EpL, tweak EpJ to 200 instead of 205; TrustPU enabled for the MinBias" + ] + }, + "Requestor": "tivanov", + "Step2": { + "MCPileup": "/Neutrino_E-10_gun/RunIISpring15PrePremix-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v2-v2/GEN-SIM-DIGI-RAW", + "PrepID": "TEST-Step2-RunIISummer15wmLHEGS-00744", + "GlobalTag": "80X_mcRun2_asymptotic_2016_TrancheIV_v6", + "InputFromOutputModule": "RAWSIMoutput", + "InputStep": "GENSIM", + "SplittingAlgo": "EventAwareLumiBased", + "ScramArch": [ + "slc6_amd64_gcc530" + ], + "ConfigCacheID": "f7e311f2c6b5a0884faea133990edcbf", + "StepName": "DIGI", + "CMSSWVersion": "CMSSW_8_0_21" + }, + "Step1": { + "MCPileup": "/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIIWinter15GS-MCRUN2_71_V1-v1/GEN-SIM", + "PrepID": "TEST-Step1-RunIISummer15wmLHEGS-00744", + "GlobalTag": "MCRUN2_71_V1::All", + "Seeding": "AutomaticSeeding", + "StepName": "GENSIM", + "SplittingAlgo": "EventBased", + "ScramArch": [ + "slc6_amd64_gcc481" + ], + "EventsPerLumi": 100, + "ConfigCacheID": "526ca0745cd309b7cfef9f23b3d43acb", + "EventsPerJob": 200, + "RequestNumEvents": 4000, + "CMSSWVersion": "CMSSW_7_1_25_patch2" + }, + "TotalInputFiles": 0, + "ChainParentageMap": { + "Step2": { + "ParentDset": "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM", + "ChildDsets": [ + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM-RAW" + ] + }, + "Step1": { + "ParentDset": null, + "ChildDsets": [ + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM", + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/LHE" + ] + } + }, + "CustodialGroup": "DataOps", + "MaxMergeSize": 4294967296, + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "SubRequestType": "", + "TrustPUSitelists": true, + "HardTimeout": 129900, + "OpenRunningTimeout": 0, + "LumiList": {}, + "DashboardHost": "cms-jobmon.cern.ch", + "EnableHarvesting": false, + "BlockCloseMaxWaitTime": 66400, + "SiteWhitelist": [ + "T1_US_FNAL", + "T2_CH_CERN" + ], + "GlobalTagConnect": null, + "DQMConfigCacheID": null, + "DeterministicPileup": false, + "ValidStatus": "PRODUCTION", + "AcquisitionEra": "DMWM_Test", + "GlobalTag": "80X_mcRun2_asymptotic_2016_TrancheIV_v6", + "Campaign": "HG2002_Val", + "Multicore": 1, + "CouchDBName": "reqmgr_config_cache", + "NonCustodialGroup": "DataOps", + "DeleteFromSource": false, + "TotalInputEvents": 4000, + "SizePerEvent": 250, + "ConfigCacheID": null, + "StepChain": 2, + "CustodialSubType": "Replica", + "GracePeriod": 300, + "MergedLFNBase": "/store/backfill/1", + "ParentageResolved": false, + "EnableNewStageout": false, + "SubscriptionPriority": "Low", + "RobustMerge": true, + "Team": "testbed-vocms0193", + "AllowOpportunistic": false, + "DQMHarvestUnit": "byRun", + "AutoApproveSubscriptionSites": [], + "RequestWorkflow": "https://cmsweb-testbed.cern.ch/couchdb/reqmgr_workload_cache/tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925/spec", + "BlockCloseMaxFiles": 500, + "VoGroup": "unknown", + "RequestType": "StepChain", + "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/", + "SiteBlacklist": [], + "RequestStatus": "completed", + "SoftTimeout": 129600, + "Group": "DATAOPS", + "BlockCloseMaxEvents": 25000000, + "CustodialSites": [], + "ProcessingString": "SC_MultiPU_HG2002_Val_Todor_v13", + "ScramArch": [ + "slc6_amd64_gcc530" + ], + "TrustSitelists": false, + "UnmergedLFNBase": "/store/unmerged", + "DashboardPort": 8884, + "TimePerEvent": 140, + "DQMSequences": [], + "OutputModulesLFNBases": [ + "/store/unmerged/DMWM_Test/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/GEN-SIM/SC_MultiPU_HG2002_Val_Todor_v13-v20", + "/store/unmerged/DMWM_Test/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/GEN-SIM-RAW/SC_MultiPU_HG2002_Val_Todor_v13-v20", + "/store/unmerged/DMWM_Test/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/LHE/SC_MultiPU_HG2002_Val_Todor_v13-v20" + ], + "OutputDatasets": [ + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM", + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/LHE", + "/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM-RAW" + ], + "PrimaryDataset": "DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8", + "NonCustodialSubType": "Replica", + "RunNumber": 0, + "OverrideCatalog": null, + "PeriodicHarvestInterval": 0, + "TotalEstimatedJobs": 20, + "MaxWaitTime": 86400, + "RequestString": "SC_MultiPU_HG2002_Val", + "RequestorDN": "", + "RequestDate": [ + 2020, + 1, + 21, + 3, + 36, + 59 + ], + "InitialPriority": 316000, + "EventStreams": null, + "Memory": 2300, + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, + "IncludeParents": false, + "VoRole": "unknown", + "PriorityTransition": [ + { + "Priority": 316000, + "DN": "", + "UpdateTime": 1579577819 + } + ], + "CMSSWVersion": "CMSSW_8_0_21", + "NonCustodialSites": [], + "BlockCloseMaxSize": 5000000000000, + "MinMergeSize": 2147483648, + "RequestPriority": 316000, + "ProcessingVersion": 20, + "TotalInputLumis": 40, + "FirstEvent": 1, + "RequestName": "tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925", + "CouchURL": "https://cmsweb-testbed.cern.ch/couchdb", + "CouchWorkloadDBName": "reqmgr_workload_cache", + "RequestTransition": [ + { + "Status": "new", + "DN": "", + "UpdateTime": 1579577819 + }, + { + "DN": "", + "Status": "assignment-approved", + "UpdateTime": 1579577822 + }, + { + "DN": "", + "Status": "assigned", + "UpdateTime": 1579577822 + }, + { + "DN": "", + "Status": "staging", + "UpdateTime": 1579577895 + }, + { + "DN": "", + "Status": "staged", + "UpdateTime": 1579578050 + }, + { + "DN": "", + "Status": "acquired", + "UpdateTime": 1579578770 + }, + { + "DN": "", + "Status": "running-open", + "UpdateTime": 1579578770 + }, + { + "DN": "", + "Status": "running-closed", + "UpdateTime": 1579579378 + }, + { + "DN": "", + "Status": "completed", + "UpdateTime": 1579587203 + } + ], + "FirstLumi": 1, + "DQMUploadProxy": null, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "_id": "tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925", + "Dashboard": "integration" +} diff --git a/test/data/ReqMgr/requests/Static/ReRecoRequestDump.json b/test/data/ReqMgr/requests/Static/ReRecoRequestDump.json new file mode 100644 index 0000000000..8aba557cc1 --- /dev/null +++ b/test/data/ReqMgr/requests/Static/ReRecoRequestDump.json @@ -0,0 +1,234 @@ +{ + "PrepID": "ReReco-Run2017F-SingleElectron-09Aug2019_UL2017_EcalRecovery-00001", + "MaxMergeEvents": 20000000, + "Comments": "", + "Requestor": "pdmvserv", + "RunBlacklist": [], + "TotalInputFiles": 2729, + "CustodialGroup": "DataOps", + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "SubRequestType": "", + "TrustPUSitelists": false, + "HardTimeout": 159900, + "OpenRunningTimeout": 0, + "LumiList": {}, + "DashboardHost": "cms-jobmon.cern.ch", + "EnableHarvesting": false, + "Scenario": "pp", + "BlockCloseMaxWaitTime": 172800, + "SiteWhitelist": [ + "T1_IT_CNAF", + "T1_ES_PIC", + "T2_CH_CSCS", + "T2_UA_KIPT", + "T2_US_Purdue", + "T2_TW_NCHC", + "T2_UK_SGrid_RALPP", + "T2_FR_GRIF_LLR", + "T2_BE_UCL", + "T2_ES_IFCA", + "T2_BE_IIHE", + "T2_FR_IPHC", + "T2_DE_DESY", + "T1_UK_RAL", + "T1_US_FNAL", + "T2_US_Caltech", + "T2_UK_London_Brunel", + "T2_RU_JINR", + "T2_IT_Pisa", + "T2_US_Vanderbilt", + "T2_IN_TIFR", + "T2_FR_CCIN2P3", + "T1_FR_CCIN2P3", + "T2_US_Florida", + "T2_FR_GRIF_IRFU", + "T2_UK_London_IC", + "T2_IT_Bari", + "T2_US_Nebraska", + "T1_DE_KIT", + "T2_US_UCSD", + "T2_ES_CIEMAT", + "T1_RU_JINR", + "T1_RU_JINR", + "T2_PL_Warsaw", + "T2_US_Wisconsin", + "T2_US_MIT", + "T2_DE_RWTH", + "T2_CH_CERN", + "T2_PT_NCG_Lisbon", + "T2_BR_SPRACE" + ], + "VoRole": "unknown", + "DQMConfigCacheID": "335e35b42a8778432d1e8783300459e1", + "ValidStatus": "PRODUCTION", + "PriorityTransition": [ + { + "Priority": 500000, + "DN": "", + "UpdateTime": 1588759495 + } + ], + "GlobalTag": "106X_dataRun2_v20", + "Campaign": "UltraLegacy2017", + "ScramArch": [ + "slc7_amd64_gcc700" + ], + "CouchDBName": "reqmgr_config_cache", + "NonCustodialGroup": "DataOps", + "EventsPerJob": 9600, + "InputDataset": "/SingleElectron/Run2017F-v1/RAW", + "DeleteFromSource": false, + "TotalInputEvents": 15066632, + "SizePerEvent": 300, + "ConfigCacheID": "335e35b42a8778432d1e878330036b02", + "CustodialSubType": "Replica", + "GracePeriod": 300, + "MergedLFNBase": "/store/data", + "EnableNewStageout": false, + "TransientOutputModules": [], + "SubscriptionPriority": "Low", + "RobustMerge": true, + "SplittingAlgo": "EventAwareLumiBased", + "Team": "production", + "AllowOpportunistic": false, + "DQMHarvestUnit": "byRun", + "AutoApproveSubscriptionSites": [ + "T1_IT_CNAF_Disk" + ], + "RequestWorkflow": "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146/spec", + "BlockBlacklist": [], + "BlockCloseMaxFiles": 500, + "VoGroup": "unknown", + "RequestType": "ReReco", + "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "SiteBlacklist": [], + "RequestStatus": "completed", + "SoftTimeout": 159600, + "Group": "PPD", + "BlockCloseMaxEvents": 200000000, + "CustodialSites": [], + "ProcessingString": "09Aug2019_UL2017_EcalRecovery", + "Multicore": 8, + "TrustSitelists": false, + "UnmergedLFNBase": "/store/unmerged", + "DashboardPort": 8884, + "TimePerEvent": 3, + "DQMSequences": [], + "OutputModulesLFNBases": [ + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/EcalESAlign-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/AOD/09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/HcalCalIterativePhiSym-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/MINIAOD/09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/EcalUncalZElectron-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/ALCARECO/EcalUncalWElectron-09Aug2019_UL2017_EcalRecovery-v1", + "/store/unmerged/Run2017F/SingleElectron/DQMIO/09Aug2019_UL2017_EcalRecovery-v1" + ], + "OutputDatasets": [ + "/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/MINIAOD", + "/SingleElectron/Run2017F-EcalUncalWElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO", + "/SingleElectron/Run2017F-EcalUncalZElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO", + "/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/AOD", + "/SingleElectron/Run2017F-EcalESAlign-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO", + "/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/DQMIO", + "/SingleElectron/Run2017F-HcalCalIterativePhiSym-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO" + ], + "PrimaryDataset": null, + "NonCustodialSubType": "Replica", + "BlockCloseMaxSize": 5000000000000, + "MaxMergeSize": 4294967296, + "OverrideCatalog": null, + "PeriodicHarvestInterval": 0, + "TotalEstimatedJobs": 1575, + "MaxWaitTime": 86400, + "RequestString": "Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery", + "RequestorDN": "", + "RunWhitelist": [ + 306459, + 306456, + 305064, + 306125 + ], + "RequestDate": [ + 2020, + 5, + 6, + 10, + 4, + 55 + ], + "InitialPriority": 500000, + "RunNumber": 0, + "EventStreams": null, + "Memory": 15900, + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/PRODUCTION" + }, + "FilesPerJob": 1, + "IncludeParents": false, + "BlockWhitelist": [], + "LumisPerJob": 8, + "AcquisitionEra": "Run2017F", + "CMSSWVersion": "CMSSW_10_6_11", + "NonCustodialSites": [ + "T1_IT_CNAF_Disk" + ], + "GlobalTagConnect": null, + "MinMergeSize": 2147483648, + "RequestPriority": 500000, + "ProcessingVersion": 1, + "TotalInputLumis": 8110, + "RequestName": "pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146", + "CouchURL": "https://cmsweb.cern.ch/couchdb", + "CouchWorkloadDBName": "reqmgr_workload_cache", + "RequestTransition": [ + { + "Status": "new", + "DN": "", + "UpdateTime": 1588759495 + }, + { + "DN": "", + "Status": "assignment-approved", + "UpdateTime": 1588759500 + }, + { + "DN": "", + "Status": "assigned", + "UpdateTime": 1588760963 + }, + { + "DN": "", + "Status": "staging", + "UpdateTime": 1588761258 + }, + { + "DN": "", + "Status": "staged", + "UpdateTime": 1588761688 + }, + { + "DN": "", + "Status": "acquired", + "UpdateTime": 1588762166 + }, + { + "DN": "", + "Status": "running-open", + "UpdateTime": 1588763098 + }, + { + "DN": "", + "Status": "running-closed", + "UpdateTime": 1588775354 + }, + { + "DN": "", + "Status": "completed", + "UpdateTime": 1589041121 + } + ], + "DQMUploadProxy": null, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "_id": "pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146", + "Dashboard": "reprocessing" + } diff --git a/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py b/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py index c1b3dc58f7..dc0c32a151 100644 --- a/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py +++ b/test/python/WMCore_t/MicroService_t/DataStructs_t/MSRuleCleanerWflow_t.py @@ -26,6 +26,9 @@ def setUp(self): self.maxDiff = None self.taskChainFile = getTestFile('data/ReqMgr/requests/Static/TaskChainRequestDump.json') self.stepChainFile = getTestFile('data/ReqMgr/requests/Static/StepChainRequestDump.json') + self.reRecoFile = getTestFile('data/ReqMgr/requests/Static/ReRecoRequestDump.json') + self.includeParentsFile = getTestFile('data/ReqMgr/requests/Static/IncludeParentsRequestDump.json') + self.multiPUFile = getTestFile('data/ReqMgr/requests/Static/MultiPURequestDump.json') self.reqRecordsFile = getTestFile('data/ReqMgr/requests/Static/BatchRequestsDump.json') with open(self.reqRecordsFile) as fd: self.reqRecords = json.load(fd) @@ -33,7 +36,12 @@ def setUp(self): self.taskChainReq = json.load(fd) with open(self.stepChainFile) as fd: self.stepChainReq = json.load(fd) - + with open(self.reRecoFile) as fd: + self.reRecoReq = json.load(fd) + with open(self.includeParentsFile) as fd: + self.includeParentsReq = json.load(fd) + with open(self.multiPUFile) as fd: + self.multiPUReq = json.load(fd) super(MSRuleCleanerWflowTest, self).setUp() def testTaskChainDefaults(self): @@ -56,63 +64,167 @@ def testTaskChainDefaults(self): self.assertEqual(wflow["IsArchivalDelayExpired"], False) self.assertEqual(wflow["ForceArchive"], False) self.assertEqual(wflow["RequestTransition"], []) + self.assertEqual(wflow['IncludeParents'], False) + self.assertEqual(wflow['DataPileup'], []) + self.assertEqual(wflow['MCPileup'], []) + self.assertEqual(wflow['InputDataset'], None) + self.assertEqual(wflow['ParentDataset'], []) def testTaskChain(self): # Test Taskchain: wflow = MSRuleCleanerWflow(self.taskChainReq) expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': False, 'IsClean': False, 'IsLogDBClean': False, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': None, 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - 'RequestTransition': [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {}, 'TargetStatus': None, 'TransferDone': False} self.assertDictEqual(wflow, expectedWflow) + def testReReco(self): + # Test ReReco workflow: + wflow = MSRuleCleanerWflow(self.reRecoReq) + expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/SingleElectron/Run2017F-v1/RAW', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/MINIAOD', + u'/SingleElectron/Run2017F-EcalUncalWElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO', + u'/SingleElectron/Run2017F-EcalUncalZElectron-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO', + u'/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/AOD', + u'/SingleElectron/Run2017F-EcalESAlign-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO', + u'/SingleElectron/Run2017F-09Aug2019_UL2017_EcalRecovery-v1/DQMIO', + u'/SingleElectron/Run2017F-HcalCalIterativePhiSym-09Aug2019_UL2017_EcalRecovery-v1/ALCARECO'], + 'ParentDataset': [], + 'ParentageResolved': True, + 'PlineMarkers': None, + 'RequestName': u'pdmvserv_Run2017F-v1_SingleElectron_09Aug2019_UL2017_EcalRecovery_200506_120455_3146', + 'RequestStatus': u'completed', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1588759495}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1588759500}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1588760963}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1588761258}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1588761688}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1588762166}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1588763098}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1588775354}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1589041121}], + 'RequestType': u'ReReco', + 'RulesToClean': {}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + + def testIncludeParents(self): + # Test include parents:: + wflow = MSRuleCleanerWflow(self.includeParentsReq) + expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': True, + 'InputDataset': u'/Cosmics/Commissioning2015-PromptReco-v1/RECO', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/Cosmics/Integ_Test-CosmicSP-StepChain_InclParents_HG2004_Val_Privv12-v11/RAW-RECO'], + 'ParentDataset': [], + 'ParentageResolved': False, + 'PlineMarkers': None, + 'RequestName': u'amaltaro_StepChain_InclParents_April2020_Val_200414_120713_81', + 'RequestStatus': u'completed', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1586858833}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1586858834}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1586858835}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1586859358}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1586859733}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1586860322}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1586860927}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1586861535}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1586863942}], + 'RequestType': u'StepChain', + 'RulesToClean': {}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + + def testMultiPU(self): + # Test workflow with multiple pileups:: + wflow = MSRuleCleanerWflow(self.multiPUReq) + expectedWflow = {'CleanupStatus': {}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': None, + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [u'/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIIWinter15GS-MCRUN2_71_V1-v1/GEN-SIM', + u'/Neutrino_E-10_gun/RunIISpring15PrePremix-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v2-v2/GEN-SIM-DIGI-RAW'], + 'OutputDatasets': [u'/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM', + u'/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/LHE', + u'/DYJetsToLL_Pt-50To100_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/DMWM_Test-SC_MultiPU_HG2002_Val_Todor_v13-v20/GEN-SIM-RAW'], + 'ParentDataset': [], + 'ParentageResolved': False, + 'PlineMarkers': None, + 'RequestName': u'tivanov_SC_MultiPU_HG2002_Val_200121_043659_4925', + 'RequestStatus': u'completed', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1579577819}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1579577822}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1579577822}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1579577895}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1579578050}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1579578770}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1579578770}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1579579378}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1579587203}], + 'RequestType': u'StepChain', + 'RulesToClean': {}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + if __name__ == '__main__': unittest.main() diff --git a/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py b/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py index 56704539d4..5f4462e035 100644 --- a/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py +++ b/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py @@ -85,52 +85,35 @@ def testPipelineAgentBlock(self): wflow = MSRuleCleanerWflow(self.taskChainReq) self.msRuleCleaner.plineAgentBlock.run(wflow) expectedWflow = {'CleanupStatus': {'plineAgentBlock': True}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': False, 'IsClean': False, 'IsLogDBClean': False, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': ['plineAgentBlock'], 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - "RequestTransition": [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', u'Status': u'new', u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {'plineAgentBlock': []}, 'TargetStatus': None, @@ -142,58 +125,125 @@ def testPipelineAgentCont(self): wflow = MSRuleCleanerWflow(self.taskChainReq) self.msRuleCleaner.plineAgentCont.run(wflow) expectedWflow = {'CleanupStatus': {'plineAgentCont': True}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': False, 'IsClean': False, 'IsLogDBClean': False, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': ['plineAgentCont'], 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - "RequestTransition": [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', u'Status': u'new', u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {'plineAgentCont': []}, 'TargetStatus': None, 'TransferDone': False} self.assertDictEqual(wflow, expectedWflow) + def testPipelineMSTrBlock(self): + # Test plineAgentCont + wflow = MSRuleCleanerWflow(self.taskChainReq) + self.msRuleCleaner.plineMSTrBlock.run(wflow) + expectedWflow = {'CleanupStatus': {'plineMSTrBlock': True}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', + u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', + u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], + 'ParentageResolved': True, + 'PlineMarkers': ['plineMSTrBlock'], + 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', + 'RequestStatus': u'announced', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], + 'RequestType': u'TaskChain', + 'RulesToClean': {'plineMSTrBlock': []}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + + def testPipelineMSTrCont(self): + # Test plineAgentCont + wflow = MSRuleCleanerWflow(self.taskChainReq) + self.msRuleCleaner.plineMSTrCont.run(wflow) + expectedWflow = {'CleanupStatus': {'plineMSTrCont': True}, + 'DataPileup': [], + 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', + 'IsArchivalDelayExpired': False, + 'IsClean': False, + 'IsLogDBClean': False, + 'MCPileup': [], + 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', + u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', + u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', + u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], + 'ParentageResolved': True, + 'PlineMarkers': ['plineMSTrCont'], + 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', + 'RequestStatus': u'announced', + 'RequestTransition': [{u'DN': u'', + u'Status': u'new', + u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], + 'RequestType': u'TaskChain', + 'RulesToClean': {'plineMSTrCont': []}, + 'TargetStatus': None, + 'TransferDone': False} + self.assertDictEqual(wflow, expectedWflow) + def testPipelineArchive(self): # Test plineAgentCont wflow = MSRuleCleanerWflow(self.taskChainReq) @@ -210,15 +260,20 @@ def testPipelineArchive(self): with self.assertRaises(MSRuleCleanerArchivalSkip): self.msRuleCleaner.plineArchive.run(wflow) expectedWflow = {'CleanupStatus': {'plineAgentBlock': True, 'plineAgentCont': True}, + 'DataPileup': [], 'ForceArchive': False, + 'IncludeParents': False, + 'InputDataset': u'/JetHT/Run2012C-v1/RAW', 'IsArchivalDelayExpired': True, 'IsClean': True, 'IsLogDBClean': True, + 'MCPileup': [], 'OutputDatasets': [u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO', u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO', u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO', u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'], + 'ParentDataset': [], 'ParentageResolved': True, 'PlineMarkers': ['plineArchive', 'plineAgentBlock', @@ -226,39 +281,17 @@ def testPipelineArchive(self): 'plineArchive'], 'RequestName': u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891', 'RequestStatus': u'announced', - "RequestTransition": [{"Status": "new", - "DN": "", - "UpdateTime": 1606723304}, - {"DN": "", - "Status": "assignment-approved", - "UpdateTime": 1606723305}, - {"DN": "", - "Status": "assigned", - "UpdateTime": 1606723306}, - {"DN": "", - "Status": "staging", - "UpdateTime": 1606723461}, - {"DN": "", - "Status": "staged", - "UpdateTime": 1606723590}, - {"DN": "", - "Status": "acquired", - "UpdateTime": 1606723968}, - {"DN": "", - "Status": "running-open", - "UpdateTime": 1606724572}, - {"DN": "", - "Status": "running-closed", - "UpdateTime": 1606724573}, - {"DN": "", - "Status": "completed", - "UpdateTime": 1607018413}, - {"DN": "", - "Status": "closed-out", - "UpdateTime": 1607347706}, - {"DN": "", - "Status": "announced", - "UpdateTime": 1607359514}], + 'RequestTransition': [{u'DN': u'', u'Status': u'new', u'UpdateTime': 1606723304}, + {u'DN': u'', u'Status': u'assignment-approved', u'UpdateTime': 1606723305}, + {u'DN': u'', u'Status': u'assigned', u'UpdateTime': 1606723306}, + {u'DN': u'', u'Status': u'staging', u'UpdateTime': 1606723461}, + {u'DN': u'', u'Status': u'staged', u'UpdateTime': 1606723590}, + {u'DN': u'', u'Status': u'acquired', u'UpdateTime': 1606723968}, + {u'DN': u'', u'Status': u'running-open', u'UpdateTime': 1606724572}, + {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1606724573}, + {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1607018413}, + {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706}, + {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}], 'RequestType': u'TaskChain', 'RulesToClean': {'plineAgentBlock': [], 'plineAgentCont': []}, 'TargetStatus': 'normal-archived',