Merge pull request #4728 from dballesteros7/fixEventBasedACDC

Fix corner case in EventBased ACDC

src/python/WMCore/JobSplitting/EventBased.py

@@ -122,9 +122,9 @@ def algorithm(self, *args, **kwargs):
                 else:
                     if acdcFileList:
                         if f['lfn'] in [x['lfn'] for x in acdcFileList]:
-                            self.createACDCJobs(f, acdcFileList,
-                                                timePerEvent, sizePerEvent, memoryRequirement,
-                                                lheInput)
+                            totalJobs = self.createACDCJobs(f, acdcFileList,
+                                                            timePerEvent, sizePerEvent, memoryRequirement,
+                                                            lheInput, totalJobs)
                         continue
                     #This assumes there's only one run which is the case for MC
                     lumis = runs[0].lumis
@@ -184,14 +184,13 @@ def algorithm(self, *args, **kwargs):
 
     def createACDCJobs(self, fakeFile, acdcFileInfo,
                        timePerEvent, sizePerEvent, memoryRequirement,
-                       lheInputOption):
+                       lheInputOption, totalJobs = 0):
         """
         _createACDCJobs_
 
         Create ACDC production jobs, this are treated differentely
         since it is an exact copy of the failed jobs.
         """
-        totalJobs = 0
         for acdcFile in acdcFileInfo:
             if fakeFile['lfn'] == acdcFile['lfn']:
                 self.newJob(name = self.getJobName(length = totalJobs))
@@ -207,4 +206,4 @@ def createACDCJobs(self, fakeFile, acdcFileInfo,
                                                      memory = memoryRequirement,
                                                      disk = diskRequired)
                 totalJobs += 1
-        return
+        return totalJobs

test/python/WMCore_t/WMBS_t/JobSplitting_t/EventBased_t.py

@@ -169,35 +169,40 @@ def populateACDCCouch(self, numFiles = 2, lumisPerJob = 35,
 
     def generateFakeMCFile(self, numEvents = 100, firstEvent = 1,
                            lastEvent = 100, firstLumi = 1, lastLumi = 10,
-                           index = 1):
+                           index = 1, existingSub = None):
         """
         _generateFakeMCFile_
 
         Generates a fake MC file for testing production EventBased
-        creation of jobs
+        creation of jobs, it creates a single file subscription if no
+        existing subscription is provided.
         """
-        # MC comes with only one MCFakeFile
-        singleMCFileset = Fileset(name = "MCTestFileset-%i" % index)
-        singleMCFileset.create()
+        # MC comes with MCFakeFile(s)
         newFile = File("MCFakeFile-some-hash-%s" % str(index).zfill(5), size = 1000,
                        events = numEvents,
                        locations = set(["somese.cern.ch"]))
         newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
         newFile["first_event"] = firstEvent
         newFile["last_event"] = lastEvent
         newFile.create()
-        singleMCFileset.addFile(newFile)
-        singleMCFileset.commit()
-        testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
+        if existingSub is None:
+            singleMCFileset = Fileset(name = "MCTestFileset-%i" % index)
+            singleMCFileset.create()
+            singleMCFileset.addFile(newFile)
+            singleMCFileset.commit()
+            testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
                                 name = "wf001", task = "Test")
-        testWorkflow.create()
-
-        singleMCFileSubscription = Subscription(fileset = singleMCFileset,
-                                                workflow = testWorkflow,
-                                                split_algo = "EventBased",
-                                                type = "Production")
-        singleMCFileSubscription.create()
-        return singleMCFileSubscription
+            testWorkflow.create()
+            singleMCFileSubscription = Subscription(fileset = singleMCFileset,
+                                                    workflow = testWorkflow,
+                                                    split_algo = "EventBased",
+                                                    type = "Production")
+            singleMCFileSubscription.create()
+            return singleMCFileSubscription
+        else:
+            existingSub['fileset'].addFile(newFile)
+            existingSub['fileset'].commit()
+            return existingSub
 
     def testExactEvents(self):
         """
@@ -523,8 +528,10 @@ def testACDCProduction(self):
         Test the ability of the EventBased algorithm of creating
         jobs from ACDC correctly
         """
-        self.populateACDCCouch()
+        self.populateACDCCouch(numFiles = 4)
         mcSubscription = self.generateFakeMCFile(20000, 1, 20001, 1, 8750, 0)
+        mcSubscription = self.generateFakeMCFile(20000, 1, 20001, 8751, 17500, 1, mcSubscription)
+        mcSubscription = self.generateFakeMCFile(20000, 1, 20001, 17501, 26250, 2, mcSubscription)
         splitter = SplitterFactory()
         jobFactory = splitter(package = "WMCore.WMBS",
                               subscription = mcSubscription)
@@ -537,11 +544,10 @@ def testACDCProduction(self):
 
         self.assertEqual(1, len(jobGroups))
         jobGroup = jobGroups[0]
-        self.assertEqual(250, len(jobGroup.jobs))
+        self.assertEqual(750, len(jobGroup.jobs))
 
         for job in jobGroup.jobs:
             self.assertEqual(1, len(job["input_files"]))
-            self.assertEqual("MCFakeFile-some-hash-00000", job["input_files"][0]["lfn"])
             mask = job["mask"]
             self.assertEqual(35, mask["LastLumi"] - mask["FirstLumi"])
             self.assertEqual(20000, mask["LastEvent"] - mask["FirstEvent"])