update WMBSHelper logic; add check for store/unmerged in addition to …

…merged

src/python/WMComponent/JobAccountant/AccountantWorker.py

@@ -380,12 +380,14 @@ def findDBSParents(self, lfn):
                                                        conn=self.getDBConn(),
                                                        transaction=self.existingTransaction())
         newParents = set()
+        logging.info("AMR found %d parents for lfn: %s", len(parentsInfo), lfn)
         for parentInfo in parentsInfo:
-            logging.info("AMR potential parent is: %s", parentInfo)
+            logging.info("AMR lfn %s potential parent is: %s", lfn, parentInfo)
             # This will catch straight to merge files that do not have redneck
             # parents.  We will mark the straight to merge file from the job
             # as a child of the merged parent.
-            if int(parentInfo["merged"]) == 1:
+            if int(parentInfo["merged"]) == 1 and not parentInfo["lfn"].startswith("/store/unmerged/"):
+                logging.info("AMR parent merged file added: %s", parentInfo["lfn"])
                 newParents.add(parentInfo["lfn"])
 
             elif parentInfo['gpmerged'] is None:
@@ -395,7 +397,8 @@ def findDBSParents(self, lfn):
             # children.  We have to setup parentage and then check on whether or
             # not this file has any redneck children and update their parentage
             # information.
-            elif int(parentInfo["gpmerged"]) == 1:
+            elif int(parentInfo["gpmerged"]) == 1 and not parentInfo["gplfn"].startswith("/store/unmerged/"):
+                logging.info("AMR great parent merged file added: %s", parentInfo["gplfn"])
                 newParents.add(parentInfo["gplfn"])
 
             # If that didn't work, we've reached the great-grandparents

src/python/WMCore/WorkQueue/WMBSHelper.py

@@ -669,22 +669,20 @@ def _addACDCFileToWMBSFile(self, acdcFile, inFileset=True):
         adds the ACDC files into WMBS database
         """
         wmbsParents = []
-        # TODO:  this check can be removed when ErrorHandler filters parents file for unmerged data
-        if acdcFile["parents"]:
-            firstParent = next(iter(acdcFile["parents"]))
-            # If files is merged and has unmerged parents skip the wmbs population
-            if acdcFile.get("merged", 0) and ("/store/unmerged/" in firstParent or "MCFakeFile" in firstParent):
-                # don't set the parents
-                pass
-            else:
-                # set the parentage for all the unmerged parents
-                for parent in acdcFile["parents"]:
-                    logging.debug("WMBS ACDC Parent File: %s", parent)
-                    parent = self._addACDCFileToWMBSFile(DatastructFile(lfn=parent,
-                                                                        locations=acdcFile["locations"],
-                                                                        merged=True),
-                                                         inFileset=False)
-                    wmbsParents.append(parent)
+        # If file is merged, then it will be the parent of whatever output that
+        # process this job (it twists my mind!). Meaning, block below can be skipped
+        if int(acdcFile.get("merged", 0)) == 0 and acdcFile["parents"]:
+            # set the parentage for all the unmerged parents
+            for parent in acdcFile["parents"]:
+                if parent.startswith("/store/unmerged/") or parent.startswith("MCFakeFile"):
+                    logging.warning("WMBS ACDC skipped parent invalid file: %s", parent)
+                    continue
+                logging.debug("WMBS ACDC Parent File: %s", parent)
+                parent = self._addACDCFileToWMBSFile(DatastructFile(lfn=parent,
+                                                                    locations=acdcFile["locations"],
+                                                                    merged=True),
+                                                     inFileset=False)
+                wmbsParents.append(parent)
 
         # pass empty check sum since it won't be updated to dbs anyway
         checksums = {}