Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WorkQueue: convert PNN to PSN for pileup location #12066

Merged
merged 1 commit into from
Aug 6, 2024
Merged

WorkQueue: convert PNN to PSN for pileup location #12066

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/python/WMCore/WorkQueue/DataLocationMapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def locationsFromMSPileup(self, dataItems, dbsUrl):
:param dbsUrl: str, dbs url to check which dbs server
:return: dict, dict of pileup name keys with location set values
"""
self.logger.info(f'Fetching locations from MSPileup for {len(dataItems)}')
self.logger.info(f'Fetching locations from MSPileup for {len(dataItems)} containers')

result = defaultdict(set)
# TODO: Fetch multiple pileups in single request
Expand All @@ -148,7 +148,8 @@ def locationsFromMSPileup(self, dataItems, dbsUrl):
msPileupUrl = f"https://cmsweb{pileupInstance}.cern.ch/ms-pileup/data/pileup"
doc = getPileupDocs(msPileupUrl, queryDict, method='POST')[0]
self.logger.info(f'locationsFromPileup - name: {dataItem}, currentRSEs: {doc["currentRSEs"]}, containerFraction: {doc["containerFraction"]}')
result[dataItem] = doc['currentRSEs']
# resolve PNNs into PSNs
result[dataItem] = self.cric.PNNstoPSNs(doc['currentRSEs'])
except IndexError:
self.logger.error('Did not find any pileup document for query: %s', queryDict['query'])
except Exception as ex:
Expand Down Expand Up @@ -249,6 +250,7 @@ def updatePileupLocation(self):
for dataMapping in listvalues(dataLocations):
for data, locations in viewitems(dataMapping):
elements = self.backend.getElementsForPileupData(data)
self.logger.info("Found %d elements using pileup: %s", len(elements), data)
for element in elements:
if element.get('NoPileupUpdate', False):
continue
Expand Down
2 changes: 1 addition & 1 deletion src/python/WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ def getDatasetLocationsFromMSPileup(self, datasetsWithDbsURL):
'filters': ['expectedRSEs', 'currentRSEs', 'pileupName', 'containerFraction', 'ruleIds']}
try:
doc = getPileupDocs(msPileupUrl, queryDict, method='POST')[0]
currentRSEs = doc['currentRSEs']
currentRSEs = self.cric.PNNstoPSNs(doc['currentRSEs'])
self.logger.debug(f'Retrieved MSPileup document: {doc}')
if len(currentRSEs) == 0:
self.logger.warning(f'No RSE has a copy of the desired pileup dataset. Expected RSEs: {doc["expectedRSEs"]}')
Expand Down