From 9b52abfd13644ad1271712d79ec66d400c6de1a8 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Tue, 16 Feb 2021 11:47:42 +0100 Subject: [PATCH 1/2] [py2py3] src/python/A/B/C slice 8 - issue #10132 - src --- .../WMCore/MicroService/Service/Data.py | 4 ++-- src/python/WMCore/ProcessPool/ProcessPool.py | 6 +++-- src/python/WMCore/Services/DBS/DBS3Reader.py | 22 ++++++++++--------- .../WMCore/Services/DBS/DBSWriterObjects.py | 8 ++++--- .../WMCore/Services/DBS/ProdException.py | 9 ++++---- src/python/WMQuality/TestInitCouchApp.py | 3 ++- 6 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/python/WMCore/MicroService/Service/Data.py b/src/python/WMCore/MicroService/Service/Data.py index 61e9b57dc2..83b94d0671 100644 --- a/src/python/WMCore/MicroService/Service/Data.py +++ b/src/python/WMCore/MicroService/Service/Data.py @@ -69,7 +69,7 @@ def validate(self, apiobj, method, api, param, safe): """ if method == 'GET': - for prop in param.kwargs.keys(): + for prop in list(param.kwargs): safe.kwargs[prop] = param.kwargs.pop(prop) safe.kwargs['API'] = api if param.args: @@ -112,7 +112,7 @@ def post(self): result = {'status': 'Not supported, %s' % msg, 'request': None} try: data = json.load(cherrypy.request.body) - if 'request' in data.keys(): + if 'request' in data: reqName = data['request'] result = self.mgr.info(reqName) return results(result) diff --git a/src/python/WMCore/ProcessPool/ProcessPool.py b/src/python/WMCore/ProcessPool/ProcessPool.py index a046b41a52..0e66823a0e 100644 --- a/src/python/WMCore/ProcessPool/ProcessPool.py +++ b/src/python/WMCore/ProcessPool/ProcessPool.py @@ -6,6 +6,8 @@ """ from __future__ import print_function +from builtins import range, object + import zmq import subprocess import sys @@ -38,7 +40,7 @@ class ProcessPoolException(WMException): """ -class ProcessPoolWorker: +class ProcessPoolWorker(object): """ _ProcessPoolWorker_ @@ -61,7 +63,7 @@ def __call__(self, input): return -class ProcessPool: +class ProcessPool(object): def __init__(self, slaveClassName, totalSlaves, componentDir, config, namespace='WMComponent', inPort='5555', outPort='5558'): diff --git a/src/python/WMCore/Services/DBS/DBS3Reader.py b/src/python/WMCore/Services/DBS/DBS3Reader.py index 1587837a0b..57a356f777 100644 --- a/src/python/WMCore/Services/DBS/DBS3Reader.py +++ b/src/python/WMCore/Services/DBS/DBS3Reader.py @@ -7,6 +7,11 @@ """ from __future__ import print_function, division +from builtins import object, str, bytes +from future.utils import viewitems + +from Utils.Utilities import encodeUnicodeToBytes, decodeBytesToUnicode + import logging from collections import defaultdict @@ -37,8 +42,8 @@ def remapDBS3Keys(data, stringify=False, **others): 'block_name': 'BlockName', 'lumi_section_num': 'LumiSectionNumber'} mapping.update(others) - formatFunc = lambda x: str(x) if stringify and isinstance(x, unicode) else x - for name, newname in mapping.iteritems(): + formatFunc = lambda x: encodeUnicodeToBytes(x) if stringify else x + for name, newname in viewitems(mapping): if name in data: data[newname] = formatFunc(data[name]) return data @@ -212,10 +217,8 @@ def listRunLumis(self, dataset=None, block=None): ) """ # Pointless code in python3 - if isinstance(block, str): - block = unicode(block) - if isinstance(dataset, str): - dataset = unicode(dataset) + block = decodeBytesToUnicode(block) + dataset = decodeBytesToUnicode(dataset) try: if block: @@ -516,7 +519,7 @@ def listFilesInBlockWithParents(self, fileBlockName, lumis=True, validFileOnly=1 for fp in f['parent_logical_file_name']: childByParents[fp].append(f['logical_file_name']) - parentsLFNs = childByParents.keys() + parentsLFNs = list(childByParents) if len(parentsLFNs) == 0: msg = "Error in " @@ -578,7 +581,7 @@ def listFileBlockLocation(self, fileBlockNames): """ singleBlockName = None - if isinstance(fileBlockNames, basestring): + if isinstance(fileBlockNames, (str, bytes)): singleBlockName = fileBlockNames fileBlockNames = [fileBlockNames] @@ -637,8 +640,7 @@ def getFileBlockWithParents(self, fileBlockName): "Files" : { LFN : Events }, "IsOpen" : True|False} """ - if isinstance(fileBlockName, str): - fileBlockName = unicode(fileBlockName) + fileBlockName = decodeBytesToUnicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlockWithParents(%s): No matching data" diff --git a/src/python/WMCore/Services/DBS/DBSWriterObjects.py b/src/python/WMCore/Services/DBS/DBSWriterObjects.py index bb22b152df..627d99de8e 100644 --- a/src/python/WMCore/Services/DBS/DBSWriterObjects.py +++ b/src/python/WMCore/Services/DBS/DBSWriterObjects.py @@ -8,6 +8,8 @@ """ from __future__ import print_function +from builtins import int + import logging from DBSAPI.dbsApi import DbsApi @@ -292,7 +294,7 @@ def createDBSFiles(fjrFileInfo, jobType = None, apiRef = None): for runinfo in fjrFileInfo.runs: run = DbsRun( - RunNumber = long(runinfo), + RunNumber = int(runinfo), NumberOfEvents = 0, NumberOfLumiSections = 0, TotalLuminosity = 0, @@ -310,12 +312,12 @@ def createDBSFiles(fjrFileInfo, jobType = None, apiRef = None): for lumiinfo in fjrFileInfo.getLumiSections(): lumi = DbsLumiSection( - LumiSectionNumber = long(lumiinfo['LumiSectionNumber']), + LumiSectionNumber = int(lumiinfo['LumiSectionNumber']), StartEventNumber = 0, EndEventNumber = 0, LumiStartTime = 0, LumiEndTime = 0, - RunNumber = long(lumiinfo['RunNumber']), + RunNumber = int(lumiinfo['RunNumber']), ) # Isnt needed, causes monster slowdown diff --git a/src/python/WMCore/Services/DBS/ProdException.py b/src/python/WMCore/Services/DBS/ProdException.py index 37d3778e7e..0c19a0f73d 100644 --- a/src/python/WMCore/Services/DBS/ProdException.py +++ b/src/python/WMCore/Services/DBS/ProdException.py @@ -6,8 +6,7 @@ """ - - +from future.utils import viewitems import exceptions import inspect @@ -92,7 +91,7 @@ def addInfo(self, **data): Add key=value information pairs to an exception instance """ - for key, value in data.items(): + for key, value in viewitems(data): self[key] = value return @@ -106,7 +105,7 @@ def xml(self): strg += self.message strg +="\n" strg +="\n" - for key, value in self.data.items(): + for key, value in viewitems(self.data): strg +="\n" strg += "\n" strg += str(key) @@ -124,6 +123,6 @@ def __str__(self): """create a string rep of this exception""" strg = "%s\n" % self.name strg += "Message: %s\n" % self.message - for key, value in self.data.items(): + for key, value in viewitems(self.data): strg += "\t%s : %s\n" % (key, value, ) return strg diff --git a/src/python/WMQuality/TestInitCouchApp.py b/src/python/WMQuality/TestInitCouchApp.py index 6003f82806..f04103d04a 100644 --- a/src/python/WMQuality/TestInitCouchApp.py +++ b/src/python/WMQuality/TestInitCouchApp.py @@ -11,6 +11,7 @@ Copyright (c) 2010 Fermilab. All rights reserved. """ +from builtins import object from future import standard_library standard_library.install_aliases() @@ -23,7 +24,7 @@ from WMQuality.TestInit import TestInit -class CouchAppTestHarness: +class CouchAppTestHarness(object): """ Test Harness for installing a couch database instance with several couchapps in a unittest.setUp and wiping it out in a unittest.tearDown From c026ff13fb55abe550c73c9ec741c3905d947038 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Tue, 16 Feb 2021 11:47:54 +0100 Subject: [PATCH 2/2] [py2py3] src/python/A/B/C slice 8 - issue #10132 - test and pylint --- .../WMCore/Services/DBS/DBSWriterObjects.py | 15 +++++++-------- .../WMCore/Services/DBS/ProdException.py | 18 +++++++++--------- .../WMCore_t/ProcessPool_t/ProcessPool_t.py | 1 + 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/python/WMCore/Services/DBS/DBSWriterObjects.py b/src/python/WMCore/Services/DBS/DBSWriterObjects.py index 627d99de8e..ff4cffcbea 100644 --- a/src/python/WMCore/Services/DBS/DBSWriterObjects.py +++ b/src/python/WMCore/Services/DBS/DBSWriterObjects.py @@ -12,7 +12,6 @@ import logging -from DBSAPI.dbsApi import DbsApi from DBSAPI.dbsException import * from DBSAPI.dbsApiException import * from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset @@ -50,7 +49,7 @@ def createPrimaryDataset(datasetInfo, apiRef = None): else: PrimaryDatasetType = 'mc' - logging.debug("Inserting PrimaryDataset %s with Type %s"%(datasetInfo["PrimaryDataset"],PrimaryDatasetType)) + logging.debug("Inserting PrimaryDataset %s with Type %s", datasetInfo["PrimaryDataset"], PrimaryDatasetType) primary = DbsPrimaryDataset(Name = datasetInfo["PrimaryDataset"], Type=PrimaryDatasetType) if apiRef != None: @@ -242,7 +241,7 @@ def createProcessedDataset(primaryDataset, algorithm, datasetInfo, if apiRef != None: apiRef.insertProcessedDataset(processedDataset) # - logging.debug("PrimaryDataset: %s ProcessedDataset: %s DataTierList: %s requested by PhysicsGroup: %s "%(primaryDataset['Name'],name,tierList,physicsGroup)) + logging.debug("PrimaryDataset: %s ProcessedDataset: %s DataTierList: %s requested by PhysicsGroup: %s ", primaryDataset['Name'], name, tierList, physicsGroup) return processedDataset @@ -326,7 +325,7 @@ def createDBSFiles(fjrFileInfo, jobType = None, apiRef = None): lumiList.append(lumi) - logging.debug("Lumi associated to file is: %s"%([x for x in lumiList])) + logging.debug("Lumi associated to file is: %s" % ([x for x in lumiList])) # // # // Dataset info related to files and creation of DbsFile object @@ -391,16 +390,16 @@ def getDBSFileBlock(dbsApiRef, procDataset, pnn): return that """ - logging.warning("getDBSFileBlocks(): dset, pnn: %s, %s" % (procDataset, pnn)) + logging.warning("getDBSFileBlocks(): dset, pnn: %s, %s", procDataset, pnn) allBlocks = dbsApiRef.listBlocks(procDataset, block_name = "*", phedex_node_name = "*") - logging.warning("getDBSFileBlock(): all blocks %s" % allBlocks) + logging.warning("getDBSFileBlock(): all blocks %s", allBlocks) openBlocks = [b for b in allBlocks if str(b['OpenForWriting']) == "1"] - logging.warning("getDBSFileBlocks(): open blocks %s" % openBlocks) + logging.warning("getDBSFileBlocks(): open blocks %s", openBlocks) blockRef = None if len(openBlocks) > 1: @@ -450,5 +449,5 @@ def getDBSFileBlock(dbsApiRef, procDataset, pnn): - logging.warning("Open FileBlock located at PNN: %s to use is FileBlock: %s "%(pnn,blockRef['Name'])) + logging.warning("Open FileBlock located at PNN: %s to use is FileBlock: %s ", pnn, blockRef['Name']) return blockRef diff --git a/src/python/WMCore/Services/DBS/ProdException.py b/src/python/WMCore/Services/DBS/ProdException.py index 0c19a0f73d..2fa63f5699 100644 --- a/src/python/WMCore/Services/DBS/ProdException.py +++ b/src/python/WMCore/Services/DBS/ProdException.py @@ -46,29 +46,29 @@ def __init__(self, message, errorNo=1000,**data): # // # // Automatically determine the module name #// if not set - if self['ModuleName'] == None: + if self.data['ModuleName'] == None: frame = inspect.currentframe() lastframe = inspect.getouterframes(frame)[1][0] excepModule = inspect.getmodule(lastframe) if excepModule != None: modName = excepModule.__name__ - self['ModuleName'] = modName + self.data['ModuleName'] = modName # // # // Find out where the exception came from #// stack = inspect.stack(1)[1] - self['FileName'] = stack[1] - self['LineNumber'] = stack[2] - self['MethodName'] = stack[3] + self.data['FileName'] = stack[1] + self.data['LineNumber'] = stack[2] + self.data['MethodName'] = stack[3] # // # // ClassName if ClassInstance is passed #// - if self['ClassInstance'] != None: - self['ClassName'] = \ - self['ClassInstance'].__class__.__name__ + if self.data['ClassInstance'] != None: + self.data['ClassName'] = \ + self.data['ClassInstance'].__class__.__name__ logging.error(str(self)) @@ -92,7 +92,7 @@ def addInfo(self, **data): exception instance """ for key, value in viewitems(data): - self[key] = value + self.data[key] = value return def xml(self): diff --git a/test/python/WMCore_t/ProcessPool_t/ProcessPool_t.py b/test/python/WMCore_t/ProcessPool_t/ProcessPool_t.py index da0a11379b..c859392b06 100644 --- a/test/python/WMCore_t/ProcessPool_t/ProcessPool_t.py +++ b/test/python/WMCore_t/ProcessPool_t/ProcessPool_t.py @@ -4,6 +4,7 @@ Unit tests for the ProcessPool class. """ +from builtins import range import unittest import nose