From 4873de707491e602add7d9881a19fbc4252019f9 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 29 Mar 2022 17:16:43 -0400 Subject: [PATCH 1/5] Add support to EL8 jobs --- src/python/WMCore/WMRuntime/Tools/Scram.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/python/WMCore/WMRuntime/Tools/Scram.py b/src/python/WMCore/WMRuntime/Tools/Scram.py index 12ac247776..d5d9602fe9 100644 --- a/src/python/WMCore/WMRuntime/Tools/Scram.py +++ b/src/python/WMCore/WMRuntime/Tools/Scram.py @@ -39,7 +39,10 @@ from Utils.Utilities import encodeUnicodeToBytesConditional, decodeBytesToUnicodeConditional SCRAM_TO_ARCH = {'amd64': 'X86_64', 'aarch64': 'aarch64', 'ppc64le': 'ppc64le'} -ARCH_TO_OS = {'slc5': ['rhel6'], 'slc6': ['rhel6'], 'slc7': ['rhel7']} +ARCH_TO_OS = {'slc5': ['rhel6'], + 'slc6': ['rhel6'], + 'slc7': ['rhel7'], + 'cc8': ['rhel8'], 'cs8': ['rhel8'], 'alma8': ['rhel8']} OS_TO_ARCH = {} for arch, oses in viewitems(ARCH_TO_OS): for osName in oses: From 35f33a563a16989149583cec2dceabed2699dfd9 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Thu, 31 Mar 2022 08:51:58 -0400 Subject: [PATCH 2/5] Add el8 flavor to the list of rhel8 container jobs --- src/python/WMCore/WMRuntime/Tools/Scram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/WMCore/WMRuntime/Tools/Scram.py b/src/python/WMCore/WMRuntime/Tools/Scram.py index d5d9602fe9..f85bef2a54 100644 --- a/src/python/WMCore/WMRuntime/Tools/Scram.py +++ b/src/python/WMCore/WMRuntime/Tools/Scram.py @@ -42,7 +42,7 @@ ARCH_TO_OS = {'slc5': ['rhel6'], 'slc6': ['rhel6'], 'slc7': ['rhel7'], - 'cc8': ['rhel8'], 'cs8': ['rhel8'], 'alma8': ['rhel8']} + 'el8': ['rhel8'], 'cc8': ['rhel8'], 'cs8': ['rhel8'], 'alma8': ['rhel8']} OS_TO_ARCH = {} for arch, oses in viewitems(ARCH_TO_OS): for osName in oses: From f73800c8313446cdbc6c813191f29039739252d0 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Mon, 11 Apr 2022 15:59:53 -0400 Subject: [PATCH 3/5] When ScramArch is empty str/list/None, return any as required_os --- src/python/WMCore/BossAir/Plugins/BasePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/WMCore/BossAir/Plugins/BasePlugin.py b/src/python/WMCore/BossAir/Plugins/BasePlugin.py index 4eedec0d83..f6981b2d27 100644 --- a/src/python/WMCore/BossAir/Plugins/BasePlugin.py +++ b/src/python/WMCore/BossAir/Plugins/BasePlugin.py @@ -138,7 +138,7 @@ def scramArchtoRequiredOS(scramArch=None): string to be matched for OS requirements for job """ requiredOSes = set() - if scramArch is None: + if not scramArch: requiredOSes.add('any') elif isinstance(scramArch, (str, bytes)): for arch, validOSes in viewitems(ARCH_TO_OS): From 709a166b623bfb4e2a6e2e9c829a089058a2ecd9 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Fri, 8 Apr 2022 21:25:34 -0400 Subject: [PATCH 4/5] If job requires rhel8 container, use slc7 python libraries Trying the new map of OS_ARCH replace slc7 by rhel7 apply Valentins suggestion --- etc/submit_py3.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/etc/submit_py3.sh b/etc/submit_py3.sh index f5968a533c..527afbae7d 100644 --- a/etc/submit_py3.sh +++ b/etc/submit_py3.sh @@ -54,6 +54,7 @@ PY3_FUTURE_VERSION=0.18.2 # Saving START_TIME and when job finishes END_TIME. WMA_MIN_JOB_RUNTIMESECS=300 START_TIME=$(date +%s) +WMA_DEFAULT_OS=rhel7 # assign arguments SANDBOX=$1 INDEX=$2 @@ -132,20 +133,17 @@ echo "======== WMAgent COMP Python bootstrap starting at $(TZ=GMT date) ======== # slc6_ppc64le_gcc493 (from CMSSW): py2-future/0.18.2 python/2.7.15 py3-future/0.18.2 python3/3.8.2 # # NOTE: all the ppc64le ScramArchs are actually pointing to: slc7_ppc64le_gcc820 +### UPDATE on 11 April, 2022: See a new map of CVMFS packages in: +# https://github.com/dmwm/WMCore/pull/11077#issuecomment-1094814966 + # First, decide which COMP ScramArch to use based on the required OS and Architecture THIS_ARCH=`uname -m` # if it's PowerPC, it returns `ppc64le` -if [ "$THIS_ARCH" = "x86_64" ] -then - THIS_ARCH="amd64" -fi -if [ "$REQUIRED_OS" = "rhel7" ] -then - WMA_SCRAM_ARCH=slc7_${THIS_ARCH}_gcc630 -elif [ "$THIS_ARCH" = "amd64" ] +# if this job can run at any OS, then use rhel7 as default +if [ "$REQUIRED_OS" = "any" ] then - WMA_SCRAM_ARCH=slc6_${THIS_ARCH}_gcc700 + WMA_SCRAM_ARCH=${WMA_DEFAULT_OS}_${THIS_ARCH} else - WMA_SCRAM_ARCH=slc6_${THIS_ARCH}_gcc493 + WMA_SCRAM_ARCH=${REQUIRED_OS}_${THIS_ARCH} fi echo "Job requires OS: $REQUIRED_OS, thus setting ScramArch to: $WMA_SCRAM_ARCH" From 510aa937057b2d18d28a5e7ce066a2236f65c717 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 12 Apr 2022 08:38:17 -0400 Subject: [PATCH 5/5] Enhance logic to map ScramArch to OS typo in the docstring --- .../WMCore/BossAir/Plugins/BasePlugin.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/python/WMCore/BossAir/Plugins/BasePlugin.py b/src/python/WMCore/BossAir/Plugins/BasePlugin.py index f6981b2d27..0dfbab16a2 100644 --- a/src/python/WMCore/BossAir/Plugins/BasePlugin.py +++ b/src/python/WMCore/BossAir/Plugins/BasePlugin.py @@ -6,7 +6,7 @@ """ from builtins import object, str, bytes -from future.utils import viewitems, viewvalues +from future.utils import viewvalues from Utils.Utilities import decodeBytesToUnicode from WMCore.WMException import WMException @@ -130,27 +130,25 @@ def updateSiteInformation(self, jobs, siteName, excludeSite): @staticmethod def scramArchtoRequiredOS(scramArch=None): """ + Matches a ScramArch - or a list of it - against a map of Scram + to Operating System - Args: - scramArch: string or list of scramArches that are acceptable for the job - - Returns: - string to be matched for OS requirements for job + :param scramArch: string or list of scramArches defined for a given job + :return: a string with the required OS to use """ - requiredOSes = set() + defaultValue = 'any' if not scramArch: - requiredOSes.add('any') - elif isinstance(scramArch, (str, bytes)): - for arch, validOSes in viewitems(ARCH_TO_OS): - if arch in scramArch: - requiredOSes.update(validOSes) - elif isinstance(scramArch, list): - for validArch in scramArch: - for arch, validOSes in viewitems(ARCH_TO_OS): - if arch in validArch: - requiredOSes.update(validOSes) - else: - requiredOSes.add('any') + return defaultValue + + requiredOSes = set() + if isinstance(scramArch, (str, bytes)): + scramArch = [scramArch] + elif not isinstance(scramArch, (list, tuple)): + return defaultValue + + for validArch in scramArch: + scramOS = validArch.split("_")[0] + requiredOSes.update(ARCH_TO_OS.get(scramOS, [])) return ','.join(sorted(requiredOSes))