From 448e4fa11403dde190e9adac0b6de97510aeed57 Mon Sep 17 00:00:00 2001 From: khurtado Date: Mon, 25 Apr 2022 23:00:06 +0200 Subject: [PATCH] Remove support for Unpacking user tarballs. No longer used by CRAB. --- .../WMRuntime/Scripts/UnpackUserTarball.py | 136 ------------- .../WMCore/WMSpec/Steps/Executors/CMSSW.py | 28 +-- .../Scripts_t/UnpackUserTarball_t.py | 179 ------------------ 3 files changed, 2 insertions(+), 341 deletions(-) delete mode 100755 src/python/WMCore/WMRuntime/Scripts/UnpackUserTarball.py delete mode 100644 test/python/WMCore_t/WMRuntime_t/Scripts_t/UnpackUserTarball_t.py diff --git a/src/python/WMCore/WMRuntime/Scripts/UnpackUserTarball.py b/src/python/WMCore/WMRuntime/Scripts/UnpackUserTarball.py deleted file mode 100755 index 9765be877b..0000000000 --- a/src/python/WMCore/WMRuntime/Scripts/UnpackUserTarball.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python -""" -_UnpackUserTarball_ - -Unpack the user tarball and put it's contents in the right place -""" -from __future__ import print_function -from future import standard_library - -from Utils.Utilities import encodeUnicodeToBytes - -standard_library.install_aliases() - -import logging -import os -import hashlib -import re -import shutil -import subprocess -import sys -import tempfile - -import urllib.request -from urllib.parse import urlsplit -from subprocess import getstatusoutput - - -def setHttpProxy(url): - """ - Use frontier to figure out the http_proxies. - Pick one deterministically based on the url and loadbalance settings - """ - if 'http_proxy' in os.environ: - return os.environ['http_proxy'] - - status, output = getstatusoutput('cmsGetFnConnect frontier://smallfiles') - if status: - return None - - proxyList = re.findall('\(proxyurl=([\w\d\.\-\:\/]+)\)', output) - if 'loadbalance=proxies' in output: - urlHash = int(hashlib.sha1(encodeUnicodeToBytes(url)).hexdigest()[:15], 16) - proxy = proxyList[urlHash % len(proxyList)] - else: - proxy = proxyList[0] - os.environ['http_proxy'] = proxy - return proxy - - -def getRetriever(scheme): - """ - Get the right retriever function depending on the scheme. - If scheme is 'http' return urllib.urlretrieve, else if the scheme is https create a URLOpener - with certificates taken from the X509_USER_PROXY variable. If certificates are not available return - urllib.urlretrieve as for the http case. - """ - if 'X509_USER_PROXY' in os.environ and os.path.isfile(os.environ['X509_USER_PROXY']): - certfile = os.environ['X509_USER_PROXY'] - else: - if scheme == 'https': - logging.info("User proxy not found. Trying to retrieve the file without using certificates") - certfile = None - - if scheme == 'http' or not certfile: - retriever = urllib.request.urlretrieve - else: - logging.info("Using %s as X509 certificate", certfile) - op = urllib.request.URLopener(None, key_file=certfile, cert_file=certfile) - op.addheader('Accept', 'application/octet-stream') - retriever = op.retrieve - - return retriever - - -def UnpackUserTarball(): - tarballs = [] - userFiles = [] - if len(sys.argv) > 1: - tarballs = sys.argv[1].split(',') - if len(sys.argv) > 2: - userFiles = sys.argv[2].split(',') - - jobDir = os.environ['WMAGENTJOBDIR'] - - for tarball in tarballs: - splitResult = urlsplit(tarball) - tarFile = os.path.join(jobDir, os.path.basename(tarball)) - - # Is it a URL or a file that exists in the jobDir? - if splitResult[0] in ['xrootd', 'root']: - logging.info("Fetching tarball %s through xrootd", tarball) - try: - subprocess.check_call(['xrdcp', '-d', '1', '-f', tarball, 'TEMP_TARBALL.tgz']) - subprocess.check_call(['tar', 'xf', 'TEMP_TARBALL.tgz']) - except subprocess.CalledProcessError: - logging.error("Couldn't retrieve/extract file from xrootd") - raise - finally: - if os.path.exists('TEMP_TARBALL.tgz'): - os.unlink('TEMP_TARBALL.tgz') - - elif splitResult[0] in ['http', 'https'] and splitResult[1]: - retriever = getRetriever(splitResult[0]) - with tempfile.NamedTemporaryFile() as tempFile: - if setHttpProxy(tarball): - try: - logging.info('Fetching URL tarball %s through proxy server', tarball) - fileName, headers = retriever(tarball, tempFile.name) - except (RuntimeError, IOError): - del os.environ['http_proxy'] - logging.warning('Fetching URL tarball %s after proxy server failure', tarball) - fileName, headers = retriever(tarball, tempFile.name) - else: - logging.info('Fetching URL tarball %s without proxy server', tarball) - fileName, headers = retriever(tarball, tempFile.name) - - try: - subprocess.check_call(['tar', 'xf', fileName]) - except subprocess.CalledProcessError: - raise RuntimeError('Error extracting %s' % tarball) - elif os.path.isfile(tarFile): - logging.info("Untarring %s", tarFile) - subprocess.check_call(['tar', 'xf', tarFile]) - else: - raise IOError('%s does not exist' % tarFile) - - for userFile in userFiles: - if userFile: - logging.info("Moving '%s' to execution directory.", userFile) - shutil.move(userFile, '..') - - return 0 - - -if __name__ == '__main__': - sys.exit(UnpackUserTarball()) diff --git a/src/python/WMCore/WMSpec/Steps/Executors/CMSSW.py b/src/python/WMCore/WMSpec/Steps/Executors/CMSSW.py index 8479598818..2e4d045fcc 100644 --- a/src/python/WMCore/WMSpec/Steps/Executors/CMSSW.py +++ b/src/python/WMCore/WMSpec/Steps/Executors/CMSSW.py @@ -126,10 +126,6 @@ def execute(self, emulator=None): cmsswCommand = self.step.application.command.executable cmsswConfig = self.step.application.command.configuration cmsswArguments = self.step.application.command.arguments - userTarball = ','.join(self.step.user.inputSandboxes) - userFiles = ','.join(self.step.user.userFiles) - logging.info('User files are %s', userFiles) - logging.info('User sandboxes are %s', userTarball) scramArch = getSingleScramArch(scramArch) @@ -256,8 +252,6 @@ def execute(self, emulator=None): jobReportXML, cmsswCommand, cmsswConfig, - userTarball, - userFiles, cmsswArguments] logging.info("Executing CMSSW. args: %s", args) @@ -394,7 +388,7 @@ def post(self, emulator=None): if [ $# -lt $REQUIRED_ARGUMENT_COUNT ] then echo "Usage: `basename $0` \ - [Arguments for cmsRun]" + [Arguments for cmsRun]" exit 70 fi @@ -408,12 +402,8 @@ def post(self, emulator=None): JOB_REPORT=$6 EXECUTABLE=$7 CONFIGURATION=$8 -USER_TARBALL=$9 shift;shift;shift;shift;shift; -shift;shift;shift;shift; -# Can only do nine parameters at a time -USER_FILES=$1 -shift; +shift;shift;shift; echo "Setting up Frontier log level" export FRONTIER_LOG_LEVEL=warning @@ -438,20 +428,6 @@ def post(self, emulator=None): eval `$SCRAM_COMMAND runtime -sh` EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "***\nCouldn't get scram runtime: $EXIT_STATUS\n*"; exit 73; fi -if [ -n "$USER_TARBALL" ] ; then - python2 -m WMCore.WMRuntime.Scripts.UnpackUserTarball $USER_TARBALL $USER_FILES - EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 0 ]; then - echo "***\nCouldn't untar sandbox with python2: $EXIT_STATUS\n"; - echo "***\nWill try with python2.6 as it might be an old CMSSW release!" - python2.6 -m WMCore.WMRuntime.Scripts.UnpackUserTarball $USER_TARBALL $USER_FILES - EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 0 ]; then - echo "***\nCouldn't untar sandbox with python2.6: $EXIT_STATUS\n"; - exit 74; - fi - fi -fi echo "Completed SCRAM project" cd .. echo "Executing CMSSW" diff --git a/test/python/WMCore_t/WMRuntime_t/Scripts_t/UnpackUserTarball_t.py b/test/python/WMCore_t/WMRuntime_t/Scripts_t/UnpackUserTarball_t.py deleted file mode 100644 index c1525653dc..0000000000 --- a/test/python/WMCore_t/WMRuntime_t/Scripts_t/UnpackUserTarball_t.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python -""" -_UnpackUserTarball_t.py - -Tests for the user tarball unpacker and additional file mover - -""" - -from future import standard_library -standard_library.install_aliases() - -import urllib - -import logging -import os -import subprocess -import sys -import tempfile -import unittest - -from WMCore.WMRuntime.Scripts.UnpackUserTarball import UnpackUserTarball - -class UnpackUserTarballTest(unittest.TestCase): - """ - unittest for UnpackUserTarball script - - """ - - - # Set up a dummy logger - logger = logging.getLogger('UNITTEST') - logger.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.ERROR) - logger.addHandler(ch) - - - def setUp(self): - """ - Set up for unit tests - """ - - os.environ['WMAGENTJOBDIR'] = '/tmp/' - self.arch = 'slc5_ia32_gcc434' - self.version = 'CMSSW_3_8_7' - self.base = '/tmp/%s/' % self.version - self.localFile = '/%s/unittestTarball.tgz' % os.environ['WMAGENTJOBDIR'] - self.tempDir = tempfile.mkdtemp() - self.logger.debug("Using temp directory %s" % self.tempDir) - - self.origDir = os.getcwd() - - # Make a dummy CMSSW environment - - commands = [ - 'rm -rf %s' % self.base, - 'mkdir -p %s/lib/%s/' % (self.base, self.arch), - 'touch %s/lib/%s/libSomething.so' % (self.base, self.arch), - 'mkdir -p %s/src/Module/Submodule/data/' % (self.base), - 'touch %s/src/Module/Submodule/data/datafile.txt' % (self.base), - 'touch %s/extra_file.txt' % (self.base), - 'touch %s/extra_file2.txt' % (self.base), - 'touch %s/additional_file.txt' % (self.base), - 'tar -C %s -czf %s .' % (self.base, self.localFile), - ] - - for command in commands: - self.logger.debug("Executing command %s" % command) - subprocess.check_call(command.split(' ')) - os.mkdir(os.path.join(self.tempDir, self.version)) - os.chdir(os.path.join(self.tempDir, self.version)) - return - - - def tearDown(self): - """ - Clean up the files we've spewed all over - """ - os.chdir(self.origDir) - - subprocess.check_call(['rm', '-rf', self.tempDir]) - subprocess.check_call(['rm', '-rf', self.base]) - subprocess.check_call(['rm', '-f', self.localFile]) - - return - - - def testFileSandbox(self): - """ - _testFileSandbox_ - - Test a single sandbox that is a file - - """ - sys.argv = ['scriptName','unittestTarball.tgz',''] - UnpackUserTarball() - self.assertTrue(os.path.isfile('lib/slc5_ia32_gcc434/libSomething.so')) - - - def testBadFile(self): - """ - _testBadFile_ - - Test we get an exception from a non-existent file - - """ - sys.argv = ['scriptName','doesNotExist.tgz',''] - self.assertRaises(IOError, UnpackUserTarball) - - - def testUrlSandbox(self): - """ - _testUrlSandbox_ - - Test a single sandbox that is a URL - - """ - - sys.argv = ['scriptName','http://home.fnal.gov/~ewv/unittestTarball.tgz',''] - UnpackUserTarball() - self.assertTrue(os.path.isfile('lib/slc5_ia32_gcc434/libSomething.so')) - - - def testUrlNotTar(self): - """ - _testUrlSandbox_ - - Test a single sandbox that is a URL - - """ - - sys.argv = ['scriptName','http://home.fnal.gov/~ewv/index.html',''] - self.assertRaises(RuntimeError, UnpackUserTarball) - - - def testBadUrl(self): - """ - _testBadUrl_ - - Test a single sandbox that is a URL - - """ - - sys.argv = ['scriptName','http://home.fnal.gov/~ewv/not-there.txt',''] - self.assertRaises(urllib.error.HTTPError, UnpackUserTarball) - - - def testFileAndURLSandbox(self): - """ - _testFileAndURLSandbox_ - - Test two sandboxes. One a file, one a URL - - """ - - sys.argv = ['scriptName','unittestTarball.tgz,http://home.fnal.gov/~ewv/unittestTarball.tgz',''] - UnpackUserTarball() - # First is only in web tarball, second only in local - self.assertTrue(os.path.isfile('lib/slc5_ia32_gcc434/libSomething.so')) - self.assertTrue(os.path.isfile('lib/slc5_ia32_gcc434/libSomefile.so')) - - - def testFileRelocation(self): - """ - _testFileRelocation_ - - Test our ability to relocate files within the sandbox - - """ - - sys.argv = ['scriptName','unittestTarball.tgz','extra_file.txt,additional_file.txt'] - UnpackUserTarball() - self.assertTrue(os.path.isfile('../extra_file.txt')) - self.assertTrue(os.path.isfile('../additional_file.txt')) - - - -if __name__ == "__main__": - unittest.main()