From baa218c6e94972f9e50ade988997b204951437d1 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:26:26 -0700 Subject: [PATCH] Resolve Issues with `pypy` on Mac (#37734) * update `use-python-version` to no longer use local scripts for downloading additional python versions, the UsePythonVersion task now does this for us. * update `macos` agents to install `pypy39` and move to the AGENT_TOOLSDIRECTORY so that any invocations of python version `pypy39` no longer errors --- .../templates/jobs/run-cli-tests.yml | 6 +- .../templates/steps/use-python-version.yml | 48 ++--- eng/scripts/create-venv.ps1 | 8 +- .../devops_tasks/install_python_version.py | 186 ------------------ scripts/devops_tasks/use_pypy_version.py | 95 --------- 5 files changed, 21 insertions(+), 322 deletions(-) delete mode 100644 scripts/devops_tasks/install_python_version.py delete mode 100644 scripts/devops_tasks/use_pypy_version.py diff --git a/eng/pipelines/templates/jobs/run-cli-tests.yml b/eng/pipelines/templates/jobs/run-cli-tests.yml index d0a6b501b4c4..e9e966cdeb18 100644 --- a/eng/pipelines/templates/jobs/run-cli-tests.yml +++ b/eng/pipelines/templates/jobs/run-cli-tests.yml @@ -9,7 +9,7 @@ resources: parameters: - name: TargetRepoPackages type: object - default: + default: - 'sdk/core/azure-core' # a list of any resolvable pip install. EG: # - https:///blah.whl @@ -68,14 +68,14 @@ jobs: set -ev source env/bin/activate python -m pip install -e $(Build.SourcesDirectory)/${{ artifact }} - displayName: Install ${{ artifact }} + displayName: Install ${{ artifact }} - ${{ each package_spec in parameters.InjectedPackages }}: - bash: | set -ev source env/bin/activate python -m pip install -e ${{ package_spec }} - displayName: Install ${{ package_spec }} + displayName: Install ${{ package_spec }} - bash: | set -ev diff --git a/eng/pipelines/templates/steps/use-python-version.yml b/eng/pipelines/templates/steps/use-python-version.yml index 1a2fe4130e0b..e556500885da 100644 --- a/eng/pipelines/templates/steps/use-python-version.yml +++ b/eng/pipelines/templates/steps/use-python-version.yml @@ -2,42 +2,20 @@ parameters: versionSpec: '' steps: - # use python 3.8 for tooling. packaging. platform. - - task: UsePythonVersion@0 - displayName: "Use Python 3.8" - inputs: - versionSpec: 3.8 - - - pwsh: | - python -m pip install packaging==23.1 - displayName: Prep Environment - - # select the appropriate version from manifest if present - - task: PythonScript@0 - displayName: 'Install ${{ parameters.versionSpec }} from Python Manifest If Necessary' - inputs: - scriptPath: 'scripts/devops_tasks/install_python_version.py' - arguments: '${{ parameters.versionSpec }} --installer_folder="../_pyinstaller' - - # set up bypass of standard use python version - - pwsh: | - $incoming = "${{ parameters.versionSpec }}" - - if($incoming.Contains("pypy3")){ - Write-Host "##vso[task.setvariable variable=ManualInstallNecessary]true" - } - displayName: Check UsePythonVersion Necessity - - - task: PythonScript@0 - displayName: 'PyPy3 Specific Path Prepend' - condition: and(succeeded(), eq(variables.ManualInstallNecessary, 'true')) - inputs: - scriptPath: 'scripts/devops_tasks/use_pypy_version.py' - arguments: '${{ parameters.versionSpec }}' + # as of macos-14, pypy of all stripes is no longer available on the predefined MAC agents + # this script installs the newest version of pypy39 from the official pypy site into the hosted tool cache + - script: | + TOOL_LOCATION=$(AGENT_TOOLSDIRECTORY)/PyPy/3.9.4/x64 + curl -L -o pypy3.9-v7.3.16-macos_x86_64.tar.bz2 https://downloads.python.org/pypy/pypy3.9-v7.3.16-macos_x86_64.tar.bz2 + mkdir -p $TOOL_LOCATION + tar -xvjf pypy3.9-v7.3.16-macos_x86_64.tar.bz2 -C $TOOL_LOCATION --strip-components=1 + chmod -R 0755 $TOOL_LOCATION/bin + $TOOL_LOCATION/bin/python -m ensurepip + touch $TOOL_LOCATION/../x64.complete + displayName: Install pypy39 to hosted tool cache + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - # use - task: UsePythonVersion@0 - displayName: "Use Python $(PythonVersion)" - condition: and(succeeded(), not(eq(variables.ManualInstallNecessary, 'true'))) + displayName: "Use Python ${{ parameters.versionSpec }}" inputs: versionSpec: ${{ parameters.versionSpec }} diff --git a/eng/scripts/create-venv.ps1 b/eng/scripts/create-venv.ps1 index 919b3ba968fa..2824f6bc161d 100644 --- a/eng/scripts/create-venv.ps1 +++ b/eng/scripts/create-venv.ps1 @@ -1,4 +1,4 @@ -<#! +<#! .SYNOPSIS Creates a virtual environment for a CI machine. @@ -22,10 +22,12 @@ param( $venvPath = Join-Path $RepoRoot $VenvName if (!(Test-Path $venvPath)) { - Write-Host "Creating virtual environment '$VenvName'." + $invokingPython = (Get-Command "python").Source + Write-Host "Creating virtual environment '$VenvName' using python located at '$invokingPython'." python -m pip install virtualenv==20.25.1 python -m virtualenv "$venvPath" - Write-Host "Virtual environment '$VenvName' created." + $pythonVersion = python --version + Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion." Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath" Write-Host "##vso[task.setvariable variable=$($VenvName)_ACTIVATION_SCRIPT]if(`$IsWindows){. $venvPath/Scripts/Activate.ps1;}else {. $venvPath/bin/activate.ps1}" } diff --git a/scripts/devops_tasks/install_python_version.py b/scripts/devops_tasks/install_python_version.py deleted file mode 100644 index f731faf6e121..000000000000 --- a/scripts/devops_tasks/install_python_version.py +++ /dev/null @@ -1,186 +0,0 @@ -import platform -import json -import argparse -import urllib -import urllib.request -from subprocess import check_call, CalledProcessError -import sys -import os -import zipfile -import tarfile -import time - -from packaging.version import Version, InvalidVersion - -# SOURCE OF THIS FILE: https://github.com/actions/python-versions -# this is the official mapping file for gh-actions to retrieve python installers -MANIFEST_LOCATION = "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json" - -MAX_INSTALLER_RETRY = 3 -CURRENT_UBUNTU_VERSION = "20.04" # full title is ubuntu-20.04 -MAX_PRECACHED_VERSION = ( - "3.12.1" # reference: https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md#python -) - -UNIX_INSTALL_ARRAY = ["sh", "setup.sh"] -WIN_INSTALL_ARRAY = ["pwsh", "setup.ps1"] - - -def download_installer(remote_path, local_path): - retries = 0 - - while True: - try: - urllib.request.urlretrieve(remote_path, local_path) - break - except Exception as e: - print(e) - retries += 1 - - if retries >= MAX_INSTALLER_RETRY: - print("Unable to recover after attempting to download {} {} times".format(remote_path, retries)) - exit(1) - time.sleep(10) - - -def install_selected_python_version(installer_url, installer_folder): - current_plat = platform.system().lower() - - installer_folder = os.path.normpath(os.path.abspath(installer_folder)) - if not os.path.exists(installer_folder): - os.mkdir(installer_folder) - local_installer_ref = os.path.join( - installer_folder, - "local" + (".zip" if installer_folder.endswith("zip") else ".tar.gz"), - ) - - download_installer(installer_url, local_installer_ref) - - if current_plat == "windows": - with zipfile.ZipFile(local_installer_ref, "r") as zip_file: - zip_file.extractall(installer_folder) - try: - check_call(WIN_INSTALL_ARRAY, cwd=installer_folder) - except CalledProcessError as err: - print(err) - exit(1) - - else: - with tarfile.open(local_installer_ref) as tar_file: - tar_file.extractall(installer_folder) - try: - check_call(UNIX_INSTALL_ARRAY, cwd=installer_folder) - except CalledProcessError as err: - print(err) - exit(1) - - -# when given a string with major.minor only (the devops/gh standard) we need to find the latest one -# in the manifest for the version we're requesting. -def get_resolvable_version(requested_version, version_manifest): - target = Version(requested_version) - - if len(requested_version.split(".")) > 2: - return requested_version - else: - target_versions = [ - Version(version) for version in version_manifest.keys() if version.startswith(requested_version) - ] - - if target_versions: - return target_versions[0] - else: - print(f'Unable to select a valid version from manifest for version "{requested_version}"') - - -def get_installer_url(requested_version, version_manifest): - current_plat = platform.system().lower() - print("Current Platform Is {}".format(platform.platform())) - - actual_requested_version = get_resolvable_version(requested_version, version_manifest) - - if actual_requested_version in version_manifest: - found_installers = version_manifest[actual_requested_version]["files"] - - # filter anything that's not x64. we don't care. - x64_installers = [file_def for file_def in found_installers if file_def["arch"] == "x64"] - - if current_plat == "windows": - return [installer for installer in x64_installers if installer["platform"] == "win32"][0] - elif current_plat == "darwin": - return [installer for installer in x64_installers if installer["platform"] == current_plat][0] - else: - return [ - installer - for installer in x64_installers - if installer["platform"] == "linux" and installer["platform_version"] == CURRENT_UBUNTU_VERSION - ][0] - else: - print( - f"Requested version {actual_requested_version} is not available from the manifest at {MANIFEST_LOCATION}." - ) - - -def necessary_to_install(version_requested) -> bool: - version_from_spec = Version(version_requested) - precached_version = Version(MAX_PRECACHED_VERSION) - precached = True - - # Azure Devops UsePythonVersion@0 task issues a warning if the input python version is an exact value like "3.11.1" or "3.9.4." - # - # As a result, this script needs to verify that the major/minor combo is present on the box. Unfortunately, one cannot - # safely compare just against the MAX_PRECACHED_VERSION, as Version("3.11") generates to a version with value "3.11.0." - # 3.11.0 is _not_ greater than 3.11.1, and as such will fail an easy version comparison against max_precached_version. - # - # Instead, if we detect an input that has major/minor only, we compare that against major/minor of max_precached_version only. - # - # We do not include >= because if the input major.minor == the max_precached major.minor, then we know that input - # is already present. - # - # In cases where we _are_ given a full input, we can simply check against the max precached version. - if len(version_requested.split(".")) <= 2: - if version_from_spec > Version(f"{precached_version.major}.{precached_version.minor}"): - precached = False - else: - precached = version_from_spec <= precached_version - - return not precached - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="This python script ensures that a requested python version is present in the hostedtoolcache on azure devops agents. It does this by retrieving new versions of python from the gh-actions python manifest." - ) - - parser.add_argument( - "version_spec", - nargs="?", - help=("The version specifier passed in to the UsePythonVersion extended task."), - ) - - parser.add_argument( - "--installer_folder", - dest="installer_folder", - help=("The folder where the found installer will be extracted into and run from."), - ) - - args = parser.parse_args() - - try: - version_from_spec = Version(args.version_spec) - except InvalidVersion: - print("Invalid Version Spec. Skipping custom install.") - exit(0) - - if necessary_to_install(args.version_spec): - with urllib.request.urlopen(MANIFEST_LOCATION) as url: - version_manifest = json.load(url) - - version_dict = {i["version"]: i for i in version_manifest} - - print("Requested version {} is newer than versions pre-cached on agent. Invoking.".format(args.version_spec)) - - install_file_details = get_installer_url(args.version_spec, version_dict) - install_selected_python_version(install_file_details["download_url"], args.installer_folder) - else: - print(f'Requested version "{args.version_spec}" is precached on the current agent. Skipping installation.') diff --git a/scripts/devops_tasks/use_pypy_version.py b/scripts/devops_tasks/use_pypy_version.py deleted file mode 100644 index 8ef56333e355..000000000000 --- a/scripts/devops_tasks/use_pypy_version.py +++ /dev/null @@ -1,95 +0,0 @@ -import platform -import argparse -import sys -import os -from packaging.version import Version -from packaging.version import InvalidVersion - - -MAX_INSTALLER_RETRY = 3 -CURRENT_UBUNTU_VERSION = "20.04" # full title is ubuntu-20.04 -MAX_PRECACHED_VERSION = "3.10.0" -HOSTEDTOOLCACHE = os.getenv("AGENT_TOOLSDIRECTORY") - - -def walk_directory_for_pattern(spec): - target_directory = os.path.normpath(HOSTEDTOOLCACHE) - pypy_tool_cache = os.path.join(target_directory, "PyPy") - - print("Searching for {} in hosted tool cache {}".format(spec, target_directory)) - located_folders = [] - - discovered_tool_folders = os.listdir(pypy_tool_cache) - - # walk the folders, filter to the patterns established - for folder in discovered_tool_folders: - path, foldername = os.path.split(folder) - tool_version = Version(folder) - - if tool_version.major == spec.major and tool_version.minor == spec.minor: - found_tool_location = os.path.join(pypy_tool_cache, folder) - - # logic for this location is cribbed directly from existing UsePythonVersion type script task - # https://github.com/microsoft/azure-pipelines-tasks/blob/master/Tasks/UsePythonVersionV0/usepythonversion.ts#L32 - # Path should first be prepended by the install location, then by the executables directory. - # The executable directory is /bin on Ubuntu/Mac, /Scripts on Windows - # Additionally on windows, there are only x86 versions of the tool. on unix/mac there only exist x64 versions of the tool. - if platform.system() == "Windows": - install_path = os.path.join(found_tool_location, "x86") - tool_path = os.path.join(install_path, "Scripts") - - located_folders.extend([install_path, tool_path]) - else: - install_path = os.path.join(found_tool_location, "x64") - tool_path = os.path.join(install_path, "bin") - - located_folders.extend([install_path, tool_path]) - - return located_folders - - -def find_pypy_version(spec): - discovered_locations = walk_directory_for_pattern(spec) - - if not discovered_locations: - print( - "Unable to locate a valid executable folder for {}. Examined folder {}".format( - str(spec), HOSTEDTOOLCACHE - ) - ) - exit(1) - - return discovered_locations - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="""This python script is used to discover and prepend a devops agent path with a specific pypy version. - It is not intended to be generic, and should not be be used in general "Use Python Version X" kind of situations.""" - ) - - parser.add_argument( - "version_spec", - nargs="?", - help=("The version specifier passed in to the UsePythonVersion extended task."), - ) - - args = parser.parse_args() - max_precached_version = Version(MAX_PRECACHED_VERSION) - try: - version_from_spec = Version(args.version_spec.replace("pypy", "")) - except InvalidVersion: - print("Invalid Version '{}'. Exiting".format(args.version_spec)) - exit(1) - - discovered_installer_location = find_pypy_version(version_from_spec) - - print( - "Path should be prepended with discovered location{} [{}]".format( - ("s" if len(discovered_installer_location) >= 2 else ""), - ", ".join(discovered_installer_location), - ) - ) - - for path in discovered_installer_location: - print("##vso[task.prependpath]{}".format(path))