From 70509c388492a8d6facc01a9b4b815455a8ffdc2 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 19 Sep 2022 12:16:43 +0200 Subject: [PATCH 1/4] python-setup: Add support for Poetry 1.2 --- CHANGELOG.md | 1 + python-setup/auto_install_packages.py | 34 ++++++++++++++++++++++----- python-setup/install_tools.ps1 | 3 +-- python-setup/install_tools.sh | 3 +-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ef7c8eefb..a88dc80607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [UNRELEASED] - We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it. +- Add support for Python automatic dependency installation with Poetry 1.2. ## 2.1.24 - 16 Sep 2022 diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index b0a623735c..3efa955c17 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -9,27 +9,49 @@ import extractor_version -def _check_call(command): +def _check_call(command, extra_env=None): print('+ {}'.format(' '.join(command)), flush=True) - subprocess.check_call(command, stdin=subprocess.DEVNULL) + # only pass `env` argument if we need to pass in an updated environment + kwargs = {} + if extra_env: + new_env = os.environ.copy() + new_env.update(extra_env) + kwargs = {"env": new_env} -def _check_output(command): + subprocess.check_call(command, stdin=subprocess.DEVNULL, **kwargs) + + +def _check_output(command, extra_env=None): print('+ {}'.format(' '.join(command)), flush=True) - out = subprocess.check_output(command, stdin=subprocess.DEVNULL) + + # only pass `env` argument if we need to pass in an updated environment + kwargs = {} + if extra_env: + new_env = os.environ.copy() + new_env.update(extra_env) + kwargs = {"env": new_env} + + out = subprocess.check_output(command, stdin=subprocess.DEVNULL, **kwargs) print(out, flush=True) sys.stderr.flush() return out def install_packages_with_poetry(): + + # To handle poetry 1.2, which started to use keyring interaction MUCH more, we need + # add a workaround. See + # https://github.com/python-poetry/poetry/issues/2692#issuecomment-1235683370 + extra_poetry_env = {"PYTHON_KEYRING_BACKEND": "keyring.backends.null.Keyring"} + command = [sys.executable, '-m', 'poetry'] if sys.platform.startswith('win32'): # In windows the default path were the deps are installed gets wiped out between steps, # so we have to set it up to a folder that will be kept os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: - _check_call(command + ['install', '--no-root']) + _check_call(command + ['install', '--no-root'], extra_env=extra_poetry_env) except subprocess.CalledProcessError: sys.exit('package installation with poetry failed, see error above') @@ -38,7 +60,7 @@ def install_packages_with_poetry(): # virtualenv for the package, which was the case for using poetry for Python 2 when # default system interpreter was Python 3 :/ - poetry_out = _check_output(command + ['run', 'which', 'python']) + poetry_out = _check_output(command + ['run', 'which', 'python'], extra_env=extra_poetry_env) python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): diff --git a/python-setup/install_tools.ps1 b/python-setup/install_tools.ps1 index 3c78378a3d..6719966a2f 100644 --- a/python-setup/install_tools.ps1 +++ b/python-setup/install_tools.ps1 @@ -8,6 +8,5 @@ py -3 -m pip install --user --upgrade pip setuptools wheel py -2 -m pip install --user 'virtualenv<20.11' py -3 -m pip install --user 'virtualenv<20.11' -# We aren't compatible with poetry 1.2 -py -3 -m pip install --user "poetry>=1.1,<1.2" +py -3 -m pip install --user "poetry>=1.1" py -3 -m pip install --user pipenv diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index 7acb33f146..513c8892ae 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -24,8 +24,7 @@ python3 -m pip install --user 'virtualenv<20.11' # "program uses threads.", RuntimeWarning) # LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3 -# We aren't compatible with poetry 1.2 -python3 -m pip install --user "poetry>=1.1,<1.2" +python3 -m pip install --user "poetry>=1.1" python3 -m pip install --user pipenv if command -v python2 >/dev/null 2>&1; then From 1309aafb7d240acd5da19a95eaf731dbc3ade184 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 20 Sep 2022 15:43:10 +0200 Subject: [PATCH 2/4] Update CHANGELOG.md Co-authored-by: Henry Mercer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a88dc80607..ef13145604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## [UNRELEASED] - We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it. -- Add support for Python automatic dependency installation with Poetry 1.2. +- Add support for Python automatic dependency installation with Poetry 1.2 [#1258](https://github.com/github/codeql-action/pull/1258). ## 2.1.24 - 16 Sep 2022 From 22643072146458fa9db7e66439f15de42c40293d Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 10:01:57 +0200 Subject: [PATCH 3/4] python-setup: change `env` passing --- python-setup/auto_install_packages.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 3efa955c17..b134bf673a 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -9,30 +9,20 @@ import extractor_version -def _check_call(command, extra_env=None): +def _check_call(command, extra_env={}): print('+ {}'.format(' '.join(command)), flush=True) - # only pass `env` argument if we need to pass in an updated environment - kwargs = {} - if extra_env: - new_env = os.environ.copy() - new_env.update(extra_env) - kwargs = {"env": new_env} + env = os.environ.copy() + env.update(extra_env) + subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env) - subprocess.check_call(command, stdin=subprocess.DEVNULL, **kwargs) - -def _check_output(command, extra_env=None): +def _check_output(command, extra_env={}): print('+ {}'.format(' '.join(command)), flush=True) - # only pass `env` argument if we need to pass in an updated environment - kwargs = {} - if extra_env: - new_env = os.environ.copy() - new_env.update(extra_env) - kwargs = {"env": new_env} - - out = subprocess.check_output(command, stdin=subprocess.DEVNULL, **kwargs) + env = os.environ.copy() + env.update(extra_env) + out = subprocess.check_output(command, stdin=subprocess.DEVNULL, env=env) print(out, flush=True) sys.stderr.flush() return out From ca8a78d5f36e47a16a62363aed1e67b067c0cc0a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 10:02:51 +0200 Subject: [PATCH 4/4] python-setup: flush at the end of `_check_call` --- python-setup/auto_install_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index b134bf673a..abef8cc313 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -15,6 +15,8 @@ def _check_call(command, extra_env={}): env = os.environ.copy() env.update(extra_env) subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env) + sys.stdout.flush() + sys.stderr.flush() def _check_output(command, extra_env={}):