From 4f104676ac8ab7ec3b2c5ed95a9ca910fe82e4df Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 15 Sep 2022 16:20:05 +0100 Subject: [PATCH 01/19] Remove now-duplicated PR titles from release PR descriptions --- .github/update-release-branch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/update-release-branch.py b/.github/update-release-branch.py index 7697e59430..d7c5101a25 100644 --- a/.github/update-release-branch.py +++ b/.github/update-release-branch.py @@ -67,7 +67,7 @@ def open_pr( body.append('Merging ' + source_branch_short_sha + ' into ' + target_branch) body.append('') - body.append('Conductor for this PR is @' + conductor) + body.append(f'Conductor for this PR is @{conductor}.') # List all PRs merged if len(pull_requests) > 0: @@ -75,15 +75,15 @@ def open_pr( body.append('Contains the following pull requests:') for pr in pull_requests: merger = get_merger_of_pr(repo, pr) - body.append('- #' + str(pr.number) + ' - ' + pr.title +' (@' + merger + ')') + body.append(f'- #{pr.number} (@{merger})') # List all commits not part of a PR if len(commits_without_pull_requests) > 0: body.append('') body.append('Contains the following commits not from a pull request:') for commit in commits_without_pull_requests: - author_description = ' (@' + commit.author.login + ')' if commit.author is not None else '' - body.append('- ' + commit.sha + ' - ' + get_truncated_commit_message(commit) + author_description) + author_description = f' (@{commit.author.login})' if commit.author is not None else '' + body.append(f'- {commit.sha} - {get_truncated_commit_message(commit)}{author_description}') body.append('') body.append('Please review the following:') From 82495d8d86bbc6303d968ce12cfca8df2e397d03 Mon Sep 17 00:00:00 2001 From: Edoardo Pirovano Date: Fri, 16 Sep 2022 11:46:50 +0100 Subject: [PATCH 02/19] Add CHANGELOG note for TRAP caching --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9168cf5ec..c85abd3768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [UNRELEASED] -No user facing changes. +- We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it. ## 2.1.23 - 14 Sep 2022 From 9f79e5fbcf09120430a4028c39f739394d2e28ba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 16 Sep 2022 13:29:57 +0000 Subject: [PATCH 03/19] Update changelog and version after v2.1.24 --- CHANGELOG.md | 4 ++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2eea74753..c33991151d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CodeQL Action Changelog +## [UNRELEASED] + +No user facing changes. + ## 2.1.24 - 16 Sep 2022 No user facing changes. diff --git a/package-lock.json b/package-lock.json index 068c0b88a2..578467a0a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "license": "MIT", "dependencies": { "@actions/artifact": "^1.0.0", diff --git a/package.json b/package.json index 749671cc3e..ce0bb499f1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "private": true, "description": "CodeQL action", "scripts": { From e0ef82e596a4b449d841264a2eeebb044ce8f602 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 16 Sep 2022 13:51:53 +0000 Subject: [PATCH 04/19] Update checked-in dependencies --- node_modules/.package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json index 68e009d438..5fd07b39cb 100644 --- a/node_modules/.package-lock.json +++ b/node_modules/.package-lock.json @@ -1,6 +1,6 @@ { "name": "codeql", - "version": "2.1.24", + "version": "2.1.25", "lockfileVersion": 2, "requires": true, "packages": { From fb28913d5cf3c17277f02fc9dc585efb51843d76 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Fri, 16 Sep 2022 14:44:21 +0100 Subject: [PATCH 05/19] Add advice to the backport PR on how to run the checks quicker --- .github/update-release-branch.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/update-release-branch.py b/.github/update-release-branch.py index d7c5101a25..9d20e38ebf 100644 --- a/.github/update-release-branch.py +++ b/.github/update-release-branch.py @@ -86,21 +86,29 @@ def open_pr( body.append(f'- {commit.sha} - {get_truncated_commit_message(commit)}{author_description}') body.append('') - body.append('Please review the following:') + body.append('Please do the following:') if len(conflicted_files) > 0: - body.append(' - [ ] The `package.json` file contains the correct version.') - body.append(' - [ ] You have added commits to this branch that resolve the merge conflicts ' + + body.append(' - [ ] Ensure `package.json` file contains the correct version.') + body.append(' - [ ] Add commits to this branch to resolve the merge conflicts ' + 'in the following files:') body.extend([f' - [ ] `{file}`' for file in conflicted_files]) - body.append(' - [ ] Another maintainer has reviewed the additional commits you added to this ' + + body.append(' - [ ] Ensure another maintainer has reviewed the additional commits you added to this ' + 'branch to resolve the merge conflicts.') - body.append(' - [ ] The CHANGELOG displays the correct version and date.') - body.append(' - [ ] The CHANGELOG includes all relevant, user-facing changes since the last release.') - body.append(' - [ ] There are no unexpected commits being merged into the ' + target_branch + ' branch.') - body.append(' - [ ] The docs team is aware of any documentation changes that need to be released.') + body.append(' - [ ] Ensure the CHANGELOG displays the correct version and date.') + body.append(' - [ ] Ensure the CHANGELOG includes all relevant, user-facing changes since the last release.') + body.append(' - [ ] Check that there are not any unexpected commits being merged into the ' + target_branch + ' branch.') + body.append(' - [ ] Ensure the docs team is aware of any documentation changes that need to be released.') + + if not is_v2_release: + body.append(' - [ ] Remove and re-add the "Update dependencies" label to the PR to trigger just this workflow.') + body.append(' - [ ] Wait for the "Update dependencies" workflow to push a commit updating the dependencies.') + body.append(' - [ ] Mark the PR as ready for review to trigger the full set of PR checks.') + + body.append(' - [ ] Approve and merge this PR.') + if is_v2_release: - body.append(' - [ ] The mergeback PR is merged back into ' + source_branch + ' after this PR is merged.') - body.append(' - [ ] The v1 release PR is merged after this PR is merged.') + body.append(' - [ ] Merge the mergeback PR that will automatically be created once this PR is merged.') + body.append(' - [ ] Merge the v1 release PR that will automatically be created once this PR is merged.') title = 'Merge ' + source_branch + ' into ' + target_branch From 2e9fbe39e160d31dda7de132912bfb7efbef2a63 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Fri, 16 Sep 2022 14:53:45 +0100 Subject: [PATCH 06/19] Add advice to the mergeback PR on how to run the checks quicker --- .github/workflows/post-release-mergeback.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/post-release-mergeback.yml b/.github/workflows/post-release-mergeback.yml index f7a58dce21..7b8b8a7768 100644 --- a/.github/workflows/post-release-mergeback.yml +++ b/.github/workflows/post-release-mergeback.yml @@ -114,7 +114,17 @@ jobs: run: | set -exu pr_title="Mergeback ${VERSION} ${HEAD_BRANCH} into ${BASE_BRANCH}" - pr_body="Updates version and changelog." + pr_body=$(cat << EOF + This PR bumps the version number and updates the changelog after the ${VERSION} release. + + Please do the following: + + - [ ] Remove and re-add the "Update dependencies" label to the PR to trigger just this workflow. + - [ ] Wait for the "Update dependencies" workflow to push a commit updating the dependencies. + - [ ] Mark the PR as ready for review to trigger the full set of PR checks. + - [ ] Approve and merge the PR. + EOF + ) # Update the version number ready for the next release npm version patch --no-git-tag-version @@ -134,4 +144,5 @@ jobs: --title "${pr_title}" \ --label "Update dependencies" \ --body "${pr_body}" \ + --assignee "${GITHUB_ACTOR}" \ --draft From 5ffcfe95cc5ed5bae75ebdc037fff41b5b216e3c Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 16 Sep 2022 20:49:42 +0200 Subject: [PATCH 07/19] python-setup: Allow newest `virtualenv` Context for previous version is https://github.com/github/codeql-action/pull/862 Locally, I was able to install `20.15.1` with Python2. I don't see any reason why python3 version should be restricted. --- python-setup/install_tools.ps1 | 4 ++-- python-setup/install_tools.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python-setup/install_tools.ps1 b/python-setup/install_tools.ps1 index 3c78378a3d..8e0e8dc98a 100644 --- a/python-setup/install_tools.ps1 +++ b/python-setup/install_tools.ps1 @@ -5,8 +5,8 @@ py -3 -m pip install --user --upgrade pip setuptools wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't -py -2 -m pip install --user 'virtualenv<20.11' -py -3 -m pip install --user 'virtualenv<20.11' +py -2 -m pip install --user 'virtualenv!=20.12.0' +py -3 -m pip install --user virtualenv # We aren't compatible with poetry 1.2 py -3 -m pip install --user "poetry>=1.1,<1.2" diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index 7acb33f146..7468c2b15e 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -15,7 +15,7 @@ python3 -m pip install --user --upgrade pip setuptools wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't -python3 -m pip install --user 'virtualenv<20.11' +python3 -m pip install --user virtualenv # We install poetry with pip instead of the recommended way, since the recommended way # caused some problem since `poetry run` gives output like: @@ -42,5 +42,5 @@ if command -v python2 >/dev/null 2>&1; then python2 -m pip install --user --upgrade pip setuptools wheel - python2 -m pip install --user 'virtualenv<20.11' + python2 -m pip install --user 'virtualenv!=20.12.0' fi From e1ce6e3115eb3941ade27ef009d1245e4b0b74f1 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 19 Sep 2022 09:48:46 +0200 Subject: [PATCH 08/19] python-setup: Fix venv creation in Ubuntu 22.04 Fixes https://github.com/github/codeql-action/issues/1249 --- python-setup/install_tools.ps1 | 8 ++++++-- python-setup/install_tools.sh | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/python-setup/install_tools.ps1 b/python-setup/install_tools.ps1 index 3c78378a3d..796c0ecb9d 100644 --- a/python-setup/install_tools.ps1 +++ b/python-setup/install_tools.ps1 @@ -1,7 +1,11 @@ #! /usr/bin/pwsh -py -2 -m pip install --user --upgrade pip setuptools wheel -py -3 -m pip install --user --upgrade pip setuptools wheel +# while waiting for the next release of `virtualenv` after v20.16.5, we install an older +# version of `setuptools` to ensure that binaries are always put under +# `/bin`, which wouldn't always happen with the GitHub actions version of +# Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249 +py -2 -m pip install --user --upgrade pip 'setuptools<60' wheel +py -3 -m pip install --user --upgrade pip 'setuptools<60' wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index 7acb33f146..e2e5be36b6 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -11,7 +11,13 @@ set -e export PATH="$HOME/.local/bin:$PATH" # Setup Python 3 dependency installation tools. -python3 -m pip install --user --upgrade pip setuptools wheel + +# we install an older version of `setuptools` to ensure that binaries are always put +# under `/bin`, which wouldn't always happen with the GitHub actions version +# of Ubuntu 22.04. See https://github.com/github/codeql-action/issues/1249. The the next +# release of `virtualenv` after v20.16.5 will include a fix for this, so we can remove +# this bit of the logic again. +python3 -m pip install --user --upgrade pip 'setuptools<60' wheel # virtualenv is a bit nicer for setting up virtual environment, since it will provide up-to-date versions of # pip/setuptools/wheel which basic `python3 -m venv venv` won't @@ -40,7 +46,7 @@ if command -v python2 >/dev/null 2>&1; then curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2 fi - python2 -m pip install --user --upgrade pip setuptools wheel + python2 -m pip install --user --upgrade pip 'setuptools<60' wheel python2 -m pip install --user 'virtualenv<20.11' fi From 70509c388492a8d6facc01a9b4b815455a8ffdc2 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 19 Sep 2022 12:16:43 +0200 Subject: [PATCH 09/19] python-setup: Add support for Poetry 1.2 --- CHANGELOG.md | 1 + python-setup/auto_install_packages.py | 34 ++++++++++++++++++++++----- python-setup/install_tools.ps1 | 3 +-- python-setup/install_tools.sh | 3 +-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ef7c8eefb..a88dc80607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [UNRELEASED] - We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it. +- Add support for Python automatic dependency installation with Poetry 1.2. ## 2.1.24 - 16 Sep 2022 diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index b0a623735c..3efa955c17 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -9,27 +9,49 @@ import extractor_version -def _check_call(command): +def _check_call(command, extra_env=None): print('+ {}'.format(' '.join(command)), flush=True) - subprocess.check_call(command, stdin=subprocess.DEVNULL) + # only pass `env` argument if we need to pass in an updated environment + kwargs = {} + if extra_env: + new_env = os.environ.copy() + new_env.update(extra_env) + kwargs = {"env": new_env} -def _check_output(command): + subprocess.check_call(command, stdin=subprocess.DEVNULL, **kwargs) + + +def _check_output(command, extra_env=None): print('+ {}'.format(' '.join(command)), flush=True) - out = subprocess.check_output(command, stdin=subprocess.DEVNULL) + + # only pass `env` argument if we need to pass in an updated environment + kwargs = {} + if extra_env: + new_env = os.environ.copy() + new_env.update(extra_env) + kwargs = {"env": new_env} + + out = subprocess.check_output(command, stdin=subprocess.DEVNULL, **kwargs) print(out, flush=True) sys.stderr.flush() return out def install_packages_with_poetry(): + + # To handle poetry 1.2, which started to use keyring interaction MUCH more, we need + # add a workaround. See + # https://github.com/python-poetry/poetry/issues/2692#issuecomment-1235683370 + extra_poetry_env = {"PYTHON_KEYRING_BACKEND": "keyring.backends.null.Keyring"} + command = [sys.executable, '-m', 'poetry'] if sys.platform.startswith('win32'): # In windows the default path were the deps are installed gets wiped out between steps, # so we have to set it up to a folder that will be kept os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: - _check_call(command + ['install', '--no-root']) + _check_call(command + ['install', '--no-root'], extra_env=extra_poetry_env) except subprocess.CalledProcessError: sys.exit('package installation with poetry failed, see error above') @@ -38,7 +60,7 @@ def install_packages_with_poetry(): # virtualenv for the package, which was the case for using poetry for Python 2 when # default system interpreter was Python 3 :/ - poetry_out = _check_output(command + ['run', 'which', 'python']) + poetry_out = _check_output(command + ['run', 'which', 'python'], extra_env=extra_poetry_env) python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): diff --git a/python-setup/install_tools.ps1 b/python-setup/install_tools.ps1 index 3c78378a3d..6719966a2f 100644 --- a/python-setup/install_tools.ps1 +++ b/python-setup/install_tools.ps1 @@ -8,6 +8,5 @@ py -3 -m pip install --user --upgrade pip setuptools wheel py -2 -m pip install --user 'virtualenv<20.11' py -3 -m pip install --user 'virtualenv<20.11' -# We aren't compatible with poetry 1.2 -py -3 -m pip install --user "poetry>=1.1,<1.2" +py -3 -m pip install --user "poetry>=1.1" py -3 -m pip install --user pipenv diff --git a/python-setup/install_tools.sh b/python-setup/install_tools.sh index 7acb33f146..513c8892ae 100755 --- a/python-setup/install_tools.sh +++ b/python-setup/install_tools.sh @@ -24,8 +24,7 @@ python3 -m pip install --user 'virtualenv<20.11' # "program uses threads.", RuntimeWarning) # LGTM_PYTHON_SETUP_VERSION=The currently activated Python version 2.7.18 is not supported by the project (^3.5). Trying to find and use a compatible version. Using python3 (3.8.2) 3 -# We aren't compatible with poetry 1.2 -python3 -m pip install --user "poetry>=1.1,<1.2" +python3 -m pip install --user "poetry>=1.1" python3 -m pip install --user pipenv if command -v python2 >/dev/null 2>&1; then From 1309aafb7d240acd5da19a95eaf731dbc3ade184 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 20 Sep 2022 15:43:10 +0200 Subject: [PATCH 10/19] Update CHANGELOG.md Co-authored-by: Henry Mercer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a88dc80607..ef13145604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## [UNRELEASED] - We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it. -- Add support for Python automatic dependency installation with Poetry 1.2. +- Add support for Python automatic dependency installation with Poetry 1.2 [#1258](https://github.com/github/codeql-action/pull/1258). ## 2.1.24 - 16 Sep 2022 From 3f97671248b6456a0573d7033f812d6521f4ba52 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 09:08:04 +0200 Subject: [PATCH 11/19] python-setup: run tests on Ubuntu 22.04 --- .github/workflows/python-deps.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 3d64f171a3..a17d6723df 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, ubuntu-22.04, macos-latest] python_deps_type: [pipenv, poetry, requirements, setup_py] python_version: [2, 3] exclude: @@ -63,6 +63,7 @@ jobs: case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; + ubuntu-22.04*) basePath="/opt";; macos-latest*) basePath="/Users/runner";; esac echo ${basePath} @@ -86,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, ubuntu-22.04, macos-latest] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -109,6 +110,7 @@ jobs: case ${{ matrix.os }} in ubuntu-latest*) basePath="/opt";; + ubuntu-22.04*) basePath="/opt";; macos-latest*) basePath="/Users/runner";; esac echo ${basePath} From 22643072146458fa9db7e66439f15de42c40293d Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 10:01:57 +0200 Subject: [PATCH 12/19] python-setup: change `env` passing --- python-setup/auto_install_packages.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 3efa955c17..b134bf673a 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -9,30 +9,20 @@ import extractor_version -def _check_call(command, extra_env=None): +def _check_call(command, extra_env={}): print('+ {}'.format(' '.join(command)), flush=True) - # only pass `env` argument if we need to pass in an updated environment - kwargs = {} - if extra_env: - new_env = os.environ.copy() - new_env.update(extra_env) - kwargs = {"env": new_env} + env = os.environ.copy() + env.update(extra_env) + subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env) - subprocess.check_call(command, stdin=subprocess.DEVNULL, **kwargs) - -def _check_output(command, extra_env=None): +def _check_output(command, extra_env={}): print('+ {}'.format(' '.join(command)), flush=True) - # only pass `env` argument if we need to pass in an updated environment - kwargs = {} - if extra_env: - new_env = os.environ.copy() - new_env.update(extra_env) - kwargs = {"env": new_env} - - out = subprocess.check_output(command, stdin=subprocess.DEVNULL, **kwargs) + env = os.environ.copy() + env.update(extra_env) + out = subprocess.check_output(command, stdin=subprocess.DEVNULL, env=env) print(out, flush=True) sys.stderr.flush() return out From ca8a78d5f36e47a16a62363aed1e67b067c0cc0a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 10:02:51 +0200 Subject: [PATCH 13/19] python-setup: flush at the end of `_check_call` --- python-setup/auto_install_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index b134bf673a..abef8cc313 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -15,6 +15,8 @@ def _check_call(command, extra_env={}): env = os.environ.copy() env.update(extra_env) subprocess.check_call(command, stdin=subprocess.DEVNULL, env=env) + sys.stdout.flush() + sys.stderr.flush() def _check_output(command, extra_env={}): From 1fa5d728463866f63a4442b14b8b4ea5dfd1e0cc Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 14:50:13 +0200 Subject: [PATCH 14/19] python-setup: Fail early if installing for Python 2, and `python2` not available --- python-setup/auto_install_packages.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index b0a623735c..9731052716 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -5,6 +5,7 @@ import subprocess from tempfile import mkdtemp from typing import Optional +import shutil import extractor_version @@ -154,6 +155,17 @@ def install_packages(codeql_base_dir) -> Optional[str]: # get_extractor_version returns the Python version the extractor thinks this repo is using version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) + if version == 2 and not sys.platform.startswith('win32'): + # On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better + # error message than a traceback + `No such file or directory: 'python2'` + if shutil.which("python2") is None: + sys.exit( + "package installation failed: we detected this code as Python 2, but 'python2' executable was not available." + "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, " + "such as running 'sudo apt install python2' (Ubuntu 22.04)." + "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new" + ) + if os.path.exists('requirements.txt'): print('Found requirements.txt, will install packages with pip', flush=True) return install_requirements_txt_packages(version) From 93ba53f2de80a13ee5ef48ee7a6f68de121da298 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 15:34:36 +0200 Subject: [PATCH 15/19] add missing spaces --- python-setup/auto_install_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index 9731052716..edb9aa93a2 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -160,9 +160,9 @@ def install_packages(codeql_base_dir) -> Optional[str]: # error message than a traceback + `No such file or directory: 'python2'` if shutil.which("python2") is None: sys.exit( - "package installation failed: we detected this code as Python 2, but 'python2' executable was not available." + "package installation failed: we detected this code as Python 2, but 'python2' executable was not available. " "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, " - "such as running 'sudo apt install python2' (Ubuntu 22.04)." + "such as running 'sudo apt install python2' (Ubuntu 22.04). " "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new" ) From 8a893ddf189a2414cad54a3cdc31c8db322b2b27 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 15:34:44 +0200 Subject: [PATCH 16/19] python-setup: Flush even more --- python-setup/auto_install_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index edb9aa93a2..f58900b90f 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -154,6 +154,8 @@ def install_packages(codeql_base_dir) -> Optional[str]: # get_extractor_version returns the Python version the extractor thinks this repo is using version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) + sys.stdout.flush() + sys.stderr.flush() if version == 2 and not sys.platform.startswith('win32'): # On Ubuntu 22.04 'python2' is not available by default. We want to give a slightly better From b2fc1e178e0d4698fbfe13a3dacbd46e1f0dbe23 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 14:53:59 +0200 Subject: [PATCH 17/19] python-setup: Disable python2 tests on ubuntu-22.04 --- .github/workflows/python-deps.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index a17d6723df..9f43459dac 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -36,6 +36,9 @@ jobs: # Python2 and pipenv are not supported since pipenv v2021.11.5 - python_version: 2 python_deps_type: pipenv + # Python2 is not available on ubuntu-22.04 by default -- see https://github.com/github/codeql-action/pull/1257 + - python_version: 2 + os: ubuntu-22.04 env: From 32ca2cf500269f3c253386e63fb24f1ca7de5164 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 21 Sep 2022 16:06:07 +0200 Subject: [PATCH 18/19] Apply suggestions from code review Co-authored-by: Henry Mercer --- python-setup/auto_install_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-setup/auto_install_packages.py b/python-setup/auto_install_packages.py index f58900b90f..3deb5ad75a 100755 --- a/python-setup/auto_install_packages.py +++ b/python-setup/auto_install_packages.py @@ -162,9 +162,9 @@ def install_packages(codeql_base_dir) -> Optional[str]: # error message than a traceback + `No such file or directory: 'python2'` if shutil.which("python2") is None: sys.exit( - "package installation failed: we detected this code as Python 2, but 'python2' executable was not available. " + "Python package installation failed: we detected this code as Python 2, but the 'python2' executable was not available. " "To enable automatic package installation, please install 'python2' before the 'github/codeql-action/init' step, " - "such as running 'sudo apt install python2' (Ubuntu 22.04). " + "for example by running 'sudo apt install python2' (Ubuntu 22.04). " "If your code is not Python 2, but actually Python 3, please file a bug report at https://github.com/github/codeql-action/issues/new" ) From d1e2e02bee04071d46c2f6ccd3fbf3073f898cc6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 21 Sep 2022 15:40:11 +0000 Subject: [PATCH 19/19] Update changelog for v2.1.25 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef13145604..1e605219b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # CodeQL Action Changelog -## [UNRELEASED] +## 2.1.25 - 21 Sep 2022 - We will soon be rolling out a feature of the CodeQL Action that stores some information used to make future runs faster in the GitHub Actions cache. Initially, this will only be enabled on JavaScript repositories, but we plan to add more languages to this soon. The new feature can be disabled by passing the `trap-caching: false` option to your workflow's `init` step, for example if you are already using the GitHub Actions cache for a different purpose and are near the storage limit for it. - Add support for Python automatic dependency installation with Poetry 1.2 [#1258](https://github.com/github/codeql-action/pull/1258).