From 6294e5db6d3d14cbb57a3e8ed9560c19fc2173cb Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Thu, 15 Feb 2024 15:52:53 -0500 Subject: [PATCH] WIP: Move CI to GitHub Actions --- .circleci/config.yml | 118 +-------------------------------------- .github/workflows/ci.yml | 39 +++++++++++++ client/Makefile | 16 +++++- export/Makefile | 13 ++++- log/Makefile | 8 ++- proxy/Makefile | 3 + 6 files changed, 74 insertions(+), 123 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26e139e0e6..0afaf71d64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,42 +1,14 @@ --- common-steps: - - &client_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - &client_install_testing_dependencies run: name: Install testing dependencies command: | set -e - apt update && apt install -y git gnupg libarchive13 libmagic1 libqt5x11extras5 make python3-tk python3-dev gnupg python3-venv sqlite3 xvfb + apt update && apt install -y libarchive13 libmagic1 libqt5x11extras5 cd client poetry install --no-ansi - - &client_configure_locales - run: - name: Configure locales - command: | - set -e - apt update && apt install -y locales - echo "en_US ISO-8859-1" >> /etc/locale.gen - echo "en_US UTF-8" >> /etc/locale.gen - locale-gen - - &client_run_unit_tests run: name: Install requirements and run unit tests @@ -64,20 +36,6 @@ common-steps: export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata make test-functional - - &client_run_lint - run: - name: Run lint, type checking, code formatting - command: | - set -e - make -C client check-black check-isort lint mypy - - - &client_check_security - run: - name: Run static analysis on source code to find security issues - command: | - set -e - make -C client semgrep - - &client_check_source_strings run: name: Check that source strings are updated @@ -110,14 +68,6 @@ common-steps: exit 1 fi - - &export_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt update && apt install -y git gnupg make python3-dev gnupg python3-venv libnotify-bin - cd export - poetry install --no-ansi - - &export_run_unit_tests run: name: Install requirements and run unit tests @@ -126,75 +76,11 @@ common-steps: export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata make test - - &export_run_lint - run: - name: Run lint, type checking, code formatting - command: | - make -C export check-black lint - - - &export_check_security - run: - name: Run static analysis on source code to find security issues - command: | - make -C export semgrep - - - &log_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &log_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt-get install --yes --no-install-recommends git gnupg make - cd log - poetry install --no-ansi - - &log_run_tests run: name: Install requirements and run tests command: | - make -C log check - - - &proxy_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &proxy_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt-get install --yes --no-install-recommends git gnupg make - cd proxy - poetry install --no-ansi + make -C log test - &proxy_run_unit_tests run: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c4b2cc385..79e50531b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,45 @@ jobs: - name: Run lint run: make lint + component-lint: + strategy: + matrix: + component: + - client + - export + - log + - proxy + debian_version: + - bullseye + - bookworm + exclude: + - component: proxy + debian_version: bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make gnupg + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C ${{ matrix.component }} install + make -C ${{ matrix.component }} ci-install-deps + - name: Run lint + run: make -C ${{ matrix.component }} lint + safety: runs-on: ubuntu-latest container: debian:bookworm diff --git a/client/Makefile b/client/Makefile index 92d0485ac6..78b8be882c 100644 --- a/client/Makefile +++ b/client/Makefile @@ -109,12 +109,15 @@ test-functional: ## Run the functional tests test-sdk: ## Run just the sdk tests @poetry run pytest -v $(STESTS) -.PHONY: lint -lint: ## Run the linters +.PHONY: flake8 +flake8: ## Run flake8 linting @poetry run flake8 securedrop_client tests +.PHONY: lint +lint: check-black check-isort flake8 mypy semgrep ## Run all linters + .PHONY: check -check: clean check-black check-isort semgrep lint mypy test-random test-integration test-functional ## Run the full CI test suite +check: clean lint test-random test-integration test-functional ## Run the full CI test suite # Explanation of the below shell command should it ever break. # 1. Set the field separator to ": ##" and any make targets that might appear between : and ## @@ -186,3 +189,10 @@ verify-mo: ## Verify that all gettext machine objects (.mo) are reproducible fro @TERM=dumb poetry run scripts/verify-mo.py ${LOCALE_DIR}/* @# All good; now clean up. @git restore "${LOCALE_DIR}/**/*.po" + +# Install dependencies in CI +ci-install-deps: + apt-get install --yes xvfb sqlite3 locales + echo "en_US ISO-8859-1" >> /etc/locale.gen + echo "en_US UTF-8" >> /etc/locale.gen + locale-gen diff --git a/export/Makefile b/export/Makefile index ad65eaadef..4d1c318f83 100644 --- a/export/Makefile +++ b/export/Makefile @@ -1,8 +1,12 @@ .PHONY: all all: help +.PHONY: lint +lint: check-black flake8 mypy semgrep ## Run all linters + + .PHONY: check -check: lint mypy semgrep test check-black ## Run linter and tests +check: lint test ## Run linter and tests .PHONY: check-black check-black: ## Check Python source code formatting with black @@ -13,8 +17,8 @@ TESTS ?= tests test: ## Run tests poetry run pytest -v --cov-report html --cov-report term-missing --cov=securedrop_export $$TESTS -.PHONY: lint -lint: ## Run linter +.PHONY: flake8 +flake8: ## Run flake8 linter poetry run flake8 securedrop_export/ tests/ .PHONY: mypy @@ -54,3 +58,6 @@ help: ## Print this message and exit. @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ | sort \ | column -s ':' -t + +# Install dependencies in CI (none) +ci-install-deps: diff --git a/log/Makefile b/log/Makefile index 0d5febc562..fa4f78f6f4 100644 --- a/log/Makefile +++ b/log/Makefile @@ -2,7 +2,10 @@ DEFAULT_GOAL: help SHELL := /bin/bash .PHONY: check -check: flake8 mypy test +check: lint test + +.PHONY: lint +lint: flake8 mypy ## Run all linters .PHONY: flake8 flake8: ## Run flake8 to lint Python files @@ -28,3 +31,6 @@ help: ## Print this message and exit. @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ | sort \ | column -s ':' -t + +# Install dependencies in CI (none) +ci-install-deps: diff --git a/proxy/Makefile b/proxy/Makefile index a55836b46f..a72fc8080b 100644 --- a/proxy/Makefile +++ b/proxy/Makefile @@ -67,3 +67,6 @@ help: ## Print this message and exit. @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ | sort \ | column -s ':' -t + +# Install dependencies in CI (none) +ci-install-deps: