WIP: Move CI to GitHub Actions

freedomofpress · Feb 15, 2024 · 6294e5d · 6294e5d
1 parent e7b7d0c
commit 6294e5d
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 123 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,42 +1,14 @@
 ---
 common-steps:
-  - &client_install_poetry
-    run:
-      name: Install Poetry
-      command: |
-        set -e
-        source /etc/os-release
-        if [[ "$VERSION_CODENAME" == "bullseye" ]]; then
-            # Install Poetry via PyPI
-            apt-get update && apt-get install --yes --no-install-recommends python3-pip
-            pip install poetry==1.6.1
-        elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then
-            # Install Poetry via system package
-            apt-get update && apt-get install --yes --no-install-recommends python3-poetry
-        else
-            echo "Unsupported Debian version: $VERSION_CODENAME"
-            exit 1
-        fi
-
   - &client_install_testing_dependencies
     run:
       name: Install testing dependencies
       command: |
         set -e
-        apt update && apt install -y git gnupg libarchive13 libmagic1 libqt5x11extras5 make python3-tk python3-dev gnupg python3-venv sqlite3 xvfb
+        apt update && apt install -y libarchive13 libmagic1 libqt5x11extras5
         cd client
         poetry install --no-ansi
 
-  - &client_configure_locales
-    run:
-      name: Configure locales
-      command: |
-        set -e
-        apt update && apt install -y locales
-        echo "en_US ISO-8859-1" >> /etc/locale.gen
-        echo "en_US UTF-8" >> /etc/locale.gen
-        locale-gen
-
   - &client_run_unit_tests
     run:
       name: Install requirements and run unit tests
@@ -64,20 +36,6 @@ common-steps:
         export PYTHONPATH=$PYTHONPATH:.  # so alembic can get to Base metadata
         make test-functional
 
-  - &client_run_lint
-    run:
-      name: Run lint, type checking, code formatting
-      command: |
-        set -e
-        make -C client check-black check-isort lint mypy
-
-  - &client_check_security
-    run:
-      name: Run static analysis on source code to find security issues
-      command: |
-        set -e
-        make -C client semgrep
-
   - &client_check_source_strings
     run:
       name: Check that source strings are updated
@@ -110,14 +68,6 @@ common-steps:
             exit 1
         fi
 
-  - &export_install_testing_dependencies
-    run:
-      name: Install testing dependencies
-      command: |
-        apt update && apt install -y git gnupg make python3-dev gnupg python3-venv libnotify-bin
-        cd export
-        poetry install --no-ansi
-
   - &export_run_unit_tests
     run:
       name: Install requirements and run unit tests
@@ -126,75 +76,11 @@ common-steps:
         export PYTHONPATH=$PYTHONPATH:.  # so alembic can get to Base metadata
         make test
 
-  - &export_run_lint
-    run:
-      name: Run lint, type checking, code formatting
-      command: |
-        make -C export check-black lint
-
-  - &export_check_security
-    run:
-      name: Run static analysis on source code to find security issues
-      command: |
-        make -C export semgrep
-
-  - &log_install_poetry
-    run:
-      name: Install Poetry
-      command: |
-        set -e
-        source /etc/os-release
-        if [[ "$VERSION_CODENAME" == "bullseye" ]]; then
-            # Install Poetry via PyPI
-            apt-get update && apt-get install --yes --no-install-recommends python3-pip
-            pip install poetry==1.6.1
-        elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then
-            # Install Poetry via system package
-            apt-get update && apt-get install --yes --no-install-recommends python3-poetry
-        else
-            echo "Unsupported Debian version: $VERSION_CODENAME"
-            exit 1
-        fi
-
-  - &log_install_testing_dependencies
-    run:
-      name: Install testing dependencies
-      command: |
-        apt-get install --yes --no-install-recommends git gnupg make
-        cd log
-        poetry install --no-ansi
-
   - &log_run_tests
     run:
       name: Install requirements and run tests
       command: |
-        make -C log check
-
-  - &proxy_install_poetry
-    run:
-      name: Install Poetry
-      command: |
-        set -e
-        source /etc/os-release
-        if [[ "$VERSION_CODENAME" == "bullseye" ]]; then
-            # Install Poetry via PyPI
-            apt-get update && apt-get install --yes --no-install-recommends python3-pip
-            pip install poetry==1.6.1
-        elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then
-            # Install Poetry via system package
-            apt-get update && apt-get install --yes --no-install-recommends python3-poetry
-        else
-            echo "Unsupported Debian version: $VERSION_CODENAME"
-            exit 1
-        fi
-
-  - &proxy_install_testing_dependencies
-    run:
-      name: Install testing dependencies
-      command: |
-        apt-get install --yes --no-install-recommends git gnupg make
-        cd proxy
-        poetry install --no-ansi
+        make -C log test
 
   - &proxy_run_unit_tests
     run:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -52,6 +52,45 @@ jobs:
       - name: Run lint
         run: make lint
 
+  component-lint:
+    strategy:
+      matrix:
+        component:
+          - client
+          - export
+          - log
+          - proxy
+        debian_version:
+          - bullseye
+          - bookworm
+        exclude:
+          - component: proxy
+            debian_version: bookworm
+    runs-on: ubuntu-latest
+    container: debian:${{ matrix.debian_version }}
+    steps:
+      - run: |
+          apt-get update && apt-get install --yes git make gnupg
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          source /etc/os-release
+          if [[ "$VERSION_CODENAME" == "bullseye" ]]; then
+              # Install Poetry via PyPI
+              apt-get install --yes --no-install-recommends python3-pip
+              pip install poetry==1.6.1
+          elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then
+              # Install Poetry via system package
+              apt-get install --yes --no-install-recommends python3-poetry
+          else
+              echo "Unsupported Debian version: $VERSION_CODENAME"
+              exit 1
+          fi
+          poetry -C ${{ matrix.component }} install
+          make -C ${{ matrix.component }} ci-install-deps
+      - name: Run lint
+        run: make -C ${{ matrix.component }} lint
+
   safety:
     runs-on: ubuntu-latest
     container: debian:bookworm

diff --git a/client/Makefile b/client/Makefile
@@ -109,12 +109,15 @@ test-functional: ## Run the functional tests
 test-sdk: ## Run just the sdk tests
 	@poetry run pytest -v $(STESTS)
 
-.PHONY: lint
-lint: ## Run the linters
+.PHONY: flake8
+flake8: ## Run flake8 linting
 	@poetry run flake8 securedrop_client tests
 
+.PHONY: lint
+lint: check-black check-isort flake8 mypy semgrep ## Run all linters
+
 .PHONY: check
-check: clean check-black check-isort semgrep lint mypy test-random test-integration test-functional ## Run the full CI test suite
+check: clean lint test-random test-integration test-functional ## Run the full CI test suite
 
 # Explanation of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
@@ -186,3 +189,10 @@ verify-mo: ## Verify that all gettext machine objects (.mo) are reproducible fro
 	@TERM=dumb poetry run scripts/verify-mo.py ${LOCALE_DIR}/*
 	@# All good; now clean up.
 	@git restore "${LOCALE_DIR}/**/*.po"
+
+# Install dependencies in CI
+ci-install-deps:
+	apt-get install --yes xvfb sqlite3 locales
+	echo "en_US ISO-8859-1" >> /etc/locale.gen
+	echo "en_US UTF-8" >> /etc/locale.gen
+	locale-gen
diff --git a/export/Makefile b/export/Makefile
@@ -1,8 +1,12 @@
 .PHONY: all
 all: help
 
+.PHONY: lint
+lint: check-black flake8 mypy semgrep  ## Run all linters
+
+
 .PHONY: check
-check: lint mypy semgrep test check-black ## Run linter and tests
+check: lint test ## Run linter and tests
 
 .PHONY: check-black
 check-black: ## Check Python source code formatting with black
@@ -13,8 +17,8 @@ TESTS ?= tests
 test:  ## Run tests
 	poetry run pytest -v --cov-report html --cov-report term-missing --cov=securedrop_export $$TESTS
 
-.PHONY: lint
-lint:  ## Run linter
+.PHONY: flake8
+flake8:  ## Run flake8 linter
 	poetry run flake8 securedrop_export/ tests/
 
 .PHONY: mypy
@@ -54,3 +58,6 @@ help: ## Print this message and exit.
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
 		| sort \
 | column -s ':' -t
+
+# Install dependencies in CI (none)
+ci-install-deps:
diff --git a/log/Makefile b/log/Makefile
@@ -2,7 +2,10 @@ DEFAULT_GOAL: help
 SHELL := /bin/bash
 
 .PHONY: check
-check: flake8 mypy test
+check: lint test
+
+.PHONY: lint
+lint: flake8 mypy ## Run all linters
 
 .PHONY: flake8
 flake8: ## Run flake8 to lint Python files
@@ -28,3 +31,6 @@ help: ## Print this message and exit.
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
 		| sort \
 		| column -s ':' -t
+
+# Install dependencies in CI (none)
+ci-install-deps:
diff --git a/proxy/Makefile b/proxy/Makefile
@@ -67,3 +67,6 @@ help: ## Print this message and exit.
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
 		| sort \
 		| column -s ':' -t
+
+# Install dependencies in CI (none)
+ci-install-deps: