From 7eeaa34fd960f8e2c268ad09f8ff120899abea61 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Tue, 31 Dec 2024 08:50:57 +0000 Subject: [PATCH] Move Trivy and Tics tests to separate scripts We're moving the Trivy and Tics test logic to separate scripts, simplifying the GH workflows and allowing these tests to be invoked locally. --- .github/workflows/nightly-test.yaml | 31 +--------------------- .github/workflows/security-scan.yaml | 31 ++-------------------- tests/tics-scan.sh | 34 ++++++++++++++++++++++++ tests/trivy-scan.sh | 39 ++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 59 deletions(-) create mode 100755 tests/tics-scan.sh create mode 100755 tests/trivy-scan.sh diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml index 9b1c65ebcb..5fd6aadc79 100644 --- a/.github/workflows/nightly-test.yaml +++ b/.github/workflows/nightly-test.yaml @@ -73,35 +73,6 @@ jobs: working-directory: src/k8s run: go mod download - name: TICS scan - # TODO: move the following to a script. run: | export TICSAUTHTOKEN=${{ secrets.TICSAUTHTOKEN }} - - set -x - # Install python dependencies - pip install -r tests/integration/requirements-test.txt - pip install -r tests/integration/requirements-dev.txt - - cd src/k8s - - # TICS requires us to have the test results in cobertura xml format under the - # directory use below - sudo make go.unit - go install github.com/boumenot/gocover-cobertura@latest - gocover-cobertura < coverage.txt > coverage.xml - mkdir .coverage - mv ./coverage.xml ./.coverage/ - - # Install the TICS and staticcheck - go install honnef.co/go/tools/cmd/staticcheck@v0.5.1 - . <(curl --silent --show-error 'https://canonical.tiobe.com/tiobeweb/TICS/api/public/v1/fapi/installtics/Script?cfg=default&platform=linux&url=https://canonical.tiobe.com/tiobeweb/TICS/') - - # We need to have our project built - # We load the dqlite libs here instead of doing through make because TICS - # will try to build parts of the project itself - sudo add-apt-repository -y ppa:dqlite/dev - sudo apt install dqlite-tools-v2 libdqlite1.17-dev - sudo make clean - go build -a ./... - - TICSQServer -project k8s-snap -tmpdir /tmp/tics -branchdir $HOME/work/k8s-snap/k8s-snap/ + tests/tics-scan.sh diff --git a/.github/workflows/security-scan.yaml b/.github/workflows/security-scan.yaml index 044b6dc758..7d061aafd6 100644 --- a/.github/workflows/security-scan.yaml +++ b/.github/workflows/security-scan.yaml @@ -45,36 +45,9 @@ jobs: ref: ${{ inputs.checkout-ref }} # Persist downloaded artifacts clean: 'false' - # TODO: move the following steps to a separate script - - name: Setup Trivy vulnerability scanner + - name: Run Trivy vulnerability scanner run: | - mkdir -p manual-trivy/sarifs - pushd manual-trivy - VER=$(curl --silent -qI https://github.com/aquasecurity/trivy/releases/latest | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}'); - wget https://github.com/aquasecurity/trivy/releases/download/${VER}/trivy_${VER#v}_Linux-64bit.tar.gz - tar -zxvf ./trivy_${VER#v}_Linux-64bit.tar.gz - popd - - name: Run Trivy vulnerability scanner in repo mode - uses: aquasecurity/trivy-action@master - with: - scan-type: "fs" - ignore-unfixed: true - format: "sarif" - output: "trivy-k8s-repo-scan--results.sarif" - severity: "MEDIUM,HIGH,CRITICAL" - env: - TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db" - - name: Gather Trivy repo scan results - run: | - cp trivy-k8s-repo-scan--results.sarif ./manual-trivy/sarifs/ - - name: Run Trivy vulnerability scanner on the snap - run: | - for var in $(env | grep -o '^TRIVY_[^=]*'); do - unset "$var" - done - cp ${{ steps.download-snap.outputs.snap-path }} ./k8s-test.snap - unsquashfs k8s-test.snap - ./manual-trivy/trivy --db-repository public.ecr.aws/aquasecurity/trivy-db rootfs ./squashfs-root/ --format sarif > ./manual-trivy/sarifs/snap.sarif + ./tests/trivy-scan.sh ${{ steps.download-snap.outputs.snap-path }} - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v3 with: diff --git a/tests/tics-scan.sh b/tests/tics-scan.sh new file mode 100755 index 0000000000..866f69dbd8 --- /dev/null +++ b/tests/tics-scan.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(dirname "$BASH_SOURCE") + +set -ex +cd "${SCRIPT_DIR}/.." + +# Install python dependencies +pip install -r tests/integration/requirements-test.txt +pip install -r tests/integration/requirements-dev.txt + +cd src/k8s + +# TICS requires us to have the test results in cobertura xml format under the +# directory use below +sudo make go.unit +go install github.com/boumenot/gocover-cobertura@latest +gocover-cobertura < coverage.txt > coverage.xml +mkdir -p .coverage +mv ./coverage.xml ./.coverage/ + +# Install the TICS and staticcheck +go install honnef.co/go/tools/cmd/staticcheck@v0.5.1 +. <(curl --silent --show-error 'https://canonical.tiobe.com/tiobeweb/TICS/api/public/v1/fapi/installtics/Script?cfg=default&platform=linux&url=https://canonical.tiobe.com/tiobeweb/TICS/') + +# We need to have our project built +# We load the dqlite libs here instead of doing through make because TICS +# will try to build parts of the project itself +sudo add-apt-repository -y ppa:dqlite/dev +sudo apt install dqlite-tools-v2 libdqlite1.17-dev +sudo make clean +go build -a ./... + +TICSQServer -project k8s-snap -tmpdir /tmp/tics -branchdir $HOME/work/k8s-snap/k8s-snap/ diff --git a/tests/trivy-scan.sh b/tests/trivy-scan.sh new file mode 100755 index 0000000000..08c8de098b --- /dev/null +++ b/tests/trivy-scan.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(dirname "$BASH_SOURCE") + +set -ex +cd "${SCRIPT_DIR}/.." + +SNAP_PATH="$1" +if [[ ! -f $SNAP_PATH ]]; then + echo "Usage: $0 " + exit 1 +fi + +# Setup Trivy vulnerability scanner +mkdir -p manual-trivy/sarifs +pushd manual-trivy +VER=$(curl --silent -qI https://github.com/aquasecurity/trivy/releases/latest | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}'); +wget https://github.com/aquasecurity/trivy/releases/download/${VER}/trivy_${VER#v}_Linux-64bit.tar.gz +tar -zxvf ./trivy_${VER#v}_Linux-64bit.tar.gz +popd + +# Run Trivy vulnerability scanner in repo mode +./manual-trivy/trivy fs . \ + --format sarif \ + --db-repository public.ecr.aws/aquasecurity/trivy-db \ + --severity "MEDIUM,HIGH,CRITICAL" \ + --ignore-unfixed \ + > ./manual-trivy/sarifs/trivy-k8s-repo-scan--results.sarif + +for var in $(env | grep -o '^TRIVY_[^=]*'); do + unset "$var" +done +cp "${SNAP_PATH}" ./k8s-test.snap +rm -rf ./squashfs-root +unsquashfs k8s-test.snap +./manual-trivy/trivy rootfs ./squashfs-root/ \ + --format sarif \ + --db-repository public.ecr.aws/aquasecurity/trivy-db \ + > ./manual-trivy/sarifs/snap.sarif