From b5d217c91a9720c9ab0e87612caea7711cdae4bd Mon Sep 17 00:00:00 2001 From: Michael Still Date: Tue, 30 Apr 2024 12:39:58 +1000 Subject: [PATCH 01/14] Attempt to express functional tests as a matrix. --- .github/workflows/functional-tests.yml | 221 +++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 .github/workflows/functional-tests.yml diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml new file mode 100644 index 000000000..f40953334 --- /dev/null +++ b/.github/workflows/functional-tests.yml @@ -0,0 +1,221 @@ +name: Functional tests + +# NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} +# which is available as GITHUB_WORKSPACE. You can find other environment +# variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables + +on: + push: + branches: + - develop + - v*-releases + pull_request: + branches: + - develop + - v*-releases + +jobs: + requirements_check: + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} + cancel-in-progress: true + + steps: + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Attempt to install requirements + run: | + cd ${GITHUB_WORKSPACE} + python3 -mvenv venv + venv/bin/pip3 install -r shakenfist/requirements.txt + + functional_matrix: + needs: requirements_check + strategy: + matrix: + os: [ + {base_image: 'debian:12', base_image_user: 'debian', topology: 'localhost'} + ] + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} + cancel-in-progress: true + + steps: + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image={{ matrix.os.base_image }} base_image_user={{ matrix.os.base_image_user }}" \ + deploy/ansible/ci-topology-{{ matrix.os.topology }}.yml + + - name: Copy CI tools to primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -rp tools \ + {{ matrix.os.base_image_user }}@${primary}:. + + - name: Log github actions buffering status + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} python3 tools/buffer.py + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + + - name: Wait for API to start answering + run: | + set +e + + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' + + count=0 + while [ $count -lt 60 ] + do + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client instance list' + if [ $? == 0 ]; then + exit 0 + fi + + count=$(( $count + 1 )) + sleep 5 + done + + exit 1 + + - name: Import cached images + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' + + - name: Create a base level of activity in the cluster + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} \ + 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' + echo "" + + - name: Run functional tests + timeout-minutes: 120 + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + $source_path/shakenfist \ + {{ matrix.os.base_image_user }}@${primary}:shakenfist + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{concurrency}}; stestr slowest" + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/{{ matrix.os.base_image_user }}/tools/ci_event_checks.sh + + # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup + # cost. That's not really something we can control, so just ignore the CPU + # usage of that process instead. + - name: Check SF process CPU usage + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' + + - name: Check for reasonable data rates + timeout-minutes: 5 + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/{{ matrix.os.base_image_user }}/tools/ci_event_checks.sh + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + {{ matrix.os.base_image_user }}@${primary}:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user={{ matrix.os.base_image_user }} \ + ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip From 0f69d72b5578fe6e2da512d3c40605df5c0c3946 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Tue, 30 Apr 2024 13:15:14 +1000 Subject: [PATCH 02/14] Convert other functional tests. --- .github/workflows/configure-tests.py | 30 --- .../functional-tests-debian-11-localhost.yml | 195 ------------------ ...unctional-tests-debian-11-slim-primary.yml | 194 ----------------- ...unctional-tests-debian-12-slim-primary.yml | 195 ------------------ ...functional-tests-ubuntu-2004-localhost.yml | 195 ------------------ ...ctional-tests-ubuntu-2204-slim-primary.yml | 195 ------------------ .github/workflows/functional-tests.tmpl | 195 ------------------ .github/workflows/functional-tests.yml | 61 +++++- .github/workflows/python-unit-tests.yml | 47 ----- 9 files changed, 59 insertions(+), 1248 deletions(-) delete mode 100644 .github/workflows/functional-tests-debian-11-localhost.yml delete mode 100644 .github/workflows/functional-tests-debian-11-slim-primary.yml delete mode 100644 .github/workflows/functional-tests-debian-12-slim-primary.yml delete mode 100644 .github/workflows/functional-tests-ubuntu-2004-localhost.yml delete mode 100644 .github/workflows/functional-tests-ubuntu-2204-slim-primary.yml delete mode 100644 .github/workflows/functional-tests.tmpl delete mode 100644 .github/workflows/python-unit-tests.yml diff --git a/.github/workflows/configure-tests.py b/.github/workflows/configure-tests.py index 9012326a7..3b9e959f6 100644 --- a/.github/workflows/configure-tests.py +++ b/.github/workflows/configure-tests.py @@ -89,36 +89,6 @@ 'scheduled': False }, ], - 'functional-tests': [ - { - 'name': 'debian-11-localhost', - 'baseimage': 'sf://label/ci-images/debian-11', - 'baseuser': 'debian', - 'topology': 'localhost', - 'concurrency': 3 - }, - { - 'name': 'debian-12-slim-primary', - 'baseimage': 'sf://label/ci-images/debian-12', - 'baseuser': 'debian', - 'topology': 'slim-primary', - 'concurrency': 5 - }, - { - 'name': 'ubuntu-2004-localhost', - 'baseimage': 'sf://label/ci-images/ubuntu-2004', - 'baseuser': 'ubuntu', - 'topology': 'localhost', - 'concurrency': 3 - }, - { - 'name': 'ubuntu-2204-slim-primary', - 'baseimage': 'sf://label/ci-images/ubuntu-2204', - 'baseuser': 'ubuntu', - 'topology': 'slim-primary', - 'concurrency': 5 - }, - ], 'scheduled-tests': [ { 'name': 'develop-debian-11-localhost', diff --git a/.github/workflows/functional-tests-debian-11-localhost.yml b/.github/workflows/functional-tests-debian-11-localhost.yml deleted file mode 100644 index 810b4f822..000000000 --- a/.github/workflows/functional-tests-debian-11-localhost.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test debian-11-localhost - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - debian-11-localhost: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-localhost.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=3; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-debian-11-slim-primary.yml b/.github/workflows/functional-tests-debian-11-slim-primary.yml deleted file mode 100644 index 5f1f93e7b..000000000 --- a/.github/workflows/functional-tests-debian-11-slim-primary.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: Test debian-11-slim-primary - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - debian-11-slim-primary: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=5; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-debian-12-slim-primary.yml b/.github/workflows/functional-tests-debian-12-slim-primary.yml deleted file mode 100644 index 3eade4657..000000000 --- a/.github/workflows/functional-tests-debian-12-slim-primary.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test debian-12-slim-primary - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - debian-12-slim-primary: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-12 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=5; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-ubuntu-2004-localhost.yml b/.github/workflows/functional-tests-ubuntu-2004-localhost.yml deleted file mode 100644 index be6536945..000000000 --- a/.github/workflows/functional-tests-ubuntu-2004-localhost.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test ubuntu-2004-localhost - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - ubuntu-2004-localhost: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/ubuntu-2004 base_image_user=ubuntu" \ - deploy/ansible/ci-topology-localhost.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - ubuntu@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - ubuntu@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=3; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=ubuntu ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-ubuntu-2204-slim-primary.yml b/.github/workflows/functional-tests-ubuntu-2204-slim-primary.yml deleted file mode 100644 index fe582ea6c..000000000 --- a/.github/workflows/functional-tests-ubuntu-2204-slim-primary.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test ubuntu-2204-slim-primary - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - ubuntu-2204-slim-primary: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/ubuntu-2204 base_image_user=ubuntu" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - ubuntu@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - ubuntu@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=5; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=ubuntu ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests.tmpl b/.github/workflows/functional-tests.tmpl deleted file mode 100644 index f110d3359..000000000 --- a/.github/workflows/functional-tests.tmpl +++ /dev/null @@ -1,195 +0,0 @@ -name: Test {{name}} - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - {{name}}: - runs-on: self-hosted - concurrency: - group: {% raw %}${{ github.workflow }}-${{ github.ref }}{% endraw %} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA={% raw %}${{{% endraw %} github.event.pull_request.head.sha {% raw %}}}{% endraw %}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo {% raw %}${{{% endraw %} github.repository {% raw %}}}{% endraw %} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image={{baseimage}} base_image_user={{baseuser}}" \ - deploy/ansible/ci-topology-{{topology}}.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - {{baseuser}}@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - {{baseuser}}@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{concurrency}}; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user={{baseuser}} ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index f40953334..e30164631 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -15,7 +15,24 @@ on: - v*-releases jobs: + lint: + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-lint + cancel-in-progress: true + + steps: + - name: Checkout code with two commits + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Lint with flake8 + run: | + /usr/bin/tox -eflake8 + requirements_check: + needs: lint runs-on: self-hosted concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} @@ -34,12 +51,52 @@ jobs: python3 -mvenv venv venv/bin/pip3 install -r shakenfist/requirements.txt + build38: + needs: lint + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build38 + cancel-in-progress: true + + steps: + - name: Checkout code with two commits + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Run python3 unit tests + run: | + /usr/bin/tox -epy3 + functional_matrix: needs: requirements_check strategy: matrix: os: [ - {base_image: 'debian:12', base_image_user: 'debian', topology: 'localhost'} + { + base_image: 'sf://label/ci-images/debian-11', + base_image_user: 'debian', + topology: 'localhost', + concurrency: 3 + }, + { + base_image: 'sf://label/ci-images/debian-12', + base_image_user: 'debian', + topology: 'slim-primary', + concurrency: 5 + }, + { + base_image: 'sf://label/ci-images/ubuntu-2004', + base_image_user: 'ubuntu', + topology: 'localhost', + concurrency: 3 + }, + { + base_image: 'sf://label/ci-images/ubuntu-2204', + base_image_user: 'ubuntu', + topology: 'slim-primary', + concurrency: 5 + }, ] runs-on: self-hosted concurrency: @@ -159,7 +216,7 @@ jobs: $source_path/shakenfist \ {{ matrix.os.base_image_user }}@${primary}:shakenfist ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{concurrency}}; stestr slowest" + {{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{ matix.os.concurrency}}; stestr slowest" - name: Check logs if: always() diff --git a/.github/workflows/python-unit-tests.yml b/.github/workflows/python-unit-tests.yml deleted file mode 100644 index b08a17bc1..000000000 --- a/.github/workflows/python-unit-tests.yml +++ /dev/null @@ -1,47 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python application - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - lint: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-lint - cancel-in-progress: true - - steps: - - name: Checkout code with two commits - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Lint with flake8 - run: | - /usr/bin/tox -eflake8 - - build38: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-build38 - cancel-in-progress: true - - steps: - - name: Checkout code with two commits - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Run python3 unit tests - run: | - /usr/bin/tox -epy3 From 3bded44318ec17878301224d9fda4554152d3270 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Tue, 30 Apr 2024 19:08:04 +1000 Subject: [PATCH 03/14] Move ansible, node, and upgrade tests into functional tests. --- .github/workflows/ansible-module-tests.yml | 215 ---------- .github/workflows/functional-tests.yml | 469 +++++++++++++++++++++ .github/workflows/nodelifecycle-tests.yml | 169 -------- .github/workflows/upgrade-tests.yml | 124 ------ 4 files changed, 469 insertions(+), 508 deletions(-) delete mode 100644 .github/workflows/ansible-module-tests.yml delete mode 100644 .github/workflows/nodelifecycle-tests.yml delete mode 100644 .github/workflows/upgrade-tests.yml diff --git a/.github/workflows/ansible-module-tests.yml b/.github/workflows/ansible-module-tests.yml deleted file mode 100644 index 12c206149..000000000 --- a/.github/workflows/ansible-module-tests.yml +++ /dev/null @@ -1,215 +0,0 @@ -name: Ansible module testing - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - ansible-modules: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Run ansible module tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash ansiblemoduletests.sh" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary sudo chmod ugo+r /var/log/syslog - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary:/var/log/syslog \ - ${GITHUB_WORKSPACE}/syslog - - failures=0 - - echo - etcd_conns=`grep -c "Building new etcd connection" ${GITHUB_WORKSPACE}/syslog || true` - echo "This CI run created $etcd_conns etcd connections." - if [ $etcd_conns -gt 5000 ]; then - echo "FAILURE: Too many etcd clients!" - failures=1 - fi - - echo - sigterms=`grep -c "Sent SIGTERM to " ${GITHUB_WORKSPACE}/syslog || true` - echo "This CI run sent $sigterms SIGTERM signals while shutting down." - if [ $sigterms -gt 50 ]; then - echo "FAILURE: Too many SIGTERMs sent!" - failures=1 - fi - - FORBIDDEN=("Traceback (most recent call last):" - "ERROR sf" - "ERROR gunicorn" - " died" - "Extra vxlan present" - "Fork support is only compatible with the epoll1 and poll polling strategies" - "not using configured address" - "Dumping thread traces" - "because it is leased to" - "not committing online upgrade" - "Received a GOAWAY with error code ENHANCE_YOUR_CALM" - "ConnectionFailedError" - "invalid JWT in Authorization header" - "Libvirt Error: XML error" - "Cleaning up leaked IPAM" - "Cleaning up leaked vxlan" - "Waiting to acquire lock" - 'apparmor="DENIED"' - "Ignoring malformed cache entry") - IFS="" - for forbid in ${FORBIDDEN[*]} - do - if [ `grep -c "$forbid" ${GITHUB_WORKSPACE}/syslog || true` -gt 0 ] - then - echo "FAILURE: Forbidden string found in logs: $forbid" - failures=1 - fi - done - - if [ $failures -gt 0 ]; then - echo "...failures detected." - exit 1 - fi - - - name: Check process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs' - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - chown -R debian.debian /home/debian/.ssh - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index e30164631..b2ed172e3 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -31,6 +31,9 @@ jobs: run: | /usr/bin/tox -eflake8 + # NOTE(mikal): we need this one early because renovate creates a lot of PRs + # where the proposed combination of dependencies doesn't actually install and + # I don't want to waste a lot of CI resources on them. requirements_check: needs: lint runs-on: self-hosted @@ -276,3 +279,469 @@ jobs: retention-days: 90 if-no-files-found: error path: /srv/github/artifacts/bundle.zip + + ansible-modules: + needs: requirements_check + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ + deploy/ansible/ci-topology-slim-primary.yml + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + + - name: Wait for API to start answering + run: | + set +e + + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' + + count=0 + while [ $count -lt 60 ] + do + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary '. /etc/sf/sfrc; sf-client instance list' + if [ $? == 0 ]; then + exit 0 + fi + + count=$(( $count + 1 )) + sleep 5 + done + + exit 1 + + - name: Import cached images + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' + + - name: Run ansible module tests + timeout-minutes: 120 + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + $source_path/shakenfist \ + debian@$primary:shakenfist + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash ansiblemoduletests.sh" + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary sudo chmod ugo+r /var/log/syslog + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary:/var/log/syslog \ + ${GITHUB_WORKSPACE}/syslog + + failures=0 + + echo + etcd_conns=`grep -c "Building new etcd connection" ${GITHUB_WORKSPACE}/syslog || true` + echo "This CI run created $etcd_conns etcd connections." + if [ $etcd_conns -gt 5000 ]; then + echo "FAILURE: Too many etcd clients!" + failures=1 + fi + + echo + sigterms=`grep -c "Sent SIGTERM to " ${GITHUB_WORKSPACE}/syslog || true` + echo "This CI run sent $sigterms SIGTERM signals while shutting down." + if [ $sigterms -gt 50 ]; then + echo "FAILURE: Too many SIGTERMs sent!" + failures=1 + fi + + FORBIDDEN=("Traceback (most recent call last):" + "ERROR sf" + "ERROR gunicorn" + " died" + "Extra vxlan present" + "Fork support is only compatible with the epoll1 and poll polling strategies" + "not using configured address" + "Dumping thread traces" + "because it is leased to" + "not committing online upgrade" + "Received a GOAWAY with error code ENHANCE_YOUR_CALM" + "ConnectionFailedError" + "invalid JWT in Authorization header" + "Libvirt Error: XML error" + "Cleaning up leaked IPAM" + "Cleaning up leaked vxlan" + "Waiting to acquire lock" + 'apparmor="DENIED"' + "Ignoring malformed cache entry") + IFS="" + for forbid in ${FORBIDDEN[*]} + do + if [ `grep -c "$forbid" ${GITHUB_WORKSPACE}/syslog || true` -gt 0 ] + then + echo "FAILURE: Forbidden string found in logs: $forbid" + failures=1 + fi + done + + if [ $failures -gt 0 ]; then + echo "...failures detected." + exit 1 + fi + + - name: Check process CPU usage + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs' + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + chown -R debian.debian /home/debian/.ssh + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip + + + node-lifecycle: + needs: requirements_check + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ + deploy/ansible/ci-topology-slim-primary.yml + + - name: Copy CI tools to primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -rp tools \ + debian@$primary:. + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + + - name: Wait for API to start answering + run: | + set +e + + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' + + count=0 + while [ $count -lt 60 ] + do + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary '. /etc/sf/sfrc; sf-client instance list' + if [ $? == 0 ]; then + exit 0 + fi + + count=$(( $count + 1 )) + sleep 5 + done + + exit 1 + + - name: Import cached images + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' + + - name: Run node lifecycle tests + timeout-minutes: 120 + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + $source_path/shakenfist \ + debian@$primary:shakenfist + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash nodelifecycletests.sh" + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh + + - name: Restart Shaken Fist nodes so we can collect logs + if: always() + run: | + set -x + . ${GITHUB_WORKSPACE}/ci-environment.sh + for node in ${sf1_uuid} ${sf2_uuid} ${sf3_uuid} ${sf4_uuid} ${sf5_uuid}; do + sf-client instance reboot --hard ${node} + done + sleep 90 + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + chown -R debian.debian /home/debian/.ssh + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip + + + upgrade-0-6-15: + needs: requirements_check + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ + deploy/ansible/ci-topology-localhost-upgrade.yml + + - name: Copy CI tools to primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -rp tools \ + debian@$primary:. + + - name: Run getsf to only install etcd, and then restore a backup + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper --tags bootstrap,etcd + + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/*; . /etc/sf/sfrc; /srv/shakenfist/venv/bin/sf-backup restore /srv/ci/backup-0.6.15-20230319' + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/*' + + - name: Wait for five minutes + run: sleep 300 + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/nodelifecycle-tests.yml b/.github/workflows/nodelifecycle-tests.yml deleted file mode 100644 index 88f633546..000000000 --- a/.github/workflows/nodelifecycle-tests.yml +++ /dev/null @@ -1,169 +0,0 @@ -name: Node lifecycle testing - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - node-lifecycle: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Run node lifecycle tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash nodelifecycletests.sh" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Restart Shaken Fist nodes so we can collect logs - if: always() - run: | - set -x - . ${GITHUB_WORKSPACE}/ci-environment.sh - for node in ${sf1_uuid} ${sf2_uuid} ${sf3_uuid} ${sf4_uuid} ${sf5_uuid}; do - sf-client instance reboot --hard ${node} - done - sleep 90 - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - chown -R debian.debian /home/debian/.ssh - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip diff --git a/.github/workflows/upgrade-tests.yml b/.github/workflows/upgrade-tests.yml deleted file mode 100644 index 3f5264015..000000000 --- a/.github/workflows/upgrade-tests.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Upgrade testing - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - upgrade-0-6-15: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-localhost-upgrade.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Run getsf to only install etcd, and then restore a backup - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper --tags bootstrap,etcd - - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/*; . /etc/sf/sfrc; /srv/shakenfist/venv/bin/sf-backup restore /srv/ci/backup-0.6.15-20230319' - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/*' - - - name: Wait for five minutes - run: sleep 300 - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file From 9a8a034b90da99ba85c9c1c90742043af84a592d Mon Sep 17 00:00:00 2001 From: Michael Still Date: Tue, 30 Apr 2024 19:30:36 +1000 Subject: [PATCH 04/14] Be consistent with case. --- .github/workflows/functional-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index b2ed172e3..7ad540664 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -34,7 +34,7 @@ jobs: # NOTE(mikal): we need this one early because renovate creates a lot of PRs # where the proposed combination of dependencies doesn't actually install and # I don't want to waste a lot of CI resources on them. - requirements_check: + requirements-check: needs: lint runs-on: self-hosted concurrency: @@ -72,7 +72,7 @@ jobs: /usr/bin/tox -epy3 functional_matrix: - needs: requirements_check + needs: requirements-check strategy: matrix: os: [ @@ -281,7 +281,7 @@ jobs: path: /srv/github/artifacts/bundle.zip ansible-modules: - needs: requirements_check + needs: requirements-check runs-on: self-hosted concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -482,7 +482,7 @@ jobs: node-lifecycle: - needs: requirements_check + needs: requirements-check runs-on: self-hosted concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -637,7 +637,7 @@ jobs: upgrade-0-6-15: - needs: requirements_check + needs: requirements-check runs-on: self-hosted concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 91f479eae9183d959b1cabc2f4a919b837e1b577 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 09:11:04 +1000 Subject: [PATCH 05/14] Log matrix details. --- .github/workflows/functional-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 7ad540664..6b03499bb 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -107,6 +107,13 @@ jobs: cancel-in-progress: true steps: + - name: Log matrix details + run: | + echo "base_image: {{ matrix.os.base_image }}" + echo "base_image_user: {{ matrixo.s.base_image_user }}" + echo "topology: {{ matrix.os.topology }}" + echo "concurrency: {{ matrix.os.concurrency }}" + - name: Set environment variables run: | echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV From a9ed90a879717864b5ffc79dfbeed3b6257ad721 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 09:20:25 +1000 Subject: [PATCH 06/14] Fix variable substitutions. --- .github/workflows/functional-tests.yml | 54 ++++++++++++++------------ 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 6b03499bb..fe5e16c8e 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -77,24 +77,28 @@ jobs: matrix: os: [ { + job_name: 'debian-11-localhost', base_image: 'sf://label/ci-images/debian-11', base_image_user: 'debian', topology: 'localhost', concurrency: 3 }, { + job_name: 'debian-12-slim-primary', base_image: 'sf://label/ci-images/debian-12', base_image_user: 'debian', topology: 'slim-primary', concurrency: 5 }, { + job_name: 'ubuntu-2004-localhost', base_image: 'sf://label/ci-images/ubuntu-2004', base_image_user: 'ubuntu', topology: 'localhost', concurrency: 3 }, { + job_name: 'ubuntu-2204-slim-primary', base_image: 'sf://label/ci-images/ubuntu-2204', base_image_user: 'ubuntu', topology: 'slim-primary', @@ -103,16 +107,16 @@ jobs: ] runs-on: self-hosted concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os.job_name }} cancel-in-progress: true steps: - name: Log matrix details run: | - echo "base_image: {{ matrix.os.base_image }}" - echo "base_image_user: {{ matrixo.s.base_image_user }}" - echo "topology: {{ matrix.os.topology }}" - echo "concurrency: {{ matrix.os.concurrency }}" + echo "base_image: ${{ matrix.os.base_image }}" + echo "base_image_user: ${{ matrixo.s.base_image_user }}" + echo "topology: ${{ matrix.os.topology }}" + echo "concurrency: ${{ matrix.os.concurrency }}" - name: Set environment variables run: | @@ -135,8 +139,8 @@ jobs: cd ${GITHUB_WORKSPACE}/shakenfist ansible-playbook -i /home/debian/ansible-hosts \ --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image={{ matrix.os.base_image }} base_image_user={{ matrix.os.base_image_user }}" \ - deploy/ansible/ci-topology-{{ matrix.os.topology }}.yml + base_image=${{ matrix.os.base_image }} base_image_user=${{ matrix.os.base_image_user }}" \ + deploy/ansible/ci-topology-${{ matrix.os.topology }}.yml - name: Copy CI tools to primary run: | @@ -144,7 +148,7 @@ jobs: cd ${GITHUB_WORKSPACE}/shakenfist scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null -rp tools \ - {{ matrix.os.base_image_user }}@${primary}:. + ${{ matrix.os.base_image_user }}@${primary}:. - name: Log github actions buffering status run: | @@ -157,12 +161,12 @@ jobs: . ${GITHUB_WORKSPACE}/ci-environment.sh ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} /tmp/getsf-wrapper + ${{ matrix.os.base_image_user }}@${primary} /tmp/getsf-wrapper echo "" echo "" ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} \ + ${{ matrix.os.base_image_user }}@${primary} \ 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - name: Wait for API to start answering @@ -171,13 +175,13 @@ jobs: . ${GITHUB_WORKSPACE}/ci-environment.sh ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' + ${{ matrix.os.base_image_user }}@${primary} 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' count=0 while [ $count -lt 60 ] do ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client instance list' + ${{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client instance list' if [ $? == 0 ]; then exit 0 fi @@ -192,28 +196,28 @@ jobs: run: | . ${GITHUB_WORKSPACE}/ci-environment.sh ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} \ + ${{ matrix.os.base_image_user }}@${primary} \ '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} \ + ${{ matrix.os.base_image_user }}@${primary} \ '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} \ + ${{ matrix.os.base_image_user }}@${primary} \ '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} \ + ${{ matrix.os.base_image_user }}@${primary} \ '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - name: Create a base level of activity in the cluster run: | . ${GITHUB_WORKSPACE}/ci-environment.sh ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} \ + ${{ matrix.os.base_image_user }}@${primary} \ 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' echo "" echo "" ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} \ + ${{ matrix.os.base_image_user }}@${primary} \ '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' echo "" @@ -224,16 +228,16 @@ jobs: scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ $source_path/shakenfist \ - {{ matrix.os.base_image_user }}@${primary}:shakenfist + ${{ matrix.os.base_image_user }}@${primary}:shakenfist ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{ matix.os.concurrency}}; stestr slowest" + ${{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{ matix.os.concurrency}}; stestr slowest" - name: Check logs if: always() run: | . ${GITHUB_WORKSPACE}/ci-environment.sh cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/{{ matrix.os.base_image_user }}/tools/ci_event_checks.sh + tools/run_remote ${primary} sudo /home/${{ matrix.os.base_image_user }}/tools/ci_event_checks.sh # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup # cost. That's not really something we can control, so just ignore the CPU @@ -243,7 +247,7 @@ jobs: run: | . ${GITHUB_WORKSPACE}/ci-environment.sh ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' + ${{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - name: Check for reasonable data rates timeout-minutes: 5 @@ -251,14 +255,14 @@ jobs: run: | . ${GITHUB_WORKSPACE}/ci-environment.sh cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/{{ matrix.os.base_image_user }}/tools/ci_event_checks.sh + tools/run_remote ${primary} sudo /home/${{ matrix.os.base_image_user }}/tools/ci_event_checks.sh - name: Fetch and tweak inventory if: always() run: | . ${GITHUB_WORKSPACE}/ci-environment.sh scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{ matrix.os.base_image_user }}@${primary}:/etc/sf/inventory.yaml /srv/github/ + ${{ matrix.os.base_image_user }}@${primary}:/etc/sf/inventory.yaml /srv/github/ sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml echo "=====" @@ -275,7 +279,7 @@ jobs: cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user={{ matrix.os.base_image_user }} \ + --extra-vars "base_image_user=${{ matrix.os.base_image_user }} \ ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml From 55ecfd6f4e8fb86b7b2926f6dbb252578d334907 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 09:21:19 +1000 Subject: [PATCH 07/14] Typo fix. --- .github/workflows/functional-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index fe5e16c8e..b191c7d77 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -114,7 +114,7 @@ jobs: - name: Log matrix details run: | echo "base_image: ${{ matrix.os.base_image }}" - echo "base_image_user: ${{ matrixo.s.base_image_user }}" + echo "base_image_user: ${{ matrix.os.base_image_user }}" echo "topology: ${{ matrix.os.topology }}" echo "concurrency: ${{ matrix.os.concurrency }}" From 28038323b51274d022a7d375dc36039742233561 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 09:33:02 +1000 Subject: [PATCH 08/14] Job groups must be unique. --- .github/workflows/functional-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index b191c7d77..fd503d1fb 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -38,7 +38,7 @@ jobs: needs: lint runs-on: self-hosted concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }} + group: ${{ github.workflow }}-${{ github.ref }}-requirements cancel-in-progress: true steps: @@ -295,7 +295,7 @@ jobs: needs: requirements-check runs-on: self-hosted concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-ansiblemodules cancel-in-progress: true steps: @@ -496,7 +496,7 @@ jobs: needs: requirements-check runs-on: self-hosted concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-nodelifecycle cancel-in-progress: true steps: @@ -651,7 +651,7 @@ jobs: needs: requirements-check runs-on: self-hosted concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-upgrade-0-6-15 cancel-in-progress: true steps: From 58c32f809b51dc12ba3769e2d1903679cf3bb3e5 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 11:38:07 +1000 Subject: [PATCH 09/14] Artifact names must be unique, fix concurrency variable. --- .github/workflows/functional-tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index fd503d1fb..fbaf622c5 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -113,6 +113,7 @@ jobs: steps: - name: Log matrix details run: | + echo "job_name: ${{ matrix.os.job_name }}"" echo "base_image: ${{ matrix.os.base_image }}" echo "base_image_user: ${{ matrix.os.base_image_user }}" echo "topology: ${{ matrix.os.topology }}" @@ -230,7 +231,7 @@ jobs: $source_path/shakenfist \ ${{ matrix.os.base_image_user }}@${primary}:shakenfist ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ${{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{ matix.os.concurrency}}; stestr slowest" + ${{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=${{ matrix.os.concurrency}}; stestr slowest" - name: Check logs if: always() @@ -286,7 +287,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: bundle.zip + name: bundle-${{ matrix.os.job_name }}.zip retention-days: 90 if-no-files-found: error path: /srv/github/artifacts/bundle.zip @@ -486,7 +487,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: bundle.zip + name: bundle-ansible-modules.zip retention-days: 90 if-no-files-found: error path: /srv/github/artifacts/bundle.zip @@ -641,7 +642,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: bundle.zip + name: bundle-node-lifecycle.zip retention-days: 90 if-no-files-found: error path: /srv/github/artifacts/bundle.zip @@ -752,7 +753,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: bundle.zip + name: bundle-upgrade-0-6-15.zip retention-days: 90 if-no-files-found: error path: /srv/github/artifacts/bundle.zip \ No newline at end of file From b575e18ce8b6aaa3fc5e508a19d27a41777dc23e Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 12:46:47 +1000 Subject: [PATCH 10/14] Don't re-run CI on merge. --- .github/workflows/functional-tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index fbaf622c5..9d7bf2612 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -5,10 +5,6 @@ name: Functional tests # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables on: - push: - branches: - - develop - - v*-releases pull_request: branches: - develop From 4ae1ca98e8a068f0141397502078c636e6c68c8a Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 12:47:32 +1000 Subject: [PATCH 11/14] Don't run functional tests on docs only changes. --- .github/workflows/functional-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 9d7bf2612..d7ed3b15f 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -9,6 +9,8 @@ on: branches: - develop - v*-releases + paths-ignore: + - 'docs/**' jobs: lint: From 94a21db8da5bc358f3d7e3996afa99c9bb52fd4c Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 12:51:16 +1000 Subject: [PATCH 12/14] Add some simple tests for the docs. --- .github/workflows/docs-tests.yml | 40 ++++++++++++++++++++++++++++++++ tox.ini | 7 ++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/workflows/docs-tests.yml diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.yml new file mode 100644 index 000000000..a506d773c --- /dev/null +++ b/.github/workflows/docs-tests.yml @@ -0,0 +1,40 @@ +name: Documentation tests + +# NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} +# which is available as GITHUB_WORKSPACE. You can find other environment +# variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables + +on: + pull_request: + branches: + - develop + - v*-releases + paths: + - 'docs/**' + +jobs: + lint: + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-lint + cancel-in-progress: true + + steps: + - name: Checkout code with two commits + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Ensure the docs build + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + /usr/bin/tox -edocs + zip site.zip site + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: site.zip + retention-days: 90 + if-no-files-found: error + path: site.zip \ No newline at end of file diff --git a/tox.ini b/tox.ini index 73cbb320f..1a34527e9 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,13 @@ envdir = {toxworkdir}/shared commands = bash tools/flake8wrap.sh -HEAD +[testenv:docs] +description = + Ensure the docs build. +envdir = {toxworkdir}/shared +commands = + mkdocs build + [testenv:py38] description = Run python3.8 unit tests From 25c81dc7cc499536f85d47060057eca3447eb73c Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 18:46:47 +1000 Subject: [PATCH 13/14] Typo fix. --- .github/workflows/functional-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index d7ed3b15f..48c4a43c9 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -111,7 +111,7 @@ jobs: steps: - name: Log matrix details run: | - echo "job_name: ${{ matrix.os.job_name }}"" + echo "job_name: ${{ matrix.os.job_name }}" echo "base_image: ${{ matrix.os.base_image }}" echo "base_image_user: ${{ matrix.os.base_image_user }}" echo "topology: ${{ matrix.os.topology }}" From 682fbf80ec4f446cccc5d5b2a43d571d96a7858d Mon Sep 17 00:00:00 2001 From: Michael Still Date: Wed, 1 May 2024 19:00:55 +1000 Subject: [PATCH 14/14] Disable fail fast for functional jobs. --- .github/workflows/functional-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 48c4a43c9..8518185c7 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -72,6 +72,7 @@ jobs: functional_matrix: needs: requirements-check strategy: + fail-fast: false matrix: os: [ {