diff --git a/.github/workflows/ansible-module-tests.yml b/.github/workflows/ansible-module-tests.yml deleted file mode 100644 index 12c206149..000000000 --- a/.github/workflows/ansible-module-tests.yml +++ /dev/null @@ -1,215 +0,0 @@ -name: Ansible module testing - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - ansible-modules: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Run ansible module tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash ansiblemoduletests.sh" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary sudo chmod ugo+r /var/log/syslog - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary:/var/log/syslog \ - ${GITHUB_WORKSPACE}/syslog - - failures=0 - - echo - etcd_conns=`grep -c "Building new etcd connection" ${GITHUB_WORKSPACE}/syslog || true` - echo "This CI run created $etcd_conns etcd connections." - if [ $etcd_conns -gt 5000 ]; then - echo "FAILURE: Too many etcd clients!" - failures=1 - fi - - echo - sigterms=`grep -c "Sent SIGTERM to " ${GITHUB_WORKSPACE}/syslog || true` - echo "This CI run sent $sigterms SIGTERM signals while shutting down." - if [ $sigterms -gt 50 ]; then - echo "FAILURE: Too many SIGTERMs sent!" - failures=1 - fi - - FORBIDDEN=("Traceback (most recent call last):" - "ERROR sf" - "ERROR gunicorn" - " died" - "Extra vxlan present" - "Fork support is only compatible with the epoll1 and poll polling strategies" - "not using configured address" - "Dumping thread traces" - "because it is leased to" - "not committing online upgrade" - "Received a GOAWAY with error code ENHANCE_YOUR_CALM" - "ConnectionFailedError" - "invalid JWT in Authorization header" - "Libvirt Error: XML error" - "Cleaning up leaked IPAM" - "Cleaning up leaked vxlan" - "Waiting to acquire lock" - 'apparmor="DENIED"' - "Ignoring malformed cache entry") - IFS="" - for forbid in ${FORBIDDEN[*]} - do - if [ `grep -c "$forbid" ${GITHUB_WORKSPACE}/syslog || true` -gt 0 ] - then - echo "FAILURE: Forbidden string found in logs: $forbid" - failures=1 - fi - done - - if [ $failures -gt 0 ]; then - echo "...failures detected." - exit 1 - fi - - - name: Check process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs' - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - chown -R debian.debian /home/debian/.ssh - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/configure-tests.py b/.github/workflows/configure-tests.py index 9012326a7..3b9e959f6 100644 --- a/.github/workflows/configure-tests.py +++ b/.github/workflows/configure-tests.py @@ -89,36 +89,6 @@ 'scheduled': False }, ], - 'functional-tests': [ - { - 'name': 'debian-11-localhost', - 'baseimage': 'sf://label/ci-images/debian-11', - 'baseuser': 'debian', - 'topology': 'localhost', - 'concurrency': 3 - }, - { - 'name': 'debian-12-slim-primary', - 'baseimage': 'sf://label/ci-images/debian-12', - 'baseuser': 'debian', - 'topology': 'slim-primary', - 'concurrency': 5 - }, - { - 'name': 'ubuntu-2004-localhost', - 'baseimage': 'sf://label/ci-images/ubuntu-2004', - 'baseuser': 'ubuntu', - 'topology': 'localhost', - 'concurrency': 3 - }, - { - 'name': 'ubuntu-2204-slim-primary', - 'baseimage': 'sf://label/ci-images/ubuntu-2204', - 'baseuser': 'ubuntu', - 'topology': 'slim-primary', - 'concurrency': 5 - }, - ], 'scheduled-tests': [ { 'name': 'develop-debian-11-localhost', diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.yml new file mode 100644 index 000000000..a506d773c --- /dev/null +++ b/.github/workflows/docs-tests.yml @@ -0,0 +1,40 @@ +name: Documentation tests + +# NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} +# which is available as GITHUB_WORKSPACE. You can find other environment +# variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables + +on: + pull_request: + branches: + - develop + - v*-releases + paths: + - 'docs/**' + +jobs: + lint: + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-lint + cancel-in-progress: true + + steps: + - name: Checkout code with two commits + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Ensure the docs build + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + /usr/bin/tox -edocs + zip site.zip site + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: site.zip + retention-days: 90 + if-no-files-found: error + path: site.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-debian-11-localhost.yml b/.github/workflows/functional-tests-debian-11-localhost.yml deleted file mode 100644 index 810b4f822..000000000 --- a/.github/workflows/functional-tests-debian-11-localhost.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test debian-11-localhost - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - debian-11-localhost: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-localhost.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=3; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-debian-11-slim-primary.yml b/.github/workflows/functional-tests-debian-11-slim-primary.yml deleted file mode 100644 index 5f1f93e7b..000000000 --- a/.github/workflows/functional-tests-debian-11-slim-primary.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: Test debian-11-slim-primary - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - debian-11-slim-primary: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=5; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-debian-12-slim-primary.yml b/.github/workflows/functional-tests-debian-12-slim-primary.yml deleted file mode 100644 index 3eade4657..000000000 --- a/.github/workflows/functional-tests-debian-12-slim-primary.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test debian-12-slim-primary - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - debian-12-slim-primary: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-12 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=5; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-ubuntu-2004-localhost.yml b/.github/workflows/functional-tests-ubuntu-2004-localhost.yml deleted file mode 100644 index be6536945..000000000 --- a/.github/workflows/functional-tests-ubuntu-2004-localhost.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test ubuntu-2004-localhost - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - ubuntu-2004-localhost: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/ubuntu-2004 base_image_user=ubuntu" \ - deploy/ansible/ci-topology-localhost.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - ubuntu@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - ubuntu@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=3; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=ubuntu ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests-ubuntu-2204-slim-primary.yml b/.github/workflows/functional-tests-ubuntu-2204-slim-primary.yml deleted file mode 100644 index fe582ea6c..000000000 --- a/.github/workflows/functional-tests-ubuntu-2204-slim-primary.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Test ubuntu-2204-slim-primary - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - ubuntu-2204-slim-primary: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/ubuntu-2204 base_image_user=ubuntu" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - ubuntu@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - ubuntu@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=5; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ubuntu@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=ubuntu ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/functional-tests.tmpl b/.github/workflows/functional-tests.tmpl deleted file mode 100644 index f110d3359..000000000 --- a/.github/workflows/functional-tests.tmpl +++ /dev/null @@ -1,195 +0,0 @@ -name: Test {{name}} - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - {{name}}: - runs-on: self-hosted - concurrency: - group: {% raw %}${{ github.workflow }}-${{ github.ref }}{% endraw %} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA={% raw %}${{{% endraw %} github.event.pull_request.head.sha {% raw %}}}{% endraw %}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo {% raw %}${{{% endraw %} github.repository {% raw %}}}{% endraw %} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image={{baseimage}} base_image_user={{baseuser}}" \ - deploy/ansible/ci-topology-{{topology}}.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - {{baseuser}}@$primary:. - - - name: Log github actions buffering status - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} python3 tools/buffer.py - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Create a base level of activity in the cluster - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary \ - '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' - echo "" - - - name: Run functional tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - {{baseuser}}@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency={{concurrency}}; stestr slowest" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup - # cost. That's not really something we can control, so just ignore the CPU - # usage of that process instead. - - name: Check SF process CPU usage - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' - - - name: Check for reasonable data rates - timeout-minutes: 5 - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - {{baseuser}}@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user={{baseuser}} ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml new file mode 100644 index 000000000..8518185c7 --- /dev/null +++ b/.github/workflows/functional-tests.yml @@ -0,0 +1,758 @@ +name: Functional tests + +# NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} +# which is available as GITHUB_WORKSPACE. You can find other environment +# variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables + +on: + pull_request: + branches: + - develop + - v*-releases + paths-ignore: + - 'docs/**' + +jobs: + lint: + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-lint + cancel-in-progress: true + + steps: + - name: Checkout code with two commits + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Lint with flake8 + run: | + /usr/bin/tox -eflake8 + + # NOTE(mikal): we need this one early because renovate creates a lot of PRs + # where the proposed combination of dependencies doesn't actually install and + # I don't want to waste a lot of CI resources on them. + requirements-check: + needs: lint + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-requirements + cancel-in-progress: true + + steps: + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Attempt to install requirements + run: | + cd ${GITHUB_WORKSPACE} + python3 -mvenv venv + venv/bin/pip3 install -r shakenfist/requirements.txt + + build38: + needs: lint + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build38 + cancel-in-progress: true + + steps: + - name: Checkout code with two commits + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Run python3 unit tests + run: | + /usr/bin/tox -epy3 + + functional_matrix: + needs: requirements-check + strategy: + fail-fast: false + matrix: + os: [ + { + job_name: 'debian-11-localhost', + base_image: 'sf://label/ci-images/debian-11', + base_image_user: 'debian', + topology: 'localhost', + concurrency: 3 + }, + { + job_name: 'debian-12-slim-primary', + base_image: 'sf://label/ci-images/debian-12', + base_image_user: 'debian', + topology: 'slim-primary', + concurrency: 5 + }, + { + job_name: 'ubuntu-2004-localhost', + base_image: 'sf://label/ci-images/ubuntu-2004', + base_image_user: 'ubuntu', + topology: 'localhost', + concurrency: 3 + }, + { + job_name: 'ubuntu-2204-slim-primary', + base_image: 'sf://label/ci-images/ubuntu-2204', + base_image_user: 'ubuntu', + topology: 'slim-primary', + concurrency: 5 + }, + ] + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os.job_name }} + cancel-in-progress: true + + steps: + - name: Log matrix details + run: | + echo "job_name: ${{ matrix.os.job_name }}" + echo "base_image: ${{ matrix.os.base_image }}" + echo "base_image_user: ${{ matrix.os.base_image_user }}" + echo "topology: ${{ matrix.os.topology }}" + echo "concurrency: ${{ matrix.os.concurrency }}" + + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image=${{ matrix.os.base_image }} base_image_user=${{ matrix.os.base_image_user }}" \ + deploy/ansible/ci-topology-${{ matrix.os.topology }}.yml + + - name: Copy CI tools to primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -rp tools \ + ${{ matrix.os.base_image_user }}@${primary}:. + + - name: Log github actions buffering status + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} python3 tools/buffer.py + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + + - name: Wait for API to start answering + run: | + set +e + + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' + + count=0 + while [ $count -lt 60 ] + do + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client instance list' + if [ $? == 0 ]; then + exit 0 + fi + + count=$(( $count + 1 )) + sleep 5 + done + + exit 1 + + - name: Import cached images + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' + + - name: Create a base level of activity in the cluster + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} \ + 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} \ + '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' + echo "" + + - name: Run functional tests + timeout-minutes: 120 + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + $source_path/shakenfist \ + ${{ matrix.os.base_image_user }}@${primary}:shakenfist + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=${{ matrix.os.concurrency}}; stestr slowest" + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/${{ matrix.os.base_image_user }}/tools/ci_event_checks.sh + + # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup + # cost. That's not really something we can control, so just ignore the CPU + # usage of that process instead. + - name: Check SF process CPU usage + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner' + + - name: Check for reasonable data rates + timeout-minutes: 5 + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/${{ matrix.os.base_image_user }}/tools/ci_event_checks.sh + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ matrix.os.base_image_user }}@${primary}:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user=${{ matrix.os.base_image_user }} \ + ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle-${{ matrix.os.job_name }}.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip + + ansible-modules: + needs: requirements-check + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-ansiblemodules + cancel-in-progress: true + + steps: + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ + deploy/ansible/ci-topology-slim-primary.yml + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + + - name: Wait for API to start answering + run: | + set +e + + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' + + count=0 + while [ $count -lt 60 ] + do + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary '. /etc/sf/sfrc; sf-client instance list' + if [ $? == 0 ]; then + exit 0 + fi + + count=$(( $count + 1 )) + sleep 5 + done + + exit 1 + + - name: Import cached images + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' + + - name: Run ansible module tests + timeout-minutes: 120 + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + $source_path/shakenfist \ + debian@$primary:shakenfist + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash ansiblemoduletests.sh" + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary sudo chmod ugo+r /var/log/syslog + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary:/var/log/syslog \ + ${GITHUB_WORKSPACE}/syslog + + failures=0 + + echo + etcd_conns=`grep -c "Building new etcd connection" ${GITHUB_WORKSPACE}/syslog || true` + echo "This CI run created $etcd_conns etcd connections." + if [ $etcd_conns -gt 5000 ]; then + echo "FAILURE: Too many etcd clients!" + failures=1 + fi + + echo + sigterms=`grep -c "Sent SIGTERM to " ${GITHUB_WORKSPACE}/syslog || true` + echo "This CI run sent $sigterms SIGTERM signals while shutting down." + if [ $sigterms -gt 50 ]; then + echo "FAILURE: Too many SIGTERMs sent!" + failures=1 + fi + + FORBIDDEN=("Traceback (most recent call last):" + "ERROR sf" + "ERROR gunicorn" + " died" + "Extra vxlan present" + "Fork support is only compatible with the epoll1 and poll polling strategies" + "not using configured address" + "Dumping thread traces" + "because it is leased to" + "not committing online upgrade" + "Received a GOAWAY with error code ENHANCE_YOUR_CALM" + "ConnectionFailedError" + "invalid JWT in Authorization header" + "Libvirt Error: XML error" + "Cleaning up leaked IPAM" + "Cleaning up leaked vxlan" + "Waiting to acquire lock" + 'apparmor="DENIED"' + "Ignoring malformed cache entry") + IFS="" + for forbid in ${FORBIDDEN[*]} + do + if [ `grep -c "$forbid" ${GITHUB_WORKSPACE}/syslog || true` -gt 0 ] + then + echo "FAILURE: Forbidden string found in logs: $forbid" + failures=1 + fi + done + + if [ $failures -gt 0 ]; then + echo "...failures detected." + exit 1 + fi + + - name: Check process CPU usage + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs' + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + chown -R debian.debian /home/debian/.ssh + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle-ansible-modules.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip + + + node-lifecycle: + needs: requirements-check + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-nodelifecycle + cancel-in-progress: true + + steps: + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ + deploy/ansible/ci-topology-slim-primary.yml + + - name: Copy CI tools to primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -rp tools \ + debian@$primary:. + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + + - name: Wait for API to start answering + run: | + set +e + + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' + + count=0 + while [ $count -lt 60 ] + do + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary '. /etc/sf/sfrc; sf-client instance list' + if [ $? == 0 ]; then + exit 0 + fi + + count=$(( $count + 1 )) + sleep 5 + done + + exit 1 + + - name: Import cached images + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' + + - name: Run node lifecycle tests + timeout-minutes: 120 + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + $source_path/shakenfist \ + debian@$primary:shakenfist + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash nodelifecycletests.sh" + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh + + - name: Restart Shaken Fist nodes so we can collect logs + if: always() + run: | + set -x + . ${GITHUB_WORKSPACE}/ci-environment.sh + for node in ${sf1_uuid} ${sf2_uuid} ${sf3_uuid} ${sf4_uuid} ${sf5_uuid}; do + sf-client instance reboot --hard ${node} + done + sleep 90 + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + chown -R debian.debian /home/debian/.ssh + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle-node-lifecycle.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip + + + upgrade-0-6-15: + needs: requirements-check + runs-on: self-hosted + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-upgrade-0-6-15 + cancel-in-progress: true + + steps: + - name: Set environment variables + run: | + echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV + echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV + + - name: Checkout shakenfist + uses: actions/checkout@v4 + with: + path: shakenfist + fetch-depth: 0 + + - name: Determine if there is any dependency between the repositories + run: | + python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py + + - name: Build infrastructure + run: | + cd ${GITHUB_WORKSPACE}/shakenfist + ansible-playbook -i /home/debian/ansible-hosts \ + --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ + base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ + deploy/ansible/ci-topology-localhost-upgrade.yml + + - name: Copy CI tools to primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -rp tools \ + debian@$primary:. + + - name: Run getsf to only install etcd, and then restore a backup + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper --tags bootstrap,etcd + + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/*; . /etc/sf/sfrc; /srv/shakenfist/venv/bin/sf-backup restore /srv/ci/backup-0.6.15-20230319' + + - name: Run getsf installer on primary + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary /tmp/getsf-wrapper + echo "" + echo "" + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary \ + 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' + ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + debian@$primary 'sudo chmod ugo+r /etc/sf/*' + + - name: Wait for five minutes + run: sleep 300 + + - name: Check logs + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + cd ${GITHUB_WORKSPACE}/shakenfist + tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh + + - name: Fetch and tweak inventory + if: always() + run: | + . ${GITHUB_WORKSPACE}/ci-environment.sh + scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + debian@$primary:/etc/sf/inventory.yaml /srv/github/ + sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml + + echo "=====" + cat /srv/github/inventory.yaml + + - name: Gather logs + if: always() + run: | + set -x + + # We need the ssh key in the place ansible expects it to be, which isn't + # true on the CI worker node. + cp /srv/github/id_ci /home/debian/.ssh/id_rsa + cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub + + ansible-playbook -i /srv/github/inventory.yaml \ + --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ + ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bundle-upgrade-0-6-15.zip + retention-days: 90 + if-no-files-found: error + path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/.github/workflows/nodelifecycle-tests.yml b/.github/workflows/nodelifecycle-tests.yml deleted file mode 100644 index 88f633546..000000000 --- a/.github/workflows/nodelifecycle-tests.yml +++ /dev/null @@ -1,169 +0,0 @@ -name: Node lifecycle testing - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - node-lifecycle: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-slim-primary.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - - - name: Wait for API to start answering - run: | - set +e - - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog' - - count=0 - while [ $count -lt 60 ] - do - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary '. /etc/sf/sfrc; sf-client instance list' - if [ $? == 0 ]; then - exit 0 - fi - - count=$(( $count + 1 )) - sleep 5 - done - - exit 1 - - - name: Import cached images - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' - - - name: Run node lifecycle tests - timeout-minutes: 120 - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - $source_path/shakenfist \ - debian@$primary:shakenfist - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash nodelifecycletests.sh" - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Restart Shaken Fist nodes so we can collect logs - if: always() - run: | - set -x - . ${GITHUB_WORKSPACE}/ci-environment.sh - for node in ${sf1_uuid} ${sf2_uuid} ${sf3_uuid} ${sf4_uuid} ${sf5_uuid}; do - sf-client instance reboot --hard ${node} - done - sleep 90 - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - chown -R debian.debian /home/debian/.ssh - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip diff --git a/.github/workflows/python-unit-tests.yml b/.github/workflows/python-unit-tests.yml deleted file mode 100644 index b08a17bc1..000000000 --- a/.github/workflows/python-unit-tests.yml +++ /dev/null @@ -1,47 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python application - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - lint: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-lint - cancel-in-progress: true - - steps: - - name: Checkout code with two commits - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Lint with flake8 - run: | - /usr/bin/tox -eflake8 - - build38: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-build38 - cancel-in-progress: true - - steps: - - name: Checkout code with two commits - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Run python3 unit tests - run: | - /usr/bin/tox -epy3 diff --git a/.github/workflows/upgrade-tests.yml b/.github/workflows/upgrade-tests.yml deleted file mode 100644 index 3f5264015..000000000 --- a/.github/workflows/upgrade-tests.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Upgrade testing - -on: - push: - branches: - - develop - - v*-releases - pull_request: - branches: - - develop - - v*-releases - -jobs: - upgrade-0-6-15: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} - # which is available as GITHUB_WORKSPACE. You can find other environment - # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables - - steps: - - name: Set environment variables - run: | - echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV - echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV - - - name: Checkout shakenfist - uses: actions/checkout@v4 - with: - path: shakenfist - fetch-depth: 0 - - - name: Determine if there is any dependency between the repositories - run: | - python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py - - - name: Build infrastructure - run: | - cd ${GITHUB_WORKSPACE}/shakenfist - ansible-playbook -i /home/debian/ansible-hosts \ - --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ - base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ - deploy/ansible/ci-topology-localhost-upgrade.yml - - - name: Copy CI tools to primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -rp tools \ - debian@$primary:. - - - name: Run getsf to only install etcd, and then restore a backup - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper --tags bootstrap,etcd - - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/*; . /etc/sf/sfrc; /srv/shakenfist/venv/bin/sf-backup restore /srv/ci/backup-0.6.15-20230319' - - - name: Run getsf installer on primary - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary /tmp/getsf-wrapper - echo "" - echo "" - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary \ - 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' - ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - debian@$primary 'sudo chmod ugo+r /etc/sf/*' - - - name: Wait for five minutes - run: sleep 300 - - - name: Check logs - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - cd ${GITHUB_WORKSPACE}/shakenfist - tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh - - - name: Fetch and tweak inventory - if: always() - run: | - . ${GITHUB_WORKSPACE}/ci-environment.sh - scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - debian@$primary:/etc/sf/inventory.yaml /srv/github/ - sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml - - echo "=====" - cat /srv/github/inventory.yaml - - - name: Gather logs - if: always() - run: | - set -x - - # We need the ssh key in the place ansible expects it to be, which isn't - # true on the CI worker node. - cp /srv/github/id_ci /home/debian/.ssh/id_rsa - cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub - - ansible-playbook -i /srv/github/inventory.yaml \ - --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \ - ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: bundle.zip - retention-days: 90 - if-no-files-found: error - path: /srv/github/artifacts/bundle.zip \ No newline at end of file diff --git a/tox.ini b/tox.ini index 73cbb320f..1a34527e9 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,13 @@ envdir = {toxworkdir}/shared commands = bash tools/flake8wrap.sh -HEAD +[testenv:docs] +description = + Ensure the docs build. +envdir = {toxworkdir}/shared +commands = + mkdocs build + [testenv:py38] description = Run python3.8 unit tests