forked from shakenfist/shakenfist
-
Notifications
You must be signed in to change notification settings - Fork 0
240 lines (209 loc) · 10.1 KB
/
scheduled-tests-develop-ubuntu-2004-localhost.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
name: Retest develop-ubuntu-2004-localhost
on:
workflow_dispatch:
schedule:
- cron: '00 13 * * *' # utc
jobs:
develop-ubuntu-2004-localhost:
runs-on: self-hosted
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
# NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo}
# which is available as GITHUB_WORKSPACE. You can find other environment
# variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables
steps:
- name: Set environment variables
run: |
echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV
echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV
- name: Checkout shakenfist
uses: actions/checkout@v4
with:
path: shakenfist
fetch-depth: 0
ref: develop
- name: Determine if there is any dependency between the repositories
run: |
python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py
- name: Build infrastructure
run: |
cd ${GITHUB_WORKSPACE}/shakenfist
ansible-playbook -i /home/debian/ansible-hosts \
--extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \
base_image=sf://label/ci-images/ubuntu-2004 base_image_user=ubuntu" \
deploy/ansible/ci-topology-localhost.yml
- name: Copy CI tools to primary
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
cd ${GITHUB_WORKSPACE}/shakenfist
scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null -rp tools \
ubuntu@$primary:.
- name: Log github actions buffering status
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
cd ${GITHUB_WORKSPACE}/shakenfist
tools/run_remote ${primary} python3 tools/buffer.py
- name: Run getsf installer on primary
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
ubuntu@$primary /tmp/getsf-wrapper
echo ""
echo ""
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful'
- name: Wait for API to start answering
run: |
set +e
. ${GITHUB_WORKSPACE}/ci-environment.sh
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog'
count=0
while [ $count -lt 60 ]
do
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary '. /etc/sf/sfrc; sf-client instance list'
if [ $? == 0 ]; then
exit 0
fi
count=$(( $count + 1 ))
sleep 5
done
exit 1
- name: Import cached images (v0.6+)
if: true
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared'
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared'
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared'
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared'
# It turns out that scheduled tests for other branches still use the workflow from develop, so
# this version must handle all of those branches
- name: Import cached images (v0.5)
if: false
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04'
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04'
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11'
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros'
- name: Create a base level of activity in the cluster
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="'
echo ""
echo ""
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary \
'. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done'
echo ""
- name: Run functional tests
timeout-minutes: 120
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
$source_path/shakenfist \
ubuntu@$primary:shakenfist
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=3; stestr slowest"
- name: Check logs
if: always()
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
cd ${GITHUB_WORKSPACE}/shakenfist
tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh
# On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup
# cost. That's not really something we can control, so just ignore the CPU
# usage of that process instead.
- name: Check SF process CPU usage
if: always()
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner'
- name: Check for reasonable data rates
timeout-minutes: 5
if: always()
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
cd ${GITHUB_WORKSPACE}/shakenfist
tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh
- name: Fetch and tweak inventory
if: always()
run: |
. ${GITHUB_WORKSPACE}/ci-environment.sh
scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary:/etc/sf/inventory.yaml /srv/github/
sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml
echo "====="
cat /srv/github/inventory.yaml
- name: Gather logs
if: always()
run: |
set -x
. ${GITHUB_WORKSPACE}/ci-environment.sh
# We need the ssh key in the place ansible expects it to be, which isn't
# true on the CI worker node.
cp /srv/github/id_ci /home/debian/.ssh/id_rsa
cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub
scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
ubuntu@$primary:/etc/sf/inventory.yaml /srv/github/
ansible-playbook -i /srv/github/inventory.yaml \
--extra-vars "base_image_user=ubuntu ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \
${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml
- uses: actions/upload-artifact@v4
if: always()
with:
name: bundle.zip
retention-days: 90
if-no-files-found: error
path: /srv/github/artifacts/bundle.zip
- name: Snapshot localhost builds for later upgrade testing
if: success()
run: |
if [ $(echo "localhost" | grep -c "localhost") -gt 1 ]; then
base=$(echo "sf://label/ci-images/ubuntu-2004" | cut -f 5 -d "/")
sf-client instance snapshot primary \
--label_name ci-images/$base-$GITHUB_BASE_REF \
--delete-snapshot-after-label --flatten
fi
- uses: JasonEtco/create-an-issue@v2
if: failure()
id: create-issue
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SF_CI_NAME: develop-ubuntu-2004-localhost
SF_BRANCH: develop
SF_ACTION_RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
filename: shakenfist/.github/workflows/scheduled-tests-failure.md
update_existing: true
search_existing: open
- if: failure()
run: 'echo Created ${{ steps.create-issue.outputs.url }}'