Cluster Run - Minimega #463
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cluster Run - Minimega | |
| on: | |
| # pull_request: | |
| # branches: | |
| # - '*' | |
| workflow_dispatch: | |
| inputs: | |
| azure_region: | |
| description: 'Azure region to deploy resources' | |
| required: true | |
| default: 'centralus' | |
| type: choice | |
| options: | |
| - centralus | |
| - eastus | |
| - eastus2 | |
| - westus | |
| - westus2 | |
| - westus3 | |
| - northcentralus | |
| - southcentralus | |
| - canadacentral | |
| - canadaeast | |
| - uksouth | |
| - ukwest | |
| - northeurope | |
| - westeurope | |
| jobs: | |
| build-and-test-cluster: | |
| runs-on: self-hosted | |
| env: | |
| UNIQUE_ID: ${{ github.run_id }}-${{ github.run_number }} | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| IP_ADDRESS: "" | |
| LS1_IP: "" | |
| elastic: "" | |
| AZURE_IP: "" | |
| LINUX_IP: "" | |
| WINDOWS_IP: "" | |
| ENROLLMENT_TOKEN: "" | |
| ES_PASSWORD: "" | |
| KIBANA_PASSWORD: "" | |
| ELASTIC_AGENT_VERSION: "8.15.3" | |
| MINIMEGA_PASSWORD: ${{ secrets.MINIMEGA_PASSWORD }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/[email protected] | |
| - name: Setup environment variables | |
| run: | | |
| PUBLIC_IP=$(curl -s https://api.ipify.org) | |
| echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV | |
| - name: Get branch name | |
| shell: bash | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV | |
| else | |
| echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV | |
| fi | |
| - name: Set the environment for docker-compose | |
| run: | | |
| cd testing/v2/development | |
| # Get the UID and GID of the current user | |
| echo "HOST_UID=$(id -u)" > .env | |
| echo "HOST_GID=$(id -g)" >> .env | |
| cat .env | |
| - name: Build pipeline container | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} build pipeline --no-cache | |
| - name: Start pipeline container | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline | |
| - name: Install Python requirements | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers/azure && \ | |
| pip install -r requirements.txt | |
| " | |
| - name: Build an Azure instance | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T \ | |
| -e AZURE_CLIENT_ID \ | |
| -e AZURE_CLIENT_SECRET \ | |
| -e AZURE_TENANT_ID \ | |
| -e AZURE_SUBSCRIPTION_ID \ | |
| pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| python3 ./azure/build_azure_linux_network.py \ | |
| -g pipe-${{ env.UNIQUE_ID }} \ | |
| -s ${{ env.IP_ADDRESS }}/32 \ | |
| -vs Standard_E8_v4 \ | |
| -l ${{ inputs.azure_region || 'centralus' }} \ | |
| -ast 23:00 \ | |
| -y | |
| " | |
| - name: Install minimega on Azure instance | |
| run: | | |
| cd testing/v2/development | |
| sleep 30 | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ./minimega/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" | |
| " | |
| - name: Install Linux in minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ./ubuntu_qcow_maker/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" | |
| " | |
| - name: Install windows minimega on Azure instance | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| sleep 30 | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T \ | |
| -e AZURE_CLIENT_ID \ | |
| -e AZURE_CLIENT_SECRET \ | |
| -e AZURE_TENANT_ID \ | |
| -e AZURE_SUBSCRIPTION_ID \ | |
| pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ./windows_qcow/pipeline_install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" | |
| " | |
| - name: Check if linux is running in minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| sleep 120 && \ | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ssh lme-user@\$IP_ADDRESS 'sudo /opt/minimega/bin/minimega -e vm info' | |
| " | |
| - name: Get Azure, Windows, and Linux IP addresses | |
| run: | | |
| cd testing/v2/development | |
| AZURE_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt") | |
| echo "AZURE_IP=$AZURE_IP" >> $GITHUB_ENV | |
| echo "Azure IP:$AZURE_IP" | |
| LINUX_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@$AZURE_IP 'sudo /opt/minimega/bin/minimega -e .json true vm info | jq -r --arg name \"ubuntu-runner\" \".[].Data[] | select(.Name == \\\$name) | .Networks[0].IP4\"' | |
| ") | |
| EXIT_CODE=$? | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "Failed to get Linux IP. Exit code: $EXIT_CODE" >&2 | |
| exit 1 | |
| fi | |
| if [ -z "$LINUX_IP" ]; then | |
| echo "Linux IP is empty" >&2 | |
| exit 1 | |
| fi | |
| echo "LINUX_IP=$LINUX_IP" >> $GITHUB_ENV | |
| echo "Linux IP:$LINUX_IP" | |
| WINDOWS_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@$AZURE_IP 'sudo /opt/minimega/bin/minimega -e .json true vm info | jq -r --arg name \"windows-runner\" \".[].Data[] | select(.Name == \\\$name) | .Networks[0].IP4\"' | |
| ") | |
| EXIT_CODE=$? | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "Failed to get Windows IP. Exit code: $EXIT_CODE" >&2 | |
| exit 1 | |
| fi | |
| if [ -z "$WINDOWS_IP" ]; then | |
| echo "Windows IP is empty" >&2 | |
| exit 1 | |
| fi | |
| echo "WINDOWS_IP=$WINDOWS_IP" >> $GITHUB_ENV | |
| echo "Windows IP:$WINDOWS_IP" | |
| echo "Azure_IP:$AZURE_IP LINUX_IP:$LINUX_IP WINDOWS_IP:$WINDOWS_IP" | |
| - name: Run a command in Linux Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} ls -la' | |
| " | |
| - name: Install LME on Azure instance | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ./install_v2/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" ${{ env.BRANCH_NAME }} | |
| " | |
| - name: Retrieve Elastic password | |
| env: | |
| AZURE_IP: ${{ env.AZURE_IP }} | |
| run: | | |
| cd testing/v2/development | |
| ES_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$elastic'" | tail -n 1 | tr -d '\n') | |
| echo "::add-mask::$ES_PASSWORD" | |
| echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV | |
| echo "Elastic password retrieved successfully: $ES_PASSWORD" | |
| KIBANA_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$kibana_system'" | tail -n 1 | tr -d '\n') | |
| echo "::add-mask::$KIBANA_PASSWORD" | |
| echo "KIBANA_PASSWORD=$KIBANA_PASSWORD" >> $GITHUB_ENV | |
| echo "Kibana password retrieved successfully: $KIBANA_PASSWORD" | |
| - name: Install test requirements on Azure instance | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ssh lme-user@\$IP_ADDRESS 'whoami && hostname && \ | |
| wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ | |
| sudo apt install -y ./google-chrome-stable_current_amd64.deb && \ | |
| cd /home/lme-user/LME/testing/tests && \ | |
| python3 -m venv venv && \ | |
| source venv/bin/activate && \ | |
| pip install -r requirements.txt ' | |
| " | |
| - name: Retrieve Elastic policy ID and enrollment token | |
| env: | |
| KIBANA_URL: "https://localhost:5601" | |
| ES_USERNAME: "elastic" | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| run: | | |
| cd testing/v2/development | |
| # Retrieve policy ID | |
| POLICY_ID=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} ' | |
| curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \ | |
| -H \"kbn-xsrf: true\" \ | |
| -H \"Content-Type: application/json\" | | |
| jq -r '.items[0].id' | |
| ' | |
| ") | |
| echo "Retrieved Policy ID: $POLICY_ID" | |
| # Retrieve enrollment token using the policy ID | |
| ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} ' | |
| curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \ | |
| -H \"kbn-xsrf: true\" \ | |
| -H \"Content-Type: application/json\" \ | |
| -d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" | | |
| jq -r .item.api_key | |
| ' | |
| ") | |
| echo "Retrieved enrollment token: $ENROLLMENT_TOKEN" | |
| # Mask the enrollment token in logs and set it as an environment variable | |
| echo "::add-mask::$ENROLLMENT_TOKEN" | |
| echo "ENROLLMENT_TOKEN=$ENROLLMENT_TOKEN" >> $GITHUB_ENV | |
| echo "Policy ID and Enrollment Token retrieved successfully" | |
| - name: Copy the Elastic Agent installer to Linux Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \ | |
| 'sudo scp -p -o StrictHostKeyChecking=no /home/lme-user/LME/testing/v2/installers/lib/install_agent_linux.sh vmuser@${{ env.LINUX_IP }}:~' | |
| " | |
| - name: Run a command in Linux Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} ls -la' | |
| " | |
| - name: Install the Elastic Agent in Linux Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} chmod +x ./install_agent_linux.sh ' && \ | |
| ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} ./install_agent_linux.sh --token ${{ env.ENROLLMENT_TOKEN }} --version ${{ env.ELASTIC_AGENT_VERSION }}' | |
| " | |
| - name: Install the Elastic Agent in Windows Minimega | |
| run: | | |
| set +H | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c ' | |
| ssh lme-user@${{ env.AZURE_IP }} "export SSHPASS='\''${{ env.MINIMEGA_PASSWORD }}'\'' && export WINDOWS_HOST=${{ env.WINDOWS_IP }} && cd /home/lme-user/LME/testing/v2/installers/lib/ && sudo -E ./install_agent_windows.sh --token ${{ env.ENROLLMENT_TOKEN }} --version ${{ env.ELASTIC_AGENT_VERSION }} --clientip ${{ env.WINDOWS_IP }}" | |
| ' | |
| - name: Check if the Linux Elastic agent is reporting | |
| env: | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| run: | | |
| sleep 360 | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \ | |
| 'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh' | |
| " | |
| - name: Check if the Windows Elastic agent is reporting | |
| env: | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| run: | | |
| sleep 360 | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \ | |
| 'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh windows' | |
| " | |
| - name: Run api tests on Azure instance | |
| env: | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \ | |
| echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \ | |
| echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \ | |
| echo elastic=\"$ES_PASSWORD\" >> .env && \ | |
| source venv/bin/activate && \ | |
| pytest -v api_tests/' | |
| " | |
| - name: Run selenium tests on Azure instance | |
| env: | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \ | |
| echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \ | |
| echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \ | |
| echo elastic=\"$ES_PASSWORD\" >> .env && \ | |
| source venv/bin/activate && \ | |
| pytest -v selenium_tests/' | |
| " | |
| - name: Cleanup Azure resources | |
| if: always() | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_SECRET --tenant $AZURE_TENANT | |
| az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait | |
| " | |
| - name: Stop and remove containers | |
| if: always() | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} down | |
| docker system prune -af |