Skip to content

Commit

Permalink
Autotune memory
Browse files Browse the repository at this point in the history
Public runners have ~10G of ram available.

XL runners have >50G of ram available.

It's nice to be able to run tests on public
runners.

Introduce an action that:
* (Linux) frees some memory
* (Linux) adds some swap
* (Linux and macOS) calculates available memory
* (Windows) suggests 10G of available memory
  • Loading branch information
jsoref committed Feb 9, 2024
1 parent 2efccb3 commit 6c5a957
Show file tree
Hide file tree
Showing 5 changed files with 115 additions and 4 deletions.
84 changes: 84 additions & 0 deletions .github/actions/reasonable-memory/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
name: Make more memory available
description: Stops services and adds swap

inputs:
max:
description: Maximum memory to suggest using
default: 50000

outputs:
available:
description: Available memory
value: ${{ steps.memory.outputs.available }}
suggested:
description: Memory suggested for use (constrained by max and available)
value: ${{ steps.memory.outputs.suggested }}

runs:
using: composite
steps:
- name: Stop services
if: env.RUNNER_OS == 'Linux'
shell: bash
run: |
sudo systemctl disable php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd multipathd.socket snapd.socket
sudo systemctl stop php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd
sudo killall mono
- name: enable swap
if: env.RUNNER_OS == 'Linux'
shell: bash
run: |
sudo fallocate -l 10G /mnt/big-swapfile
sudo chmod 600 /mnt/big-swapfile
sudo mkswap /mnt/big-swapfile
sudo swapon /mnt/big-swapfile
- name: Report Available Memory (Linux)
id: memory-linux
if: runner.os == 'Linux'
shell: bash
run: |
free |
perl -ne '
next unless /Mem:.*\s(\d+)$/;
my $m = int($1 / 102400)*100;
print "available=$m\n";
' >> "$GITHUB_OUTPUT"
- name: Report Available Memory (macOS)
id: memory-mac
if: runner.os == 'macOS'
shell: bash
run: |
sysctl -a |
perl -e '
my $m;
while (<>) {
next unless /.*memsize.*:\s*(\d+)/;
my $m0 = int($1 >> 20);
$m = $m0 unless ($m && $m > $m0);
}
print "available=$m\n";
' >> "$GITHUB_OUTPUT"
- name: Report Available Memory (Windows)
id: memory-windows
if: runner.os == 'Windows'
shell: bash
run: |
echo "available=10737418240" >> "$GITHUB_OUTPUT"
- name: Report Memory
id: memory
shell: bash
env:
max: ${{ inputs.max }}
available: ${{ steps.memory-linux.outputs.available || steps.memory-mac.outputs.available || steps.memory-windows.outputs.available }}
run: |
perl -e '
my $available = $ENV{available};
my $max = $ENV{max} || $available;
my $s = ($m < $max ? $available : $max);
print "suggested=$s\n"."available=$available\n";
' >> "$GITHUB_OUTPUT"
13 changes: 12 additions & 1 deletion .github/workflows/csharp-qltest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ on:
paths:
- "csharp/**"
- "shared/**"
- .github/workflows/csharp-qltest.yml
- .github/actions/fetch-codeql/action.yml
- .github/actions/cache-query-compilation/action.yml
- .github/actions/reasonable-memory/action.yml
- csharp/actions/create-extractor-pack/action.yml
- codeql-workspace.yml
branches:
- main
Expand All @@ -16,6 +20,9 @@ on:
- "shared/**"
- .github/workflows/csharp-qltest.yml
- .github/actions/fetch-codeql/action.yml
- .github/actions/cache-query-compilation/action.yml
- .github/actions/reasonable-memory/action.yml
- csharp/actions/create-extractor-pack/action.yml
- codeql-workspace.yml
branches:
- main
Expand Down Expand Up @@ -83,6 +90,9 @@ jobs:
security-events: write
steps:
- uses: actions/checkout@v4
- name: Allocate memory
id: memory
uses: ./.github/actions/reasonable-memory
- uses: ./csharp/actions/create-extractor-pack
- name: Cache compilation cache
id: query-cache
Expand All @@ -91,9 +101,10 @@ jobs:
key: csharp-qltest-${{ matrix.slice }}
- name: Run QL tests
run: |
codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
codeql test run --threads=0 --ram $memory --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
env:
GITHUB_TOKEN: ${{ github.token }}
memory: ${{ steps.memory.outputs.suggested }}
unit-tests:
strategy:
matrix:
Expand Down
15 changes: 13 additions & 2 deletions .github/workflows/ruby-qltest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ on:
paths:
- "ruby/**"
- "shared/**"
- .github/workflows/ruby-build.yml
- .github/workflows/ruby-qltest.yml
- .github/actions/fetch-codeql/action.yml
- .github/actions/cache-query-compilation/action.yml
- .github/actions/reasonable-memory/action.yml
- csharp/actions/create-extractor-pack/action.yml
- codeql-workspace.yml
branches:
- main
Expand All @@ -17,10 +20,14 @@ on:
- "shared/**"
- .github/workflows/ruby-qltest.yml
- .github/actions/fetch-codeql/action.yml
- .github/actions/cache-query-compilation/action.yml
- .github/actions/reasonable-memory/action.yml
- csharp/actions/create-extractor-pack/action.yml
- codeql-workspace.yml
branches:
- main
- "rc/*"
workflow_dispatch:

env:
CARGO_TERM_COLOR: always
Expand Down Expand Up @@ -80,6 +87,9 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: Allocate memory
id: memory
uses: ./.github/actions/reasonable-memory
- uses: ./.github/actions/fetch-codeql
- uses: ./ruby/actions/create-extractor-pack
- name: Cache compilation cache
Expand All @@ -89,6 +99,7 @@ jobs:
key: ruby-qltest
- name: Run QL tests
run: |
codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
codeql test run --threads=0 --ram $memory --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
env:
GITHUB_TOKEN: ${{ github.token }}
memory: ${{ steps.memory.outputs.suggested }}
1 change: 1 addition & 0 deletions .github/workflows/swift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ on:
- main
- rc/*
- codeql-cli-*
workflow_dispatch:

permissions:
contents: read
Expand Down
6 changes: 5 additions & 1 deletion swift/actions/run-ql-tests/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ inputs:
runs:
using: composite
steps:
- name: Allocate memory
id: memory
uses: ./.github/actions/reasonable-memory
- uses: ./swift/actions/share-extractor-pack
- uses: ./.github/actions/fetch-codeql
- id: query-cache
Expand All @@ -19,7 +22,7 @@ runs:
run: |
codeql test run \
--threads=0 \
--ram 50000 \
--ram $memory \
--search-path "${{ github.workspace }}/swift/extractor-pack" \
--check-databases \
--check-unused-labels \
Expand All @@ -32,3 +35,4 @@ runs:
swift/ql/test
env:
GITHUB_TOKEN: ${{ github.token }}
memory: ${{ steps.memory.outputs.suggested }}

0 comments on commit 6c5a957

Please sign in to comment.