feat(perf): Allow for test filtering + Github composite action (#280)

* feat(perf): Allow for test filtering + Github composite action

* chore: updated workflow variables

* ci: ensure working directory is set + env variables for terraform

* chore: default the test filter to all implementations

.github/actions/run-perf-benchmark/action.yml

@@ -0,0 +1,120 @@
+name: "libp2p ping interop test"
+description: "Run the libp2p ping interoperability test suite"
+inputs:
+  test-filter:
+    description: "Filter which tests to run, only these implementations will be run"
+    required: false
+    default: "all"
+  s3-access-key-id:
+    description: "S3 Access key id for the terraform infrastructure"
+    required: true
+    default: ""
+  s3-secret-access-key:
+    description: "S3 secret key id for the terraform infrastructure"
+    required: true
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - id: ssh
+      shell: bash
+      name: Generate SSH key
+      working-directory: perf
+      run: |
+        make ssh-keygen
+        echo "key<<EOF" >> $GITHUB_OUTPUT
+        while read -r line; do
+          echo "::add-mask::$line"
+          echo "$line" >> $GITHUB_OUTPUT
+        done < terraform/modules/short_lived/files/perf
+        echo "EOF" >> $GITHUB_OUTPUT
+
+    - name: Configure SSH
+      uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0
+      with:
+        ssh-private-key: ${{ steps.ssh.outputs.key }}
+
+    - name: Configure git
+      shell: bash
+      run: |
+        git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com>"
+        git config --global user.name "${GITHUB_ACTOR}"
+
+    - name: Configure terraform
+      uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
+
+    - name: Init terraform
+      id: init
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
+        TF_IN_AUTOMATION: "1"
+        TF_INPUT: "0"
+      run: terraform init
+      working-directory: perf/terraform/configs/local
+
+    - name: Apply terraform
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
+        TF_IN_AUTOMATION: "1"
+        TF_INPUT: "0"
+      shell: bash
+      run: terraform apply -auto-approve
+      working-directory: perf/terraform/configs/local
+
+    - id: server
+      name: Retrieve server's IP
+      shell: bash
+      run: terraform output -raw server_ip
+      working-directory: perf/terraform/configs/local
+
+    - id: client
+      name: Retrieve client's IP
+      shell: bash
+      run: terraform output -raw client_ip
+      working-directory: perf/terraform/configs/local
+
+    - name: Download dependencies
+      shell: bash
+      run: npm ci
+      working-directory: perf/runner
+
+    - name: Run tests
+      shell: bash
+      env:
+        SERVER_IP: ${{ steps.server.outputs.stdout }}
+        CLIENT_IP: ${{ steps.client.outputs.stdout }}
+      run: npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP --test-filter ${{ inputs.test-filter }}
+      working-directory: perf/runner
+
+    - name: Push
+      shell: bash
+      if: github.event.inputs.push == 'true'
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        git add benchmark-results.json
+        git commit -m "perf: update benchmark results"
+        git push
+        gh pr comment --body "See new metrics at https://observablehq.com/@libp2p-workspace/performance-dashboard?branch=$(git rev-parse HEAD)" || true
+      working-directory: perf/runner
+
+    - name: Archive
+      if: github.event.intputs.push == 'false'
+      uses: actions/upload-artifact@v2
+      with:
+        name: benchmark-results
+        path: perf/runner/benchmark-results.json
+
+    - name: Destroy terraform
+      shell: bash
+      if: always() && steps.init.outputs.exitcode == 0
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
+        TF_IN_AUTOMATION: "1"
+        TF_INPUT: "0"
+      run: terraform destroy -auto-approve
+      working-directory: perf/terraform/configs/local

.github/workflows/perf.yml

@@ -14,9 +14,9 @@ on:
   workflow_dispatch:
     inputs:
       push:
-        description: 'Push the benchmark results to the repository'
+        description: "Push the benchmark results to the repository"
         required: false
-        default: 'true'
+        default: "true"
 
 jobs:
   perf:
@@ -27,78 +27,13 @@ jobs:
       run:
         shell: bash
         working-directory: perf
-    env:
-      AWS_ACCESS_KEY_ID: ${{ vars.PERF_AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.PERF_AWS_SECRET_ACCESS_KEY }}
-      TF_IN_AUTOMATION: 1
-      TF_INPUT: 0
     steps:
       - name: Checkout test-plans
         uses: actions/checkout@v3
         with:
           repository: ${{ github.repository }}
           ref: ${{ github.ref }}
-      - id: ssh
-        name: Generate SSH key
-        run: |
-          make ssh-keygen
-          echo "key<<EOF" >> $GITHUB_OUTPUT
-          while read -r line; do
-            echo "::add-mask::$line"
-            echo "$line" >> $GITHUB_OUTPUT
-          done < terraform/modules/short_lived/files/perf
-          echo "EOF" >> $GITHUB_OUTPUT
-      - name: Configure SSH
-        uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0
+      - uses: ./.github/actions/run-perf-benchmark
         with:
-          ssh-private-key: ${{ steps.ssh.outputs.key }}
-      - name: Configure git
-        run: |
-          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com>"
-          git config --global user.name "${GITHUB_ACTOR}"
-      - name: Configure terraform
-        uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
-      - name: Init terraform
-        id: init
-        run: terraform init
-        working-directory: perf/terraform/configs/local
-      - name: Apply terraform
-        run: terraform apply -auto-approve
-        working-directory: perf/terraform/configs/local
-      - id: server
-        name: Retrieve server's IP
-        run: terraform output -raw server_ip
-        working-directory: perf/terraform/configs/local
-      - id: client
-        name: Retrieve client's IP
-        run: terraform output -raw client_ip
-        working-directory: perf/terraform/configs/local
-      - name: Download dependencies
-        run: npm ci
-        working-directory: perf/runner
-      - name: Run tests
-        env:
-          SERVER_IP: ${{ steps.server.outputs.stdout }}
-          CLIENT_IP: ${{ steps.client.outputs.stdout }}
-        run: npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP
-        working-directory: perf/runner
-      - name: Push
-        if: github.event.inputs.push == 'true'
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          git add benchmark-results.json
-          git commit -m "perf: update benchmark results"
-          git push
-          gh pr comment --body "See new metrics at https://observablehq.com/@libp2p-workspace/performance-dashboard?branch=$(git rev-parse HEAD)" || true
-        working-directory: perf/runner
-      - name: Archive
-        if: github.event.intputs.push == 'false'
-        uses: actions/upload-artifact@v2
-        with:
-          name: benchmark-results
-          path: perf/runner/benchmark-results.json
-      - name: Destroy terraform
-        if: always() && steps.init.outputs.exitcode == 0
-        run: terraform destroy -auto-approve
-        working-directory: perf/terraform/configs/local
+          s3-access-key-id: ${{ vars.PERF_AWS_ACCESS_KEY_ID }}
+          s3-secret-access-key: ${{ secrets.PERF_AWS_SECRET_ACCESS_KEY }}

perf/impl/Makefile

@@ -5,17 +5,32 @@ QUIC_GO_SUBDIRS := $(wildcard quic-go/*/.)
 JS_SUBDIRS := $(wildcard js-libp2p/*/.)
 
 all: $(RUST_SUBDIRS) $(GO_SUBDIRS) $(HTTPS_SUBDIRS) $(QUIC_GO_SUBDIRS) $(JS_SUBDIRS)
+
 $(RUST_SUBDIRS):
 	$(MAKE) -C $@
+
 $(GO_SUBDIRS):
 	$(MAKE) -C $@
+
 $(HTTPS_SUBDIRS):
 	$(MAKE) -C $@
+
 $(QUIC_GO_SUBDIRS):
 	$(MAKE) -C $@
+
 $(JS_SUBDIRS):
 	$(MAKE) -C $@
 
+go-libp2p: $(GO_SUBDIRS)
+
+rust-libp2p: $(RUST_SUBDIRS)
+
+https: $(HTTPS_SUBDIRS)
+
+quic-go: $(QUIC_GO_SUBDIRS)
+
+js-libp2p: $(JS_SUBDIRS)
+
 clean: $(RUST_SUBDIRS:%=%clean) $(GO_SUBDIRS:%=%clean) $(HTTPS_SUBDIRS:%=%clean) $(QUIC_GO_SUBDIRS:%=%clean) $(JS_SUBDIRS:%=%clean)
 
 %clean:

perf/runner/src/index.ts

@@ -1,15 +1,23 @@
 import { execSync } from 'child_process';
-import { versions } from './versions';
+import { Version, versions } from './versions';
 import yargs from 'yargs';
 import fs from 'fs';
 import { BenchmarkResults, Benchmark, Result, IperfResults, PingResults, ResultValue } from './benchmark-result-type';
 
-async function main(clientPublicIP: string, serverPublicIP: string, testing: boolean) {
+async function main(clientPublicIP: string, serverPublicIP: string, testing: boolean, testFilter: string[]) {
+    const iterations = testing ? 1 : 10;
+
+    console.error(`= Starting benchmark with ${iterations} iterations on implementations ${testFilter}`);
+
     const pings = runPing(clientPublicIP, serverPublicIP, testing);
     const iperf = runIPerf(clientPublicIP, serverPublicIP, testing);
 
-    copyAndBuildPerfImplementations(serverPublicIP);
-    copyAndBuildPerfImplementations(clientPublicIP);
+    const versionsToRun = versions.filter(version => testFilter.includes('all') || testFilter.includes(version.implementation))
+
+    const implsToBuild = Array.from(new Set(versionsToRun.map(v => v.implementation))).join(' ');
+
+    copyAndBuildPerfImplementations(serverPublicIP, implsToBuild);
+    copyAndBuildPerfImplementations(clientPublicIP, implsToBuild);
 
     const benchmarks = [
         runBenchmarkAcrossVersions({
@@ -19,19 +27,19 @@ async function main(clientPublicIP: string, serverPublicIP: string, testing: boo
             uploadBytes: Number.MAX_SAFE_INTEGER,
             downloadBytes: 0,
             unit: "bit/s",
-            iterations: testing ? 1 : 10,
+            iterations,
             durationSecondsPerIteration: testing ? 5 : 20,
-        }),
+        }, versionsToRun),
         runBenchmarkAcrossVersions({
             name: "throughput/download",
             clientPublicIP,
             serverPublicIP,
             uploadBytes: 0,
             downloadBytes: Number.MAX_SAFE_INTEGER,
             unit: "bit/s",
-            iterations: testing ? 1 : 10,
+            iterations,
             durationSecondsPerIteration: testing ? 5 : 20,
-        }),
+        }, versionsToRun),
         runBenchmarkAcrossVersions({
             name: "Connection establishment + 1 byte round trip latencies",
             clientPublicIP,
@@ -41,7 +49,7 @@ async function main(clientPublicIP: string, serverPublicIP: string, testing: boo
             unit: "s",
             iterations: testing ? 1 : 100,
             durationSecondsPerIteration: Number.MAX_SAFE_INTEGER,
-        }),
+        }, versionsToRun),
     ];
 
     const benchmarkResults: BenchmarkResults = {
@@ -106,7 +114,7 @@ function runIPerf(clientPublicIP: string, serverPublicIP: string, testing: boole
         })
         .filter((bitrate): bitrate is number => bitrate !== null); // Remove any null values
 
-    return { unit: "bit/s", results:  bitrates}
+    return { unit: "bit/s", results: bitrates }
 }
 
 interface ArgsRunBenchmarkAcrossVersions {
@@ -120,12 +128,12 @@ interface ArgsRunBenchmarkAcrossVersions {
     durationSecondsPerIteration: number,
 }
 
-function runBenchmarkAcrossVersions(args: ArgsRunBenchmarkAcrossVersions): Benchmark {
-    console.error(`= Benchmark ${args.name}`)
+function runBenchmarkAcrossVersions(args: ArgsRunBenchmarkAcrossVersions, versionsToRun: Version[]): Benchmark {
+    console.error(`= Benchmark ${args.name} on versions ${versionsToRun.map(v => v.implementation).join(', ')}`)
 
     const results: Result[] = [];
 
-    for (const version of versions) {
+    for (const version of versionsToRun) {
         console.error(`== Version ${version.implementation}/${version.id}`)
 
         console.error(`=== Starting server ${version.implementation}/${version.id}`);
@@ -197,7 +205,7 @@ function runClient(args: ArgsRunBenchmark): ResultValue[] {
 
     const lines = stdout.toString().trim().split('\n');
 
-    const combined: ResultValue[]= [];
+    const combined: ResultValue[] = [];
 
     for (const line of lines) {
         const result = JSON.parse(line) as ResultValue;
@@ -220,13 +228,13 @@ function execCommand(cmd: string): string {
     }
 }
 
-function copyAndBuildPerfImplementations(ip: string) {
-    console.error(`= Building implementations on ${ip}`);
+function copyAndBuildPerfImplementations(ip: string, impls: string) {
+    console.error(`= Building implementations for ${impls} on ${ip}`);
 
     const stdout = execCommand(`rsync -avz --progress --filter=':- .gitignore' -e "ssh -o StrictHostKeyChecking=no" ../impl ec2-user@${ip}:/home/ec2-user`);
     console.error(stdout.toString());
 
-    const stdout2 = execCommand(`ssh -o StrictHostKeyChecking=no ec2-user@${ip} 'cd impl && make'`);
+    const stdout2 = execCommand(`ssh -o StrictHostKeyChecking=no ec2-user@${ip} 'cd impl && make ${impls}'`);
     console.error(stdout2.toString());
 }
 
@@ -247,9 +255,17 @@ const argv = yargs
             default: false,
             description: 'Run in testing mode',
             demandOption: false,
+        },
+        'test-filter': {
+            type: 'string',
+            array: true,
+            choices: ['js-libp2p', 'rust-libp2p', 'go-libp2p', 'https', 'quic-go', 'all'],
+            description: 'Filter tests to run, only the implementations here will be run. It defaults to all.',
+            demandOption: false,
+            default: 'all'
         }
     })
     .command('help', 'Print usage information', yargs.help)
     .parseSync();
 
-main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['testing'] as boolean);
+main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['testing'] as boolean, argv['test-filter'] as string[]);