feat(perf): add (provision, build, run) tooling

perf/Makefile

@@ -0,0 +1,17 @@
+GO_SUBDIRS := $(wildcard impl/go/*/.)
+RUST_SUBDIRS := $(wildcard impl/rust/*/.)
+ZIG_SUBDIRS := $(wildcard impl/zig/*/.)
+HTTPS_SUBDIRS := $(wildcard impl/https/.)
+
+all: $(GO_SUBDIRS) $(RUST_SUBDIRS) $(ZIG_SUBDIRS) $(HTTPS_SUBDIRS)
+$(GO_SUBDIRS):
+	$(MAKE) -C $@
+$(RUST_SUBDIRS):
+	$(MAKE) -C $@
+$(ZIG_SUBDIRS):
+	$(MAKE) -C $@
+$(HTTPS_SUBDIRS):
+	$(MAKE) -C $@
+
+
+.PHONY: $(GO_SUBDIRS) $(RUST_SUBDIRS) $(ZIG_SUBDIRS) $(HTTPS_SUBDIRS) all

perf/README.md

@@ -0,0 +1,18 @@
+# libp2p perf testing
+
+## Setup
+
+1. `cd terraform`
+2. Save your public SSH key as the file `./user.pub`.
+3. `terraform init`
+4. `terraform apply`
+
+## Execute
+
+1. `cd runner`
+2. `npm ci`
+3. `npm run start -- --client-public-ip $(terraform output -raw -state ../terraform/terraform.tfstate client_public_ip) --server-public-ip $(terraform output -raw -state ../terraform/terraform.tfstate server_public_ip)`
+
+## Add benchmark binary
+
+See `runner/src/versions.ts`.

perf/dashboard/README.md

@@ -0,0 +1,62 @@
+# Perf Dashboard [wip]
+
+This is an outline of what I'd like to see our perf dashboard look like.
+
+A lot of this is inspired by the [MsQuic
+dashboard](https://microsoft.github.io/msquic/). Please look at that first.
+
+For each combination of libp2p implementation, version, and transport we would
+have numbers that outline:
+1. Download/Upload throughput
+2. Request latency
+3. Requests per second (for some request/response protocol)
+4. Handshakes per second (useful to identify overhead in connection
+   initialization).
+5. Memory usage.
+
+The y axis on the graphs is the value for the above tests, the x axis is the
+specific version. Different lines represent different implementation+transports.
+
+The dashboards should be selectable and filterable.
+
+# Other transports (iperf/http)
+
+We have to be careful to compare apples to apples. A raw iperf number might be
+confusing here because no application will ever hit those numbers since they
+will at least want some encryption in their connection. I would suggest not
+having this or an HTTP comparison. Having HTTPS might be okay.
+
+# Example dashboard
+
+https://observablehq.com/@realmarcopolo/libp2p-perf
+
+The dashboard automatically pulls data from this repo to display it.
+
+It currently pulls example-data.json. The schema of this data is defined in
+`benchmarks.schema.json` and `benchmark-result-type.ts`.
+
+# Benchmark runner
+
+This is the thing that runs the benchmark binaries and produces the benchmark
+data (that matches the benchmarks.schema.json/benchmark-result-type.ts)
+
+# Benchmark binary
+
+This is per implementation. It's a binary that accepts the following flags:
+
+| Flag                 | description                                            |
+| -------------------- | ------------------------------------------------------ |
+| parallel_connections | number of parallel connections to use                  |
+| upload_bytes         | number of bytes to upload to server per connection     |
+| download_bytes       | number of bytes to download from server per connection |
+| n_times              | number of times to do the perf round trip              |
+| close_connection     | bool. Close the connection after a perf round trip     |
+
+The binary should report the following:
+
+* Stats on the length of the run:
+  * Total, Avg, Min, Max, p95
+* Round trips per second:
+  * Avg, Min, Max, p95
+
+Maybe more?  TODO...

perf/dashboard/benchmark-result-type.ts

@@ -0,0 +1,25 @@
+export type BenchmarkResults = {
+    benchmarks: Benchmark[],
+    // For referencing this schema in JSON
+    "$schema"?: string
+};
+
+export type Benchmark = {
+    name: string,
+    unit: "bits/s" | "s",
+    results: Result[],
+    comparisons: Comparison[],
+
+}
+
+export type Result = {
+    result: number,
+    implementation: string,
+    transportStack: string,
+    version: string
+};
+
+export type Comparison = {
+    name: string,
+    result: number,
+}

perf/dashboard/benchmarks.schema.json

@@ -0,0 +1,98 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/BenchmarkResults",
+  "definitions": {
+    "BenchmarkResults": {
+      "type": "object",
+      "properties": {
+        "benchmarks": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Benchmark"
+          }
+        },
+        "$schema": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "benchmarks"
+      ],
+      "additionalProperties": false
+    },
+    "Benchmark": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "unit": {
+          "type": "string",
+          "enum": [
+            "bits/s",
+            "s"
+          ]
+        },
+        "results": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Result"
+          }
+        },
+        "comparisons": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Comparison"
+          }
+        }
+      },
+      "required": [
+        "name",
+        "unit",
+        "results",
+        "comparisons"
+      ],
+      "additionalProperties": false
+    },
+    "Result": {
+      "type": "object",
+      "properties": {
+        "result": {
+          "type": "number"
+        },
+        "implementation": {
+          "type": "string"
+        },
+        "transportStack": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "result",
+        "implementation",
+        "transportStack",
+        "version"
+      ],
+      "additionalProperties": false
+    },
+    "Comparison": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "result": {
+          "type": "number"
+        }
+      },
+      "required": [
+        "name",
+        "result"
+      ],
+      "additionalProperties": false
+    }
+  }
+}

perf/dashboard/example-data.json

@@ -0,0 +1,161 @@
+{
+    "$schema": "./benchmarks.schema.json",
+    "benchmarks": [
+        {
+            "name": "Single Connection throughput – Upload",
+            "unit": "bits/s",
+            "comparisons": [
+                {
+                    "name": "http",
+                    "result": 1234
+                }
+            ],
+            "results": [
+                {
+                    "result": 1100,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.24.2"
+                },
+                {
+                    "result": 1000,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.25.1"
+                },
+                {
+                    "result": 8234,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.26.2"
+                },
+                {
+                    "result": 4000,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.50.0"
+                },
+                {
+                    "result": 8000,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.51.0"
+                },
+                {
+                    "result": 6000,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.52.0"
+                },
+                {
+                    "result": 9001,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.1"
+                },
+                {
+                    "result": 9002,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.2"
+                },
+                {
+                    "result": 201,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.41.0"
+                },
+                {
+                    "result": 302,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.42.0"
+                },
+                {
+                    "result": 501,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.43.0"
+                }
+            ]
+        },
+        {
+            "name": "Single Connection 1 byte round trip latency",
+            "unit": "s",
+            "comparisons": [
+                {
+                    "name": "http",
+                    "result": 1.234
+                }
+            ],
+            "results": [
+                {
+                    "result": 0.100,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.24.2"
+                },
+                {
+                    "result": 0.100,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.25.1"
+                },
+                {
+                    "result": 0.834,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.26.2"
+                },
+                {
+                    "result": 0.400,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.50.0"
+                },
+                {
+                    "result": 0.800,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.51.0"
+                },
+                {
+                    "result": 0.600,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.52.0"
+                },
+                {
+                    "result": 0.901,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.1"
+                },
+                {
+                    "result": 0.902,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.2"
+                },
+                {
+                    "result": 0.201,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.41.0"
+                },
+                {
+                    "result": 0.302,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.42.0"
+                },
+                {
+                    "result": 0.501,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.43.0"
+                }
+            ]
+        }
+    ]
+}

perf/dashboard/util/.gitignore

@@ -0,0 +1 @@
+node_modules

perf/dashboard/util/README.md

@@ -0,0 +1,3 @@
+# Util
+
+Helper to generate schemas for benchmark data.