feat(perf): add (provision, build, run) tooling

perf-dashboard/README.md

@@ -0,0 +1,62 @@
+# Perf Dashboard [wip]
+
+This is an outline of what I'd like to see our perf dashboard look like.
+
+A lot of this is inspired by the [MsQuic
+dashboard](https://microsoft.github.io/msquic/). Please look at that first.
+
+For each combination of libp2p implementation, version, and transport we would
+have numbers that outline:
+1. Download/Upload throughput
+2. Request latency
+3. Requests per second (for some request/response protocol)
+4. Handshakes per second (useful to identify overhead in connection
+   initialization).
+5. Memory usage.
+
+The y axis on the graphs is the value for the above tests, the x axis is the
+specific version. Different lines represent different implementation+transports.
+
+The dashboards should be selectable and filterable.
+
+# Other transports (iperf/http)
+
+We have to be careful to compare apples to apples. A raw iperf number might be
+confusing here because no application will ever hit those numbers since they
+will at least want some encryption in their connection. I would suggest not
+having this or an HTTP comparison. Having HTTPS might be okay.
+
+# Example dashboard
+
+https://observablehq.com/@realmarcopolo/libp2p-perf
+
+The dashboard automatically pulls data from this repo to display it.
+
+It currently pulls example-data.json. The schema of this data is defined in
+`benchmarks.schema.json` and `benchmark-result-type.ts`.
+
+# Benchmark runner
+
+This is the thing that runs the benchmark binaries and produces the benchmark
+data (that matches the benchmarks.schema.json/benchmark-result-type.ts)
+
+# Benchmark binary
+
+This is per implementation. It's a binary that accepts the following flags:
+
+| Flag                 | description                                            |
+| -------------------- | ------------------------------------------------------ |
+| parallel_connections | number of parallel connections to use                  |
+| upload_bytes         | number of bytes to upload to server per connection     |
+| download_bytes       | number of bytes to download from server per connection |
+| n_times              | number of times to do the perf round trip              |
+| close_connection     | bool. Close the connection after a perf round trip     |
+
+The binary should report the following:
+
+* Stats on the length of the run:
+  * Total, Avg, Min, Max, p95
+* Round trips per second:
+  * Avg, Min, Max, p95
+
+Maybe more?  TODO...

perf-dashboard/benchmark-result-type.ts

@@ -0,0 +1,25 @@
+export type BenchmarkResults = {
+    benchmarks: Benchmark[],
+    // For referencing this schema in JSON
+    "$schema"?: string
+};
+
+export type Benchmark = {
+    name: string,
+    unit: "bits/s" | "s",
+    results: Result[],
+    comparisons: Comparison[],
+
+}
+
+export type Result = {
+    result: number,
+    implementation: string,
+    transportStack: string,
+    version: string
+};
+
+export type Comparison = {
+    name: string,
+    result: number,
+}

perf-dashboard/benchmarks.schema.json

@@ -0,0 +1,98 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/BenchmarkResults",
+  "definitions": {
+    "BenchmarkResults": {
+      "type": "object",
+      "properties": {
+        "benchmarks": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Benchmark"
+          }
+        },
+        "$schema": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "benchmarks"
+      ],
+      "additionalProperties": false
+    },
+    "Benchmark": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "unit": {
+          "type": "string",
+          "enum": [
+            "bits/s",
+            "s"
+          ]
+        },
+        "results": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Result"
+          }
+        },
+        "comparisons": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Comparison"
+          }
+        }
+      },
+      "required": [
+        "name",
+        "unit",
+        "results",
+        "comparisons"
+      ],
+      "additionalProperties": false
+    },
+    "Result": {
+      "type": "object",
+      "properties": {
+        "result": {
+          "type": "number"
+        },
+        "implementation": {
+          "type": "string"
+        },
+        "transportStack": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "result",
+        "implementation",
+        "transportStack",
+        "version"
+      ],
+      "additionalProperties": false
+    },
+    "Comparison": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "result": {
+          "type": "number"
+        }
+      },
+      "required": [
+        "name",
+        "result"
+      ],
+      "additionalProperties": false
+    }
+  }
+}

perf-dashboard/example-data.json

@@ -0,0 +1,161 @@
+{
+    "$schema": "./benchmarks.schema.json",
+    "benchmarks": [
+        {
+            "name": "Single Connection throughput – Upload",
+            "unit": "bits/s",
+            "comparisons": [
+                {
+                    "name": "http",
+                    "result": 1234
+                }
+            ],
+            "results": [
+                {
+                    "result": 1100,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.24.2"
+                },
+                {
+                    "result": 1000,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.25.1"
+                },
+                {
+                    "result": 8234,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.26.2"
+                },
+                {
+                    "result": 4000,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.50.0"
+                },
+                {
+                    "result": 8000,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.51.0"
+                },
+                {
+                    "result": 6000,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.52.0"
+                },
+                {
+                    "result": 9001,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.1"
+                },
+                {
+                    "result": 9002,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.2"
+                },
+                {
+                    "result": 201,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.41.0"
+                },
+                {
+                    "result": 302,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.42.0"
+                },
+                {
+                    "result": 501,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.43.0"
+                }
+            ]
+        },
+        {
+            "name": "Single Connection 1 byte round trip latency",
+            "unit": "s",
+            "comparisons": [
+                {
+                    "name": "http",
+                    "result": 1.234
+                }
+            ],
+            "results": [
+                {
+                    "result": 0.100,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.24.2"
+                },
+                {
+                    "result": 0.100,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.25.1"
+                },
+                {
+                    "result": 0.834,
+                    "implementation": "go-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.26.2"
+                },
+                {
+                    "result": 0.400,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.50.0"
+                },
+                {
+                    "result": 0.800,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.51.0"
+                },
+                {
+                    "result": 0.600,
+                    "implementation": "rust-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.52.0"
+                },
+                {
+                    "result": 0.901,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.1"
+                },
+                {
+                    "result": 0.902,
+                    "implementation": "zig-libp2p",
+                    "transportStack": "quic-v1",
+                    "version": "v0.0.2"
+                },
+                {
+                    "result": 0.201,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.41.0"
+                },
+                {
+                    "result": 0.302,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.42.0"
+                },
+                {
+                    "result": 0.501,
+                    "implementation": "js-libp2p",
+                    "transportStack": "tcp+noise+yamux",
+                    "version": "v0.43.0"
+                }
+            ]
+        }
+    ]
+}

perf-dashboard/util/.gitignore

@@ -0,0 +1 @@
+node_modules

perf-dashboard/util/README.md

@@ -0,0 +1,3 @@
+# Util
+
+Helper to generate schemas for benchmark data.

perf-dashboard/util/main.ts

@@ -0,0 +1,18 @@
+import * as tsj from "ts-json-schema-generator";
+import * as fs from "fs";
+// const fs = require("fs");
+
+/** @type {import('ts-json-schema-generator/dist/src/Config').Config} */
+const config = {
+    path: "../benchmark-result-type.ts",
+    tsconfig: "./tsconfig.json",
+    type: "BenchmarkResults", // Or <type-name> if you want to generate schema for that one type only
+};
+
+const output_path = "../benchmarks.schema.json";
+
+const schema = tsj.createGenerator(config).createSchema(config.type);
+const schemaString = JSON.stringify(schema, null, 2);
+fs.writeFile(output_path, schemaString, (err) => {
+    if (err) throw err;
+});