From 61e6b667163ded20b86c58a540b6f1f3addb05ae Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Thu, 31 Aug 2023 10:30:34 +0700 Subject: [PATCH 1/4] chore: add benchmark for signature deserialization --- .../src/chain/bls/multithread/jobItem.ts | 7 ++++++- packages/beacon-node/test/perf/bls/bls.test.ts | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index 5e64ba334360..b339daab32b8 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -67,7 +67,12 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkR { publicKey: bls.PublicKey.aggregate(job.sets.map((set) => set.publicKey)).toBytes(format), signature: bls.Signature.aggregate( - // validate signature = true + // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true) + // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s + // to deserialize 750_000 signatures per epoch + // cpu profile on main thread has 250s idle so this only works until we reach 3M validators + // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators + // and not a problem in the near future job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)) ).toBytes(format), message: job.message, diff --git a/packages/beacon-node/test/perf/bls/bls.test.ts b/packages/beacon-node/test/perf/bls/bls.test.ts index bd6a26a3f692..a982cc55e499 100644 --- a/packages/beacon-node/test/perf/bls/bls.test.ts +++ b/packages/beacon-node/test/perf/bls/bls.test.ts @@ -77,6 +77,21 @@ describe("BLS ops", function () { }); } + // this is total time we deserialize all signatures of validators per epoch + // ideally we want to track 700_000, 1_400_000, 2_100_000 validators but it takes too long + for (const numValidators of [10_000, 100_000]) { + const signatures = linspace(0, numValidators - 1).map((i) => getSet(i % 256).signature); + itBench({ + id: `BLS deserializing ${numValidators} signatures`, + fn: () => { + for (const signature of signatures) { + // true = validate signature + bls.Signature.fromBytes(signature, CoordType.affine, true); + } + }, + }); + } + // An aggregate and proof object has 3 signatures. // We may want to bundle up to 32 sets in a single batch. // TODO: figure out why it does not work with 256 or more From 7d84c50af356fd53785e6117348c9dd9a8ad6c66 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Thu, 31 Aug 2023 10:36:43 +0700 Subject: [PATCH 2/4] chore: add reference to v1.11.0 profiles --- packages/beacon-node/src/chain/bls/multithread/jobItem.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index b339daab32b8..41b5a68454ae 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -73,6 +73,7 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkR // cpu profile on main thread has 250s idle so this only works until we reach 3M validators // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators // and not a problem in the near future + // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)) ).toBytes(format), message: job.message, From ec60202ed144a340ab2f250575bb543977b258d6 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Fri, 1 Sep 2023 15:00:10 +0700 Subject: [PATCH 3/4] feat: track time to deserialize and validate signature on main thread --- .../src/chain/bls/multithread/index.ts | 2 +- .../src/chain/bls/multithread/jobItem.ts | 30 +++++++++++-------- .../src/metrics/metrics/lodestar.ts | 4 +++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index 7a223a89ac3a..db8791e46e92 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -391,7 +391,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { try { // Note: This can throw, must be handled per-job. // Pubkey and signature aggregation is defered here - workReq = jobItemWorkReq(job, this.format); + workReq = jobItemWorkReq(job, this.format, this.metrics); } catch (e) { this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index 41b5a68454ae..cf26e4737c69 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -4,6 +4,7 @@ import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {VerifySignatureOpts} from "../interface.js"; import {getAggregatedPubkey} from "../utils.js"; import {LinkedList} from "../../../util/array.js"; +import {Metrics} from "../../../metrics/metrics.js"; import {BlsWorkReq} from "./types.js"; export type JobQueueItem = JobQueueItemDefault | JobQueueItemSameMessage; @@ -48,7 +49,7 @@ export function jobItemSigSets(job: JobQueueItem): number { * Prepare BlsWorkReq from JobQueueItem * WARNING: May throw with untrusted user input */ -export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkReq { +export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics: Metrics | null): BlsWorkReq { switch (job.type) { case JobQueueItemType.default: return { @@ -60,26 +61,31 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkR message: set.signingRoot, })), }; - case JobQueueItemType.sameMessage: + case JobQueueItemType.sameMessage: { + // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true) + // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s + // to deserialize 750_000 signatures per epoch + // cpu profile on main thread has 250s idle so this only works until we reach 3M validators + // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators + // and not a problem in the near future + // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 + const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); + const signatures = job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)); + if (timer) { + timer(); + } + return { opts: job.opts, sets: [ { publicKey: bls.PublicKey.aggregate(job.sets.map((set) => set.publicKey)).toBytes(format), - signature: bls.Signature.aggregate( - // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true) - // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s - // to deserialize 750_000 signatures per epoch - // cpu profile on main thread has 250s idle so this only works until we reach 3M validators - // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators - // and not a problem in the near future - // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 - job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)) - ).toBytes(format), + signature: bls.Signature.aggregate(signatures).toBytes(format), message: job.message, }, ], }; + } } } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 840d47dfbd06..8b8ce0f0c2bc 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -447,6 +447,10 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_batchable_sig_sets_total", help: "Count of total batchable signature sets", }), + signatureDeserializationMainThreadDuration: register.gauge({ + name: "lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds", + help: "Total time spent deserializing signatures on main thread", + }), }, // BLS time on single thread mode From 86a5ba81bf33045fbff2bd25b0d7dfc41292d283 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sun, 3 Sep 2023 15:21:57 +0700 Subject: [PATCH 4/4] chore: use timer?.() --- packages/beacon-node/src/chain/bls/multithread/jobItem.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index cf26e4737c69..4ae05cdab913 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -71,9 +71,7 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics: // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); const signatures = job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)); - if (timer) { - timer(); - } + timer?.(); return { opts: job.opts,