diff --git a/packages/validator/src/services/indices.ts b/packages/validator/src/services/indices.ts index 0846a7a90418..3ae707610101 100644 --- a/packages/validator/src/services/indices.ts +++ b/packages/validator/src/services/indices.ts @@ -3,6 +3,13 @@ import {ILogger} from "@chainsafe/lodestar-utils"; import {toHexString} from "@chainsafe/ssz"; import {Api} from "@chainsafe/lodestar-api"; import {ValidatorStore} from "./validatorStore"; +import {batchItems} from "../util/batch"; + +/** + * URLs have a limitation on size, adding an unbounded num of pubkeys will break the request. + * For reasoning on the specific number see: https://github.com/ChainSafe/lodestar/pull/2730#issuecomment-866749083 + */ +const PUBKEYS_PER_REQUEST = 10; // To assist with readability type PubkeyHex = string; @@ -58,9 +65,21 @@ export class IndicesService { } // Query the remote BN to resolve a pubkey to a validator index. + // support up to 1000 pubkeys per poll const pubkeysHex = pubkeysToPoll.map((pubkey) => toHexString(pubkey)); - const validatorsState = await this.api.beacon.getStateValidators("head", {indices: pubkeysHex}); + const pubkeysHexBatches = batchItems(pubkeysHex, {batchSize: PUBKEYS_PER_REQUEST}); + + const newIndices: number[] = []; + for (const pubkeysHexBatch of pubkeysHexBatches) { + const validatorIndicesArr = await this.fetchValidatorIndices(pubkeysHexBatch); + newIndices.push(...validatorIndicesArr); + } + this.logger.info("Discovered new validators", {count: newIndices.length}); + return newIndices; + } + private async fetchValidatorIndices(pubkeysHex: string[]): Promise { + const validatorsState = await this.api.beacon.getStateValidators("head", {indices: pubkeysHex}); const newIndices = []; for (const validatorState of validatorsState.data) { const pubkeyHex = toHexString(validatorState.validator.pubkey); @@ -71,7 +90,6 @@ export class IndicesService { newIndices.push(validatorState.index); } } - return newIndices; } } diff --git a/packages/validator/src/util/batch.ts b/packages/validator/src/util/batch.ts new file mode 100644 index 000000000000..f0cfa9c7ee9d --- /dev/null +++ b/packages/validator/src/util/batch.ts @@ -0,0 +1,16 @@ +/** + * Divide pubkeys into batches, each batch contains at most 5 http requests, + * each request can work on at most 40 pubkeys. + */ +export function batchItems(items: T[], opts: {batchSize: number; maxBatches?: number}): T[][] { + const batches: T[][] = []; + const maxBatches = opts.maxBatches ?? Math.ceil(items.length / opts.batchSize); + + for (let i = 0; i < maxBatches; i++) { + const batch = items.slice(opts.batchSize * i, opts.batchSize * (i + 1)); + if (batch.length === 0) break; + batches.push(batch); + } + + return batches; +} diff --git a/packages/validator/test/unit/utils/batch.test.ts b/packages/validator/test/unit/utils/batch.test.ts new file mode 100644 index 000000000000..71e51dc6e82a --- /dev/null +++ b/packages/validator/test/unit/utils/batch.test.ts @@ -0,0 +1,42 @@ +import {expect} from "chai"; +import {batchItems} from "../../../src/util/batch"; + +describe("util / batch", function () { + const testCases: {items: string[]; expected: string[][]}[] = [ + {items: [], expected: []}, + {items: ["1"], expected: [["1"]]}, + {items: ["1", "2"], expected: [["1", "2"]]}, + {items: ["1", "2", "3"], expected: [["1", "2"], ["3"]]}, + { + items: ["1", "2", "3", "4"], + expected: [ + ["1", "2"], + ["3", "4"], + ], + }, + {items: ["1", "2", "3", "4", "5"], expected: [["1", "2"], ["3", "4"], ["5"]]}, + { + items: ["1", "2", "3", "4", "5", "6"], + expected: [ + ["1", "2"], + ["3", "4"], + ["5", "6"], + ], + }, + // Ignore item 7 since it's over the max + { + items: ["1", "2", "3", "4", "5", "6", "7"], + expected: [ + ["1", "2"], + ["3", "4"], + ["5", "6"], + ], + }, + ]; + + for (const {items: pubkeys, expected} of testCases) { + it(`Batch ${pubkeys.length} items`, () => { + expect(batchItems(pubkeys, {batchSize: 2, maxBatches: 3})).to.deep.equal(expected); + }); + } +});