Poll node in batch to get validator indices

packages/validator/src/services/indices.ts

@@ -58,9 +58,22 @@ export class IndicesService {
     }
 
     // Query the remote BN to resolve a pubkey to a validator index.
-    const pubkeysHex = pubkeysToPoll.map((pubkey) => toHexString(pubkey));
-    const validatorsState = await this.api.beacon.getStateValidators("head", {indices: pubkeysHex});
+    // support up to 1000 pubkeys per poll
+    const pubkeysHex = pubkeysToPoll.map((pubkey) => toHexString(pubkey)).slice(0, MAX_PUBKEYS_PER_POLL);
+    const batches = pubkeysToBatches(pubkeysHex);
 
+    const newIndicesArr = [];
+    for (const batch of batches) {
+      const validatorIndicesArr = await Promise.all(batch.map(this.getIndicesPerHttpRequest));
+      newIndicesArr.push(...validatorIndicesArr.flat());
+    }
+    const newIndices = newIndicesArr.flat();
+    this.logger.info("Discovered new validators", {count: newIndices.length});
+    return newIndices;
+  }
+
+  private getIndicesPerHttpRequest = async (pubkeysHex: string[]): Promise<ValidatorIndex[]> => {
+    const validatorsState = await this.api.beacon.getStateValidators("head", {indices: pubkeysHex});
     const newIndices = [];
     for (const validatorState of validatorsState.data) {
       const pubkeyHex = toHexString(validatorState.validator.pubkey);
@@ -71,7 +84,42 @@ export class IndicesService {
         newIndices.push(validatorState.index);
       }
     }
-
     return newIndices;
+  };
+}
+
+type Batch = string[][];
+const PUBKEYS_PER_REQUEST = 10;
+const REQUESTS_PER_BATCH = 5;
+const MAX_PUBKEYS_PER_POLL = 5 * PUBKEYS_PER_REQUEST * REQUESTS_PER_BATCH;
+
+/**
+ * Divide pubkeys into batches, each batch contains at most 5 http requests,
+ * each request can work on at most 40 pubkeys.
+ * @param pubkeysHex
+ */
+export function pubkeysToBatches(
+  pubkeysHex: string[],
+  maxPubkeysPerRequest: number = PUBKEYS_PER_REQUEST,
+  maxRequesPerBatch: number = REQUESTS_PER_BATCH
+): Batch[] {
+  if (!pubkeysHex || pubkeysHex.length === 0) {
+    return [[[]]];
+  }
+  const batches: Batch[] = [];
+
+  const pubkeysPerBatch = maxPubkeysPerRequest * maxRequesPerBatch;
+  let batch: Batch = [];
+  let pubkeysPerRequest: string[];
+  for (let i = 0; i < pubkeysHex.length; i += maxPubkeysPerRequest) {
+    if (i % pubkeysPerBatch === 0) {
+      batch = [];
+      batches.push(batch);
+    }
+    if (i % maxPubkeysPerRequest === 0) {
+      pubkeysPerRequest = pubkeysHex.slice(i, Math.min(i + maxPubkeysPerRequest, pubkeysHex.length));
+      batch.push(pubkeysPerRequest);
+    }
   }
+  return batches;
 }

packages/validator/test/unit/services/indices.test.ts

@@ -0,0 +1,50 @@
+import {expect} from "chai";
+import {pubkeysToBatches} from "../../../src/services/indices";
+
+describe("pubkeysToBatches", function () {
+  const testCases: {pubkeys: string[]; expected: string[][][]}[] = [
+    {pubkeys: [], expected: [[[]]]},
+    {pubkeys: ["1"], expected: [[["1"]]]},
+    {pubkeys: ["1", "2"], expected: [[["1", "2"]]]},
+    {pubkeys: ["1", "2", "3"], expected: [[["1", "2"], ["3"]]]},
+    {
+      pubkeys: ["1", "2", "3", "4"],
+      expected: [
+        [
+          ["1", "2"],
+          ["3", "4"],
+        ],
+      ],
+    },
+    {pubkeys: ["1", "2", "3", "4", "5"], expected: [[["1", "2"], ["3", "4"], ["5"]]]},
+    {
+      pubkeys: ["1", "2", "3", "4", "5", "6"],
+      expected: [
+        [
+          ["1", "2"],
+          ["3", "4"],
+          ["5", "6"],
+        ],
+      ],
+    },
+    // new batch
+    {
+      pubkeys: ["1", "2", "3", "4", "5", "6", "7"],
+      expected: [
+        [
+          ["1", "2"],
+          ["3", "4"],
+          ["5", "6"],
+        ],
+        [["7"]],
+      ],
+    },
+  ];
+  const maxPubkeysPerRequest = 2;
+  const maxRequestsPerBatch = 3;
+  for (const {pubkeys, expected} of testCases) {
+    it(`should work with ${pubkeys.length} pubkeys`, () => {
+      expect(pubkeysToBatches(pubkeys, maxPubkeysPerRequest, maxRequestsPerBatch)).to.be.deep.equals(expected);
+    });
+  }
+});