From b3abebf44b0479eadee182f780a7753e50f4dd5b Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 12 Mar 2024 16:14:50 +0700 Subject: [PATCH 1/3] feat: n-historical states feature flags --- packages/beacon-node/src/chain/chain.ts | 33 +- .../beacon-node/src/chain/forkChoice/index.ts | 13 +- packages/beacon-node/src/chain/options.ts | 11 + .../stateContextCheckpointsCache.ts | 5 +- .../beacon-node/src/chain/validation/block.ts | 4 +- .../stateCache/nHistoricalStates.test.ts | 434 ++++++++++++++++++ packages/beacon-node/test/mocks/forkchoice.ts | 108 +++++ .../stateContextCheckpointsCache.test.ts | 6 +- .../test/unit/chain/validation/block.test.ts | 8 +- .../src/options/beaconNodeOptions/chain.ts | 42 ++ .../unit/options/beaconNodeOptions.test.ts | 8 + 11 files changed, 653 insertions(+), 19 deletions(-) create mode 100644 packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts create mode 100644 packages/beacon-node/test/mocks/forkchoice.ts diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 08743165cd05..fb8419f8f42a 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -41,6 +41,7 @@ import {IExecutionEngine, IExecutionBuilder} from "../execution/index.js"; import {Clock, ClockEvent, IClock} from "../util/clock.js"; import {ensureDir, writeIfNotExist} from "../util/file.js"; import {isOptimisticBlock} from "../util/forkChoice.js"; +import {BufferPool} from "../util/bufferPool.js"; import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js"; import {ChainEventEmitter, ChainEvent} from "./emitter.js"; import {IBeaconChain, ProposerPreparationData, BlockHash, StateGetOpts, CommonBlockBody} from "./interface.js"; @@ -80,7 +81,11 @@ import {BlockRewards, computeBlockRewards} from "./rewards/blockRewards.js"; import {ShufflingCache} from "./shufflingCache.js"; import {StateContextCache} from "./stateCache/stateContextCache.js"; import {SeenGossipBlockInput} from "./seenCache/index.js"; -import {CheckpointStateCache} from "./stateCache/stateContextCheckpointsCache.js"; +import {InMemoryCheckpointStateCache} from "./stateCache/stateContextCheckpointsCache.js"; +import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js"; +import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js"; +import {DbCPStateDatastore} from "./stateCache/datastore/db.js"; +import {FileCPStateDatastore} from "./stateCache/datastore/file.js"; import {SyncCommitteeRewards, computeSyncCommitteeRewards} from "./rewards/syncCommitteeRewards.js"; /** @@ -239,9 +244,28 @@ export class BeaconChain implements IBeaconChain { this.pubkey2index = cachedState.epochCtx.pubkey2index; this.index2pubkey = cachedState.epochCtx.index2pubkey; - const stateCache = new StateContextCache({metrics}); - const checkpointStateCache = new CheckpointStateCache({metrics}); - + const fileDataStore = opts.nHistoricalStatesFileDataStore ?? false; + const stateCache = this.opts.nHistoricalStates + ? new FIFOBlockStateCache(this.opts, {metrics}) + : new StateContextCache({metrics}); + const checkpointStateCache = this.opts.nHistoricalStates + ? new PersistentCheckpointStateCache( + { + metrics, + logger, + clock, + shufflingCache: this.shufflingCache, + getHeadState: this.getHeadState.bind(this), + bufferPool: new BufferPool(anchorState.type.tree_serializedSize(anchorState.node), metrics), + datastore: fileDataStore + ? // debug option if we want to investigate any issues with the DB + new FileCPStateDatastore() + : // production option + new DbCPStateDatastore(this.db), + }, + this.opts + ) + : new InMemoryCheckpointStateCache({metrics}); const {checkpoint} = computeAnchorCheckpoint(config, anchorState); stateCache.add(cachedState); stateCache.setHeadState(cachedState); @@ -335,6 +359,7 @@ export class BeaconChain implements IBeaconChain { /** Populate in-memory caches with persisted data. Call at least once on startup */ async loadFromDisk(): Promise { + await this.regen.init(); await this.opPool.fromPersisted(this.db); } diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index 7e195a84922d..b032159f2119 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -7,7 +7,7 @@ import { ForkChoiceStore, ExecutionStatus, JustifiedBalancesGetter, - ForkChoiceOpts, + ForkChoiceOpts as RealForkChoiceOpts, } from "@lodestar/fork-choice"; import { CachedBeaconStateAllForks, @@ -21,7 +21,10 @@ import {ChainEventEmitter} from "../emitter.js"; import {ChainEvent} from "../emitter.js"; import {GENESIS_SLOT} from "../../constants/index.js"; -export type {ForkChoiceOpts}; +export type ForkChoiceOpts = RealForkChoiceOpts & { + // for testing only + forkchoiceConstructor?: typeof ForkChoice; +}; /** * Fork Choice extended with a ChainEventEmitter @@ -47,7 +50,11 @@ export function initializeForkChoice( const justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(state); - return new ForkChoice( + // forkchoiceConstructor is only used for some test cases + // production code use ForkChoice constructor directly + const forkchoiceConstructor = opts.forkchoiceConstructor ?? ForkChoice; + + return new forkchoiceConstructor( config, new ForkChoiceStore( diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index e687099a0cb4..c0d32449b072 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -4,12 +4,17 @@ import {ArchiverOpts} from "./archiver/index.js"; import {ForkChoiceOpts} from "./forkChoice/index.js"; import {LightClientServerOpts} from "./lightClient/index.js"; import {ShufflingCacheOpts} from "./shufflingCache.js"; +import {DEFAULT_MAX_BLOCK_STATES, FIFOBlockStateCacheOpts} from "./stateCache/fifoBlockStateCache.js"; +import {PersistentCheckpointStateCacheOpts} from "./stateCache/persistentCheckpointsCache.js"; +import {DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY} from "./stateCache/persistentCheckpointsCache.js"; export type IChainOptions = BlockProcessOpts & PoolOpts & SeenCacheOpts & ForkChoiceOpts & ArchiverOpts & + FIFOBlockStateCacheOpts & + PersistentCheckpointStateCacheOpts & ShufflingCacheOpts & LightClientServerOpts & { blsVerifyAllMainThread?: boolean; @@ -31,6 +36,8 @@ export type IChainOptions = BlockProcessOpts & broadcastValidationStrictness?: string; minSameMessageSignatureSetsToBatch: number; archiveBlobEpochs?: number; + nHistoricalStates?: boolean; + nHistoricalStatesFileDataStore?: boolean; }; export type BlockProcessOpts = { @@ -103,4 +110,8 @@ export const defaultChainOptions: IChainOptions = { // batching too much may block the I/O thread so if useWorker=false, suggest this value to be 32 // since this batch attestation work is designed to work with useWorker=true, make this the lowest value minSameMessageSignatureSetsToBatch: 2, + nHistoricalStates: false, + nHistoricalStatesFileDataStore: false, + maxBlockStates: DEFAULT_MAX_BLOCK_STATES, + maxCPStateEpochsInMemory: DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY, }; diff --git a/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts index 873e7ac9e465..8f720f85487a 100644 --- a/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts @@ -6,7 +6,7 @@ import {routes} from "@lodestar/api"; import {Metrics} from "../../metrics/index.js"; import {StateCloneOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; -import {CheckpointStateCache as CheckpointStateCacheInterface, CacheItemType} from "./types.js"; +import {CheckpointStateCache, CacheItemType} from "./types.js"; export type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; const MAX_EPOCHS = 10; @@ -16,9 +16,8 @@ const MAX_EPOCHS = 10; * belonging to checkpoint * * Similar API to Repository - * TODO: rename to MemoryCheckpointStateCache in the next PR of n-historical states */ -export class CheckpointStateCache implements CheckpointStateCacheInterface { +export class InMemoryCheckpointStateCache implements CheckpointStateCache { private readonly cache: MapTracker; /** Epoch -> Set */ private readonly epochIndex = new MapDef>(() => new Set()); diff --git a/packages/beacon-node/src/chain/validation/block.ts b/packages/beacon-node/src/chain/validation/block.ts index daf99cf5365c..1d12110ad1c3 100644 --- a/packages/beacon-node/src/chain/validation/block.ts +++ b/packages/beacon-node/src/chain/validation/block.ts @@ -111,13 +111,13 @@ export async function validateGossipBlock( }); } - // getBlockSlotState also checks for whether the current finalized checkpoint is an ancestor of the block. + // use getPreState to reload state if needed. It also checks for whether the current finalized checkpoint is an ancestor of the block. // As a result, we throw an IGNORE (whereas the spec says we should REJECT for this scenario). // this is something we should change this in the future to make the code airtight to the spec. // [IGNORE] The block's parent (defined by block.parent_root) has been seen (via both gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is retrieved). // [REJECT] The block's parent (defined by block.parent_root) passes validation. const blockState = await chain.regen - .getBlockSlotState(parentRoot, blockSlot, {dontTransferCache: true}, RegenCaller.validateGossipBlock) + .getPreState(block, {dontTransferCache: true}, RegenCaller.validateGossipBlock) .catch(() => { throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot}); }); diff --git a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts new file mode 100644 index 000000000000..ac76d1e206f8 --- /dev/null +++ b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts @@ -0,0 +1,434 @@ +import {describe, it, afterEach, expect} from "vitest"; +import {Gauge, Histogram} from "prom-client"; +import {ChainConfig} from "@lodestar/config"; +import {Slot, phase0} from "@lodestar/types"; +import {TimestampFormatCode} from "@lodestar/logger"; +import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {routes} from "@lodestar/api"; +import {LogLevel, TestLoggerOpts, testLogger} from "../../../utils/logger.js"; +import {getDevBeaconNode} from "../../../utils/node/beacon.js"; +import {getAndInitDevValidators} from "../../../utils/node/validator.js"; +import {waitForEvent} from "../../../utils/events/resolver.js"; +import {ChainEvent, ReorgEventData} from "../../../../src/chain/emitter.js"; +import {connect, onPeerConnect} from "../../../utils/network.js"; +import {CacheItemType} from "../../../../src/chain/stateCache/types.js"; +import {ReorgedForkChoice} from "../../../mocks/forkchoice.js"; + +/** + * Test different reorg scenarios to make sure the StateCache implementations are correct. + * This includes several tests which make >6 min to pass in CI, so let's only run 1 of them and leave remaining ones + * for local investigation. + */ +describe( + "regen/reload states with n-historical states configuration", + function () { + const validatorCount = 8; + const testParams: Pick = { + // eslint-disable-next-line @typescript-eslint/naming-convention + SECONDS_PER_SLOT: 2, + }; + + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + + // all tests run until this slot + const LAST_SLOT = 33; + + /** + * (n+1) + * -----------------| + * / + * |---------|---------| + * ^ ^ + * (n+1-x) reorgedSlot n + * ^ + * commonAncestor + * |<--reorgDistance-->| + */ + const testCases: { + name: string; + reorgedSlot: number; + reorgDistance: number; + maxBlockStates: number; + maxCPStateEpochsInMemory: number; + reloadCount: number; + persistCount: number; + numStatesInMemory: number; + numStatesPersisted: number; + numEpochsInMemory: number; + numEpochsPersisted: number; + skip?: boolean; + }[] = [ + /** + * Block slot 28 has parent slot 25, block slot 26 and 27 are reorged + * --------------------|--- + * / ^ ^ ^ ^ + * / 28 29 32 33 + * |----------------|---------- + * ^ ^ ^ ^ + * 24 25 26 27 + * */ + { + name: "0 historical state, reorg in same epoch", + reorgedSlot: 27, + reorgDistance: 3, + maxBlockStates: 1, + maxCPStateEpochsInMemory: 0, + // reload at cp epoch 1 once to regen state 9 (12 - 3) + reloadCount: 1, + // persist for epoch 0 to 4, no need to persist cp epoch 3 again + persistCount: 5, + // run through slot 33, no state in memory + numStatesInMemory: 0, + // epoch 0 1 2 3 4 but finalized at epoch 2 so store checkpoint states for epoch 2 3 4 + numStatesPersisted: 3, + numEpochsInMemory: 0, + // epoch 0 1 2 3 4 but finalized at eopch 2 so store checkpoint states for epoch 2 3 4 + numEpochsPersisted: 3, + // chain is finalized at epoch 2 end of test + skip: true, + }, + /** + * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * --------------------------|--- + * / | ^ ^ ^ ^ + * / | 28 29 32 33 + * |----------------|---------- + * 16 ^ ^ ^ ^ ^ + * ^ 23 24 25 26 27 + * reload ^ + * 2 checkpoint states at epoch 3 are persisted + */ + { + name: "0 historical state, reorg 1 epoch", + reorgedSlot: 27, + reorgDistance: 5, + maxBlockStates: 1, + maxCPStateEpochsInMemory: 0, + // reload at cp epoch 2 once to regen state 23 (28 - 5) + reloadCount: 1, + // 1 cp state for epoch 0 1 2 4, and 2 cp states for epoch 3 (different roots) + persistCount: 6, + numStatesInMemory: 0, + // epoch 0 1 2 4 has 1 cp state, epoch 3 has 2 checkpoint states + numStatesPersisted: 6, + numEpochsInMemory: 0, + // epoch 0 1 2 3 4 + numEpochsPersisted: 5, + // chain is not finalized end of test + skip: true, + }, + /** + * Block slot 28 has parent slot 25, block slot 26 and 27 are reorged + * --------------------|--- + * / ^ ^ ^ ^ + * / 28 29 32 33 + * |----------------|---------- + * ^ ^ ^ ^ + * 24 25 26 27 + * */ + { + name: "maxCPStateEpochsInMemory=1, reorg in same epoch", + reorgedSlot: 27, + reorgDistance: 3, + maxBlockStates: 1, + maxCPStateEpochsInMemory: 1, + // no need to reload as cp state epoch 3 is available in memory + reloadCount: 0, + // 1 time for epoch 0 1 2 3, cp state epoch 4 is in memory + persistCount: 4, + // epoch 4, one for Current Root Checkpoint State and one for Previous Root Checkpoint State + numStatesInMemory: 2, + // epoch 2 3, epoch 4 is in-memory + numStatesPersisted: 2, + // epoch 3 + numEpochsInMemory: 1, + // epoch 2 3, epoch 4 is in-memory + numEpochsPersisted: 2, + // chain is finalized at epoch 2 end of test + skip: true, + }, + /** + * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * --------------------------|--- + * / | ^ ^ ^ ^ + * / | 28 29 32 33 + * |----------------|---------- + * 16 ^ ^ ^ ^ ^ + * 23 24 25 26 27 + * ^ + * PRCS at epoch 3 is persisted, CRCS is pruned + */ + { + name: "maxCPStateEpochsInMemory=1, reorg last slot of previous epoch", + reorgedSlot: 27, + reorgDistance: 5, + maxBlockStates: 1, + maxCPStateEpochsInMemory: 1, + // PRCS at epoch 3 is available in memory so no need to reload + reloadCount: 0, + // 1 cp state for epoch 0 1 2 3 + persistCount: 4, + // epoch 4, one for Current Root Checkpoint State and one for Previous Root Checkpoint State + numStatesInMemory: 2, + // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted + numStatesPersisted: 4, + // epoch 4 + numEpochsInMemory: 1, + // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted + numEpochsPersisted: 4, + // chain is NOT finalized end of test + skip: true, + }, + /** + * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * --------------------------------|--- + * / | ^ ^ ^ ^ + * / | 28 29 32 33 + * |----------------|---------- + * 16 ^ ^ ^ ^ ^ ^ + * 19 23 24 25 26 27 + * ^ + * PRCS at epoch 3 is persisted, CRCS is pruned + */ + { + name: "maxCPStateEpochsInMemory=1, reorg middle slot of previous epoch", + reorgedSlot: 27, + reorgDistance: 9, + maxBlockStates: 1, + maxCPStateEpochsInMemory: 1, + // reload CP state epoch 2 (slot = 16) + reloadCount: 1, + // 1 cp state for epoch 0 1 2 3 + persistCount: 4, + // epoch 4, one for Current Root Checkpoint State and one for Previous Root Checkpoint State + numStatesInMemory: 2, + // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted + numStatesPersisted: 4, + // epoch 4 + numEpochsInMemory: 1, + // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted + numEpochsPersisted: 4, + // chain is NOT finalized end of test + skip: true, + }, + /** + * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * --------------------------------------------|--- + * / | ^ ^ ^ ^ + * / | 28 29 32 33 + * |----------------|----------------|---------- + * ^ ^ 16 ^ ^ ^ ^ ^ ^ + * 8 15 19 23 24 25 26 27 + *reload ^ + * PRCS at epoch 3 is persisted, CRCS is pruned + */ + { + name: "maxCPStateEpochsInMemory=1, reorg 2 epochs", + reorgedSlot: 27, + reorgDistance: 13, + maxBlockStates: 1, + maxCPStateEpochsInMemory: 1, + // reload CP state epoch 2 (slot = 16) + reloadCount: 1, + // 1 cp state for epoch 0 1, 2 CP states for epoch 2, 1 cp state for epoch 3 + persistCount: 5, + // epoch 4, one for Current Root Checkpoint State and one for Previous Root Checkpoint State + numStatesInMemory: 2, + // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted, epoch 2 has 2 CP states + numStatesPersisted: 5, + // epoch 4 + numEpochsInMemory: 1, + // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted + numEpochsPersisted: 4, + // chain is NOT finalized end of test + }, + ]; + + for (const { + name, + reorgedSlot, + reorgDistance, + maxBlockStates, + maxCPStateEpochsInMemory, + reloadCount, + persistCount, + numStatesInMemory, + numStatesPersisted, + numEpochsInMemory, + numEpochsPersisted, + skip, + } of testCases) { + const wrappedIt = skip ? it.skip : it; + wrappedIt(`${name} reorgedSlot=${reorgedSlot} reorgDistance=${reorgDistance}`, async function () { + // the node needs time to transpile/initialize bls worker threads + const genesisSlotsDelay = 7; + const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; + const testLoggerOpts: TestLoggerOpts = { + level: LogLevel.debug, + timestampFormat: { + format: TimestampFormatCode.EpochSlot, + genesisTime, + slotsPerEpoch: SLOTS_PER_EPOCH, + secondsPerSlot: testParams.SECONDS_PER_SLOT, + }, + }; + + const loggerNodeA = testLogger("Reorg-Node-A", testLoggerOpts); + const loggerNodeB = testLogger("FollowUp-Node-B", {...testLoggerOpts, level: LogLevel.debug}); + + const reorgedBn = await getDevBeaconNode({ + params: testParams, + options: { + sync: {isSingleNode: true}, + network: {allowPublishToZeroPeers: true, mdns: true, useWorker: false}, + // run the first bn with ReorgedForkChoice, no nHistoricalStates flag so it does not have to reload + chain: { + blsVerifyAllMainThread: true, + forkchoiceConstructor: ReorgedForkChoice, + proposerBoostEnabled: true, + }, + }, + validatorCount, + genesisTime, + logger: loggerNodeA, + }); + + // stop bn after validators + afterEachCallbacks.push(() => reorgedBn.close()); + + const followupBn = await getDevBeaconNode({ + params: testParams, + options: { + api: {rest: {enabled: false}}, + network: {mdns: true, useWorker: false}, + // run the 2nd bn with nHistoricalStates flag and the configured maxBlockStates, maxCPStateEpochsInMemory + chain: { + blsVerifyAllMainThread: true, + forkchoiceConstructor: ReorgedForkChoice, + nHistoricalStates: true, + maxBlockStates, + maxCPStateEpochsInMemory, + proposerBoostEnabled: true, + }, + metrics: {enabled: true}, + }, + validatorCount, + genesisTime, + logger: loggerNodeB, + }); + + afterEachCallbacks.push(() => followupBn.close()); + + const connected = Promise.all([onPeerConnect(followupBn.network), onPeerConnect(reorgedBn.network)]); + await connect(followupBn.network, reorgedBn.network); + await connected; + loggerNodeB.info("Node B connected to Node A"); + + const {validators} = await getAndInitDevValidators({ + node: reorgedBn, + logPrefix: "Val-Node-A", + validatorsPerClient: validatorCount, + validatorClientCount: 1, + startIndex: 0, + useRestApi: false, + testLoggerOpts, + }); + + afterEachCallbacks.push(() => Promise.all(validators.map((v) => v.close()))); + + // wait for checkpoint 3 at slot 24, both nodes should reach same checkpoint + const checkpoints = await Promise.all( + [reorgedBn, followupBn].map((bn) => + waitForEvent(bn.chain.emitter, ChainEvent.checkpoint, 240000, (cp) => cp.epoch === 3) + ) + ); + expect(checkpoints[0]).toEqual(checkpoints[1]); + expect(checkpoints[0].epoch).toEqual(3); + const head = reorgedBn.chain.forkChoice.getHead(); + loggerNodeA.info("Node A emitted checkpoint event, head slot: " + head.slot); + + // setup reorg data for both bns + for (const bn of [reorgedBn, followupBn]) { + (bn.chain.forkChoice as ReorgedForkChoice).reorgedSlot = reorgedSlot; + (bn.chain.forkChoice as ReorgedForkChoice).reorgDistance = reorgDistance; + } + + // both nodes see the reorg event + const reorgDatas = await Promise.all( + [reorgedBn, followupBn].map((bn) => + waitForEvent( + bn.chain.emitter, + routes.events.EventType.chainReorg, + 240000, + (reorgData) => reorgData.slot === reorgedSlot + 1 + ) + ) + ); + for (const reorgData of reorgDatas) { + expect(reorgData.slot).toEqual(reorgedSlot + 1); + expect(reorgData.depth).toEqual(reorgDistance); + } + + // make sure both nodes can reach another checkpoint + const checkpoints2 = await Promise.all( + [reorgedBn, followupBn].map((bn) => + waitForEvent(bn.chain.emitter, ChainEvent.checkpoint, 240000, (cp) => cp.epoch === 4) + ) + ); + expect(checkpoints2[0]).toEqual(checkpoints2[1]); + expect(checkpoints2[0].epoch).toEqual(4); + + // wait for 1 more slot to persist states + await waitForEvent<{slot: Slot}>( + reorgedBn.chain.emitter, + routes.events.EventType.block, + 240000, + ({slot}) => slot === LAST_SLOT + ); + + const reloadMetricValues = await (followupBn.metrics?.cpStateCache.stateReloadDuration as Histogram).get(); + expect( + reloadMetricValues?.values.find( + (value) => value.metricName === "lodestar_cp_state_cache_state_reload_seconds_count" + )?.value + ).toEqual(reloadCount); + + const persistMetricValues = await (followupBn.metrics?.cpStateCache.statePersistDuration as Histogram).get(); + expect( + persistMetricValues?.values.find( + (value) => value.metricName === "lodestar_cp_state_cache_state_persist_seconds_count" + )?.value + ).toEqual(persistCount); + + // assert number of persisted/in-memory states + const stateSizeMetricValues = await (followupBn.metrics?.cpStateCache.size as unknown as Gauge).get(); + const numStateInMemoryItem = stateSizeMetricValues?.values.find( + (value) => value.labels.type === CacheItemType.inMemory + ); + const numStatePersistedItem = stateSizeMetricValues?.values.find( + (value) => value.labels.type === CacheItemType.persisted + ); + expect(numStateInMemoryItem?.value).toEqual(numStatesInMemory); + expect(numStatePersistedItem?.value).toEqual(numStatesPersisted); + + // assert number of epochs persisted/in-memory + const epochSizeMetricValues = await (followupBn.metrics?.cpStateCache.epochSize as unknown as Gauge).get(); + const numEpochsInMemoryItem = epochSizeMetricValues?.values.find( + (value) => value.labels.type === CacheItemType.inMemory + ); + const numEpochsPersistedItem = epochSizeMetricValues?.values.find( + (value) => value.labels.type === CacheItemType.persisted + ); + expect(numEpochsInMemoryItem?.value).toEqual(numEpochsInMemory); + expect(numEpochsPersistedItem?.value).toEqual(numEpochsPersisted); + }); + } + }, + {timeout: 96_000} +); diff --git a/packages/beacon-node/test/mocks/forkchoice.ts b/packages/beacon-node/test/mocks/forkchoice.ts new file mode 100644 index 000000000000..5ec594e4ed81 --- /dev/null +++ b/packages/beacon-node/test/mocks/forkchoice.ts @@ -0,0 +1,108 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {ForkChoice, ForkChoiceOpts, IForkChoiceStore, ProtoArray, ProtoBlock} from "@lodestar/fork-choice"; +import {Slot} from "@lodestar/types"; + +/** + * Specific implementation of ForkChoice that reorg at a given slot and distance. + * (n+1) + * -----------------| + * / + * |---------|---------| + * ^ ^ + * (n+1-x) reorgedSlot n + * ^ + * commonAncestor + * |<--reorgDistance-->| + **/ +export class ReorgedForkChoice extends ForkChoice { + /** + * These need to be in the constructor, however we want to keep the constructor signature the same. + * So they are set after construction in the test instead. + */ + reorgedSlot: Slot | undefined; + reorgDistance: number | undefined; + private readonly _fcStore: IForkChoiceStore; + // these flags to mark if the current call of getHead() is to produce a block + // the other way to check this is to check the n-th call of getHead() in the same slot, but this is easier + private calledUpdateHead = false; + private calledUpdateTime = false; + + constructor( + config: ChainForkConfig, + fcStore: IForkChoiceStore, + /** The underlying representation of the block DAG. */ + protoArray: ProtoArray, + opts?: ForkChoiceOpts + ) { + super(config, fcStore, protoArray, opts); + this._fcStore = fcStore; + } + + /** + * Override the getHead() method + * - produceBlock: to reorg at a given slot and distance. + * - produceAttestation: to build on the latest node after the reorged slot + * - importBlock: to return the old branch at the reorged slot to produce the reorg event + */ + getHead = (): ProtoBlock => { + const currentSlot = this._fcStore.currentSlot; + const producingBlock = this.calledUpdateHead && this.calledUpdateTime; + if (this.reorgedSlot === undefined || this.reorgDistance === undefined) { + return super.getHead(); + } + + this.calledUpdateTime = false; + this.calledUpdateHead = false; + + // produceBlock: at reorgedSlot + 1, build new branch + if (currentSlot === this.reorgedSlot + 1 && producingBlock) { + const nodes = super.getAllNodes(); + const headSlot = currentSlot - this.reorgDistance; + const headNode = nodes.find((node) => node.slot === headSlot); + if (headNode !== undefined) { + return headNode; + } + } + + // this is mainly for producing attestations + produceBlock for latter slots + if (currentSlot > this.reorgedSlot + 1) { + // from now on build on latest node which reorged at the given slot + const nodes = super.getAllNodes(); + return nodes[nodes.length - 1]; + } + + // importBlock flow at "this.reorgedSlot + 1" returns the old branch for oldHead computation which trigger reorg event + return super.getHead(); + }; + + updateTime(currentSlot: Slot): void { + // set flag to signal produceBlock flow + this.calledUpdateTime = true; + super.updateTime(currentSlot); + } + + /** + * Override this function to: + * - produceBlock flow: mark flags to indicate that the current call of getHead() is to produce a block + * - importBlock: return the new branch after the reorged slot, this is for newHead computation + */ + updateHead = (): ProtoBlock => { + if (this.reorgedSlot === undefined || this.reorgDistance === undefined) { + return super.updateHead(); + } + // in all produce blocks flow, it always call updateTime() first then recomputeForkChoiceHead() + if (this.calledUpdateTime) { + this.calledUpdateHead = true; + } + const currentSlot = this._fcStore.currentSlot; + if (currentSlot <= this.reorgedSlot) { + return super.updateHead(); + } + + // since reorgSlot, always return the latest node + const nodes = super.getAllNodes(); + const head = nodes[nodes.length - 1]; + super.updateHead(); + return head; + }; +} diff --git a/packages/beacon-node/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts b/packages/beacon-node/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts index cf0ab1fa16b2..1ebb21b324ea 100644 --- a/packages/beacon-node/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts +++ b/packages/beacon-node/test/perf/chain/stateCache/stateContextCheckpointsCache.test.ts @@ -2,17 +2,17 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {ssz, phase0} from "@lodestar/types"; import {generateCachedState} from "../../../utils/state.js"; -import {CheckpointStateCache, toCheckpointHex} from "../../../../src/chain/stateCache/index.js"; +import {InMemoryCheckpointStateCache, toCheckpointHex} from "../../../../src/chain/stateCache/index.js"; describe("CheckpointStateCache perf tests", function () { setBenchOpts({noThreshold: true}); let state: CachedBeaconStateAllForks; let checkpoint: phase0.Checkpoint; - let checkpointStateCache: CheckpointStateCache; + let checkpointStateCache: InMemoryCheckpointStateCache; before(() => { - checkpointStateCache = new CheckpointStateCache({}); + checkpointStateCache = new InMemoryCheckpointStateCache({}); state = generateCachedState(); checkpoint = ssz.phase0.Checkpoint.defaultValue(); }); diff --git a/packages/beacon-node/test/unit/chain/validation/block.test.ts b/packages/beacon-node/test/unit/chain/validation/block.test.ts index 65f8114ebe93..ad2f2e690ddd 100644 --- a/packages/beacon-node/test/unit/chain/validation/block.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/block.test.ts @@ -132,7 +132,7 @@ describe("gossip block validation", function () { // Returned parent block is latter than proposed block forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen not able to get the parent block state - regen.getBlockSlotState.mockRejectedValue(undefined); + regen.getPreState.mockRejectedValue(undefined); await expectRejectedWithLodestarError( validateGossipBlock(config, chain, job, ForkName.phase0), @@ -146,7 +146,7 @@ describe("gossip block validation", function () { // Returned parent block is latter than proposed block forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state - regen.getBlockSlotState.mockResolvedValue(generateCachedState()); + regen.getPreState.mockResolvedValue(generateCachedState()); // BLS signature verifier returns invalid verifySignature.mockResolvedValue(false); @@ -163,7 +163,7 @@ describe("gossip block validation", function () { forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state const state = generateCachedState(); - regen.getBlockSlotState.mockResolvedValue(state); + regen.getPreState.mockResolvedValue(state); // BLS signature verifier returns valid verifySignature.mockResolvedValue(true); // Force proposer shuffling cache to return wrong value @@ -182,7 +182,7 @@ describe("gossip block validation", function () { forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state const state = generateCachedState(); - regen.getBlockSlotState.mockResolvedValue(state); + regen.getPreState.mockResolvedValue(state); // BLS signature verifier returns valid verifySignature.mockResolvedValue(true); // Force proposer shuffling cache to return wrong value diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index 390ffb3ad2f6..0fee440a792b 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -27,6 +27,10 @@ export type ChainArgs = { "chain.minSameMessageSignatureSetsToBatch"?: number; "chain.maxShufflingCacheEpochs"?: number; "chain.archiveBlobEpochs"?: number; + "chain.nHistoricalStates"?: boolean; + "chain.nHistoricalStatesFileDataStore"?: boolean; + "chain.maxBlockStates"?: number; + "chain.maxCPStateEpochsInMemory"?: number; }; export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { @@ -55,6 +59,11 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { args["chain.minSameMessageSignatureSetsToBatch"] ?? defaultOptions.chain.minSameMessageSignatureSetsToBatch, maxShufflingCacheEpochs: args["chain.maxShufflingCacheEpochs"] ?? defaultOptions.chain.maxShufflingCacheEpochs, archiveBlobEpochs: args["chain.archiveBlobEpochs"], + nHistoricalStates: args["chain.nHistoricalStates"] ?? defaultOptions.chain.nHistoricalStates, + nHistoricalStatesFileDataStore: + args["chain.nHistoricalStatesFileDataStore"] ?? defaultOptions.chain.nHistoricalStatesFileDataStore, + maxBlockStates: args["chain.maxBlockStates"] ?? defaultOptions.chain.maxBlockStates, + maxCPStateEpochsInMemory: args["chain.maxCPStateEpochsInMemory"] ?? defaultOptions.chain.maxCPStateEpochsInMemory, }; } @@ -220,4 +229,37 @@ Will double processing times. Use only for debugging purposes.", type: "number", group: "chain", }, + + "chain.nHistoricalStates": { + hidden: true, + description: + "Use the new FIFOBlockStateCache and PersistentCheckpointStateCache or not which make lodestar heap size bounded instead of unbounded as before", + type: "boolean", + default: defaultOptions.chain.nHistoricalStates, + group: "chain", + }, + + "chain.nHistoricalStatesFileDataStore": { + hidden: true, + description: "Use fs to store checkpoint state for PersistentCheckpointStateCache or not", + type: "boolean", + default: defaultOptions.chain.nHistoricalStatesFileDataStore, + group: "chain", + }, + + "chain.maxBlockStates": { + hidden: true, + description: "Max block states to cache in memory, used for FIFOBlockStateCache", + type: "number", + default: defaultOptions.chain.maxBlockStates, + group: "chain", + }, + + "chain.maxCPStateEpochsInMemory": { + hidden: true, + description: "Max epochs to cache checkpoint states in memory, used for PersistentCheckpointStateCache", + type: "number", + default: defaultOptions.chain.maxCPStateEpochsInMemory, + group: "chain", + }, }; diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 1f36b3c9751c..c6ab8ec1291e 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -37,6 +37,10 @@ describe("options / beaconNodeOptions", () => { "chain.minSameMessageSignatureSetsToBatch": 32, "chain.maxShufflingCacheEpochs": 100, "chain.archiveBlobEpochs": 10000, + "chain.nHistoricalStates": true, + "chain.nHistoricalStatesFileDataStore": true, + "chain.maxBlockStates": 100, + "chain.maxCPStateEpochsInMemory": 100, emitPayloadAttributes: false, eth1: true, @@ -141,6 +145,10 @@ describe("options / beaconNodeOptions", () => { minSameMessageSignatureSetsToBatch: 32, maxShufflingCacheEpochs: 100, archiveBlobEpochs: 10000, + nHistoricalStates: true, + nHistoricalStatesFileDataStore: true, + maxBlockStates: 100, + maxCPStateEpochsInMemory: 100, }, eth1: { enabled: true, From d80f310eb4b3b3a9c18ff80a5eff23f4dd85d328 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 13 Mar 2024 10:53:37 +0700 Subject: [PATCH 2/3] fix: change vitest config for the using keyword --- packages/beacon-node/vitest.e2e.config.ts | 4 ++++ packages/beacon-node/vitest.spec.config.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/beacon-node/vitest.e2e.config.ts b/packages/beacon-node/vitest.e2e.config.ts index b9f913705ef2..18034e3eedd1 100644 --- a/packages/beacon-node/vitest.e2e.config.ts +++ b/packages/beacon-node/vitest.e2e.config.ts @@ -1,9 +1,13 @@ import {defineConfig, mergeConfig} from "vitest/config"; +import {buildTargetPlugin} from "../../scripts/vitest/plugins/buildTargetPlugin"; import vitestConfig from "../../vitest.base.e2e.config"; export default mergeConfig( vitestConfig, defineConfig({ + // We need to change the build target to test code which is based on `using` keyword + // Note this target is not fully supported for the browsers + plugins: [buildTargetPlugin("es2022")], test: { globalSetup: ["./test/globalSetup.ts"], }, diff --git a/packages/beacon-node/vitest.spec.config.ts b/packages/beacon-node/vitest.spec.config.ts index f4f301c7985f..5a34374d2374 100644 --- a/packages/beacon-node/vitest.spec.config.ts +++ b/packages/beacon-node/vitest.spec.config.ts @@ -1,9 +1,13 @@ import {defineConfig, mergeConfig} from "vitest/config"; +import {buildTargetPlugin} from "../../scripts/vitest/plugins/buildTargetPlugin"; import vitestConfig from "../../vitest.base.spec.config"; export default mergeConfig( vitestConfig, defineConfig({ + // We need to change the build target to test code which is based on `using` keyword + // Note this target is not fully supported for the browsers + plugins: [buildTargetPlugin("es2022")], test: { globalSetup: ["./test/globalSetup.ts"], }, From 590edef016a159f07d77dbf5e92519e9b5fd7102 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 20 Mar 2024 18:18:41 +0700 Subject: [PATCH 3/3] chore: address PR comments --- packages/beacon-node/src/chain/forkChoice/index.ts | 4 ++-- .../test/e2e/chain/stateCache/nHistoricalStates.test.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index b032159f2119..da50dd1f48ad 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -7,7 +7,7 @@ import { ForkChoiceStore, ExecutionStatus, JustifiedBalancesGetter, - ForkChoiceOpts as RealForkChoiceOpts, + ForkChoiceOpts as RawForkChoiceOpts, } from "@lodestar/fork-choice"; import { CachedBeaconStateAllForks, @@ -21,7 +21,7 @@ import {ChainEventEmitter} from "../emitter.js"; import {ChainEvent} from "../emitter.js"; import {GENESIS_SLOT} from "../../constants/index.js"; -export type ForkChoiceOpts = RealForkChoiceOpts & { +export type ForkChoiceOpts = RawForkChoiceOpts & { // for testing only forkchoiceConstructor?: typeof ForkChoice; }; diff --git a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts index ac76d1e206f8..6d35329ba272 100644 --- a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts +++ b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts @@ -94,7 +94,7 @@ describe( skip: true, }, /** - * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * Block slot 28 has parent slot 23, block slot 24 25 26 and 27 are reorged * --------------------------|--- * / | ^ ^ ^ ^ * / | 28 29 32 33 @@ -154,7 +154,7 @@ describe( skip: true, }, /** - * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * Block slot 28 has parent slot 23, block slot 24 25 26 and 27 are reorged * --------------------------|--- * / | ^ ^ ^ ^ * / | 28 29 32 33 @@ -186,7 +186,7 @@ describe( skip: true, }, /** - * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * Block slot 28 has parent slot 23, block slot 24 25 26 and 27 are reorged * --------------------------------|--- * / | ^ ^ ^ ^ * / | 28 29 32 33 @@ -218,7 +218,7 @@ describe( skip: true, }, /** - * Block slot 28 has parent slot 23, block slot 824 25 26 and 27 are reorged + * Block slot 28 has parent slot 23, block slot 24 25 26 and 27 are reorged * --------------------------------------------|--- * / | ^ ^ ^ ^ * / | 28 29 32 33