optimistic sync p1 - attest only validated head

.github/workflows/test.yml

@@ -2,6 +2,9 @@ name: Tests
 
 on: [pull_request, push]
 
+env:
+  GOERLI_RPC_DEFAULT_URL: https://goerli.infura.io/v3/84842078b09946638c03157f83405213
+
 jobs:
   tests-main:
     name: Tests
@@ -53,6 +56,6 @@ jobs:
       - name: E2e tests
         run: yarn test:e2e
         env:
-          GOERLI_RPC_URL: ${{ secrets.GOERLI_RPC_URL }}
+          GOERLI_RPC_URL: ${{ secrets.GOERLI_RPC_URL!=0 && secrets.GOERLI_RPC_URL || env.GOERLI_RPC_DEFAULT_URL }}
       - name: README check
         run: yarn run check-readme

packages/fork-choice/src/forkChoice/forkChoice.ts

@@ -11,7 +11,7 @@ import {
 import {IChainConfig, IChainForkConfig} from "@chainsafe/lodestar-config";
 
 import {computeDeltas} from "../protoArray/computeDeltas";
-import {HEX_ZERO_HASH, IVoteTracker, IProtoBlock} from "../protoArray/interface";
+import {HEX_ZERO_HASH, IVoteTracker, IProtoBlock, ExecutionStatus} from "../protoArray/interface";
 import {ProtoArray} from "../protoArray/protoArray";
 
 import {IForkChoiceMetrics} from "../metrics";
@@ -296,9 +296,8 @@ export class ForkChoice implements IForkChoice {
     }
 
     if (
-      merge.isMergeStateType(state) &&
-      merge.isMergeBlockBodyType(block.body) &&
-      merge.isMergeBlock(state, block.body)
+      preCachedData?.isMergeBlock ||
+      (merge.isMergeStateType(state) && merge.isMergeBlockBodyType(block.body) && merge.isMergeBlock(state, block.body))
     )
       assertValidTerminalPowBlock(this.config, (block as unknown) as merge.BeaconBlock, preCachedData);
 
@@ -368,13 +367,20 @@ export class ForkChoice implements IForkChoice {
       parentRoot: parentRootHex,
       targetRoot: toHexString(targetRoot),
       stateRoot: toHexString(block.stateRoot),
-      executionPayloadBlockHash: merge.isMergeBlockBodyType(block.body)
-        ? toHexString(block.body.executionPayload.blockHash)
-        : null,
+
       justifiedEpoch: stateJustifiedEpoch,
       justifiedRoot: toHexString(state.currentJustifiedCheckpoint.root),
       finalizedEpoch: finalizedCheckpoint.epoch,
       finalizedRoot: toHexString(state.finalizedCheckpoint.root),
+
+      ...(merge.isMergeBlockBodyType(block.body) &&
+      merge.isMergeStateType(state) &&
+      merge.isExecutionEnabled(state, block.body)
+        ? {
+            executionPayloadBlockHash: toHexString(block.body.executionPayload.blockHash),
+            executionStatus: this.getPostMergeExecStatus(preCachedData),
+          }
+        : {executionPayloadBlockHash: null, executionStatus: this.getPreMergeExecStatus(preCachedData)}),
     });
   }
 
@@ -629,6 +635,35 @@ export class ForkChoice implements IForkChoice {
     return newNode.slot - commonAncestor.slot;
   }
 
+  /**
+   * Optimistic sync validate till validated latest hash, invalidate any decendant branch if invalidate till hash provided
+   * TODO: implementation:
+   * 1. verify is_merge_block if the mergeblock has not yet been validated
+   * 2. Throw critical error and exit if a block in finalized chain gets invalidated
+   */
+  validateLatestHash(_latestValidHash: RootHex, _invalidateTillHash: RootHex | null): void {
+    // Silently ignore for now if all calls were valid
+    return;
+  }
+
+  private getPreMergeExecStatus(preCachedData?: OnBlockPrecachedData): ExecutionStatus.PreMerge {
+    const executionStatus = preCachedData?.executionStatus || ExecutionStatus.PreMerge;
+    if (executionStatus !== ExecutionStatus.PreMerge)
+      throw Error(`Invalid pre-merge execution status: expected: ${ExecutionStatus.PreMerge}, got ${executionStatus}`);
+    return executionStatus;
+  }
+
+  private getPostMergeExecStatus(
+    preCachedData?: OnBlockPrecachedData
+  ): ExecutionStatus.Valid | ExecutionStatus.Syncing {
+    const executionStatus = preCachedData?.executionStatus || ExecutionStatus.Syncing;
+    if (executionStatus === ExecutionStatus.PreMerge)
+      throw Error(
+        `Invalid post-merge execution status: expected: ${ExecutionStatus.Syncing} or ${ExecutionStatus.Valid} , got ${executionStatus}`
+      );
+    return executionStatus;
+  }
+
   private updateJustified(justifiedCheckpoint: CheckpointWithHex, justifiedBalances: number[]): void {
     this.synced = false;
     this.justifiedBalances = justifiedBalances;
@@ -922,6 +957,10 @@ function assertValidTerminalPowBlock(
       );
   } else {
     // If no TERMINAL_BLOCK_HASH override, check ttd
+
+    // Delay powBlock checks if the payload execution status is unknown because of syncing response in executePayload call while verifying
+    if (preCachedData?.executionStatus === ExecutionStatus.Syncing) return;
+
     const {powBlock, powBlockParent} = preCachedData || {};
     if (!powBlock) throw Error("onBlock preCachedData must include powBlock");
     if (!powBlockParent) throw Error("onBlock preCachedData must include powBlock");

packages/fork-choice/src/forkChoice/interface.ts

@@ -1,5 +1,5 @@
 import {Epoch, Slot, ValidatorIndex, phase0, allForks, Root, RootHex} from "@chainsafe/lodestar-types";
-import {IProtoBlock} from "../protoArray/interface";
+import {IProtoBlock, ExecutionStatus} from "../protoArray/interface";
 import {CheckpointWithHex} from "./store";
 
 export type CheckpointHex = {
@@ -132,6 +132,10 @@ export interface IForkChoice {
   getBlockSummariesAtSlot(slot: Slot): IProtoBlock[];
   /** Returns the distance of common ancestor of nodes to newNode. Returns null if newNode is descendant of prevNode */
   getCommonAncestorDistance(prevBlock: IProtoBlock, newBlock: IProtoBlock): number | null;
+  /**
+   * Optimistic sync validate till validated latest hash, invalidate any decendant branch if invalidated branch decendant provided
+   */
+  validateLatestHash(latestValidHash: RootHex, invalidateTillHash: RootHex | null): void;
 }
 
 /** Same to the PowBlock but we want RootHex to work with forkchoice conveniently */
@@ -150,14 +154,19 @@ export type OnBlockPrecachedData = {
    * powBlock = getPowBlock((block as merge.BeaconBlock).body.executionPayload.parentHash)
    * ```
    */
-  powBlock?: PowBlockHex;
+  powBlock?: PowBlockHex | null;
   /**
    * POW chain block's block parent, from getPowBlock() `eth_getBlockByHash` JSON RPC endpoint
    * ```ts
    * const powParent = getPowBlock(powBlock.parentHash);
    * ```
    */
-  powBlockParent?: PowBlockHex;
+  powBlockParent?: PowBlockHex | null;
+  /**
+   * Optimistic sync fields
+   */
+  isMergeBlock?: boolean;
+  executionStatus?: ExecutionStatus;
 };
 
 export interface ILatestMessage {

packages/fork-choice/src/index.ts

@@ -1,5 +1,5 @@
 export {ProtoArray} from "./protoArray/protoArray";
-export {IProtoBlock, IProtoNode} from "./protoArray/interface";
+export {IProtoBlock, IProtoNode, ExecutionStatus} from "./protoArray/interface";
 
 export {ForkChoice} from "./forkChoice/forkChoice";
 export {

packages/fork-choice/src/protoArray/interface.ts

@@ -13,12 +13,22 @@ export interface IVoteTracker {
   nextEpoch: Epoch;
 }
 
+export enum ExecutionStatus {
+  Valid = "Valid",
+  Syncing = "Syncing",
+  PreMerge = "PreMerge",
+}
+
+type BlockExecution =
+  | {executionPayloadBlockHash: RootHex; executionStatus: ExecutionStatus.Valid | ExecutionStatus.Syncing}
+  | {executionPayloadBlockHash: null; executionStatus: ExecutionStatus.PreMerge};
 /**
  * A block that is to be applied to the fork choice
  *
  * A simplified version of BeaconBlock
  */
-export interface IProtoBlock {
+
+export type IProtoBlock = BlockExecution & {
   /**
    * The slot is not necessary for ProtoArray,
    * it just exists so external components can easily query the block slot.
@@ -39,25 +49,20 @@ export interface IProtoBlock {
    * it also just exists for upstream components (namely attestation verification)
    */
   targetRoot: RootHex;
-  /**
-   * `executionPayloadBlockHash` is not necessary for ProtoArray either.
-   * Here to do ExecutionEngine.notify_forkchoice_updated() easier.
-   * TODO: Check with other teams if this is the optimal strategy
-   */
-  executionPayloadBlockHash: RootHex | null;
+
   justifiedEpoch: Epoch;
   justifiedRoot: RootHex;
   finalizedEpoch: Epoch;
   finalizedRoot: RootHex;
-}
+};
 
 /**
  * A block root with additional metadata required to form a DAG
  * with vote weights and best blocks stored as metadata
  */
-export interface IProtoNode extends IProtoBlock {
+export type IProtoNode = IProtoBlock & {
   parent?: number;
   weight: number;
   bestChild?: number;
   bestDescendant?: number;
-}
+};

packages/fork-choice/src/protoArray/protoArray.ts

@@ -50,7 +50,7 @@ export class ProtoArray {
       ...block,
       // We are using the blockROot as the targetRoot, since it always lies on an epoch boundary
       targetRoot: block.blockRoot,
-    });
+    } as IProtoBlock);
     return protoArray;
   }
 

packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts

@@ -1,4 +1,4 @@
-import {ForkChoice, IForkChoiceStore, IProtoBlock, ProtoArray} from "../../../src";
+import {ForkChoice, IForkChoiceStore, IProtoBlock, ProtoArray, ExecutionStatus} from "../../../src";
 import {config} from "@chainsafe/lodestar-config/default";
 import {expect} from "chai";
 import {fromHexString} from "@chainsafe/ssz";
@@ -18,12 +18,15 @@ describe("Forkchoice", function () {
     stateRoot,
     parentRoot,
     blockRoot: finalizedRoot,
-    executionPayloadBlockHash: null,
+
     justifiedEpoch: genesisEpoch,
     justifiedRoot: genesisRoot,
     finalizedEpoch: genesisEpoch,
     finalizedRoot: genesisRoot,
-  });
+
+    executionPayloadBlockHash: null,
+    executionStatus: ExecutionStatus.PreMerge,
+  } as Omit<IProtoBlock, "targetRoot">);
 
   // Add block that is a finalized descendant.
   const block: IProtoBlock = {
@@ -32,11 +35,14 @@ describe("Forkchoice", function () {
     parentRoot: finalizedRoot,
     stateRoot,
     targetRoot: finalizedRoot,
-    executionPayloadBlockHash: null,
+
     justifiedEpoch: genesisEpoch,
     justifiedRoot: genesisRoot,
     finalizedEpoch: genesisEpoch,
     finalizedRoot: genesisRoot,
+
+    executionPayloadBlockHash: null,
+    executionStatus: ExecutionStatus.PreMerge,
   };
 
   const fcStore: IForkChoiceStore = {

packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts

@@ -1,5 +1,5 @@
 import {expect} from "chai";
-import {ProtoArray} from "../../../src";
+import {ProtoArray, ExecutionStatus} from "../../../src";
 
 describe("getCommonAncestor", () => {
   const blocks: {slot: number; root: string; parent: string}[] = [
@@ -29,11 +29,13 @@ describe("getCommonAncestor", () => {
     stateRoot: "-",
     parentRoot: "-",
     blockRoot: "0",
-    executionPayloadBlockHash: null,
+
     justifiedEpoch: 0,
     justifiedRoot: "-",
     finalizedEpoch: 0,
     finalizedRoot: "-",
+
+    ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge},
   });
 
   for (const block of blocks) {
@@ -43,11 +45,13 @@ describe("getCommonAncestor", () => {
       parentRoot: block.parent,
       stateRoot: "-",
       targetRoot: "-",
-      executionPayloadBlockHash: null,
+
       justifiedEpoch: 0,
       justifiedRoot: "-",
       finalizedEpoch: 0,
       finalizedRoot: "-",
+
+      ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge},
     });
   }
 

packages/fork-choice/test/unit/protoArray/protoArray.test.ts

@@ -1,7 +1,7 @@
 import {RootHex} from "@chainsafe/lodestar-types";
 import {expect} from "chai";
 
-import {ProtoArray} from "../../../src";
+import {ProtoArray, ExecutionStatus} from "../../../src";
 
 describe("ProtoArray", () => {
   it("finalized descendant", () => {
@@ -19,11 +19,13 @@ describe("ProtoArray", () => {
       stateRoot,
       parentRoot,
       blockRoot: finalizedRoot,
-      executionPayloadBlockHash: null,
+
       justifiedEpoch: genesisEpoch,
       justifiedRoot: stateRoot,
       finalizedEpoch: genesisEpoch,
       finalizedRoot: stateRoot,
+
+      ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge},
     });
 
     // Add block that is a finalized descendant.
@@ -33,11 +35,13 @@ describe("ProtoArray", () => {
       parentRoot: finalizedRoot,
       stateRoot,
       targetRoot: finalizedRoot,
-      executionPayloadBlockHash: null,
+
       justifiedEpoch: genesisEpoch,
       justifiedRoot: stateRoot,
       finalizedEpoch: genesisEpoch,
       finalizedRoot: stateRoot,
+
+      ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge},
     });
 
     // Add block that is *not* a finalized descendant.
@@ -47,11 +51,13 @@ describe("ProtoArray", () => {
       parentRoot: unknown,
       stateRoot,
       targetRoot: finalizedRoot,
-      executionPayloadBlockHash: null,
+
       justifiedEpoch: genesisEpoch,
       justifiedRoot: stateRoot,
       finalizedEpoch: genesisEpoch,
       finalizedRoot: stateRoot,
+
+      ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge},
     });
 
     // ancestorRoot, descendantRoot, isDescendant