REST server fixes and improvements.

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -339,11 +339,17 @@ proc getForkedBlock*(
     dag.db.getForkedBlock(root)
 
 func isCanonical*(dag: ChainDAGRef, bid: BlockId): bool =
-  ## Return true iff the given `bid` is part of the history selected by `dag.head`
+  ## Returns `true` if the given `bid` is part of the history selected by
+  ## `dag.head`.
   let current = dag.getBlockIdAtSlot(bid.slot).valueOr:
     return false # We don't know, so ..
   return current.bid == bid
 
+func isFinalized*(dag: ChainDAGRef, bid: BlockId): bool =
+  ## Returns `true` if the given `bid` is part of the finalized history
+  ## selected by `dag.finalizedHead`.
+  dag.isCanonical(bid) and (bid.slot <= dag.finalizedHead.slot)
+
 func parent*(dag: ChainDAGRef, bid: BlockId): Opt[BlockId] =
   if bid.slot == 0:
     return err()

beacon_chain/deposits.nim

@@ -322,7 +322,7 @@ proc restValidatorExit(config: BeaconNodeConf) {.async.} =
         else:
           hadErrors = true
           let responseError = try:
-            Json.decode(response.data, RestErrorMessage)
+            RestJson.decode(response.data, RestErrorMessage)
           except CatchableError as exc:
             error "Failed to decode invalid error server response on " &
                   "`submitPoolVoluntaryExit` request", reason = exc.msg

beacon_chain/libnimbus_lc/libnimbus_lc.nim

@@ -148,7 +148,7 @@ proc ETHBeaconStateCreateFromSsz(
   ## * https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/capella/beacon-chain.md#beaconstate
   ## * https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/configs/README.md
   let
-    consensusFork = decodeEthConsensusVersion($consensusVersion).valueOr:
+    consensusFork = ConsensusFork.decodeString($consensusVersion).valueOr:
       return nil
     state = ForkedHashedBeaconState.new()
   try:
@@ -328,7 +328,7 @@ proc ETHLightClientStoreCreateFromBootstrap(
   ## * https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/phase0/weak-subjectivity.md#weak-subjectivity-period
   let
     mediaType = MediaType.init($mediaType)
-    consensusFork = decodeEthConsensusVersion($consensusVersion).valueOr:
+    consensusFork = ConsensusFork.decodeString($consensusVersion).valueOr:
       return nil
   var bootstrap =
     try:
@@ -603,7 +603,7 @@ proc ETHLightClientStoreProcessFinalityUpdate(
           finUpdateBytes.toOpenArray(0, numFinUpdateBytes - 1),
           Opt.none(ConsensusFork), cfg[])
       else:
-        let consensusFork = decodeEthConsensusVersion(
+        let consensusFork = ConsensusFork.decodeString(
             $consensusVersion).valueOr:
           return 1
         ForkedLightClientFinalityUpdate.decodeHttpLightClientObject(
@@ -688,7 +688,7 @@ proc ETHLightClientStoreProcessOptimisticUpdate(
           optUpdateBytes.toOpenArray(0, numOptUpdateBytes - 1),
           Opt.none(ConsensusFork), cfg[])
       else:
-        let consensusFork = decodeEthConsensusVersion(
+        let consensusFork = ConsensusFork.decodeString(
             $consensusVersion).valueOr:
           return 1
         ForkedLightClientOptimisticUpdate.decodeHttpLightClientObject(

beacon_chain/rpc/rest_beacon_api.nim

@@ -167,9 +167,10 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
                                          $error)
 
     node.withStateForBlockSlotId(bslot):
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         (root: stateRoot),
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
 
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
@@ -190,7 +191,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
                                           $error)
 
     node.withStateForBlockSlotId(bslot):
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         (
           previous_version:
             getStateField(state, fork).previous_version,
@@ -199,7 +200,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
           epoch:
             getStateField(state, fork).epoch
         ),
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
 
@@ -220,7 +222,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
                                          $error)
 
     node.withStateForBlockSlotId(bslot):
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         (
           previous_justified:
             getStateField(state, previous_justified_checkpoint),
@@ -229,7 +231,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
           finalized:
             getStateField(state, finalized_checkpoint)
         ),
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
 
@@ -255,7 +258,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
                                            InvalidValidatorIdValueError)
         let ires = id.get()
         if len(ires) > ServerMaximumValidatorIds:
-          return RestApiResponse.jsonError(Http400,
+          return RestApiResponse.jsonError(Http414,
                                            MaximumNumberOfValidatorIdsError)
         ires
 
@@ -353,9 +356,10 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
                 res.add(RestValidator.init(index, balance, toString(status),
                                            validator))
           res
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         response,
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
 
@@ -417,9 +421,10 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
                                                ValidatorStatusNotFoundError,
                                                $sres.get())
             toString(sres.get())
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         RestValidator.init(vindex, balance, status, validator),
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
 
@@ -507,9 +512,10 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
               let balance = getStateField(state, balances).item(index)
               res.add(RestValidatorBalance.init(index, balance))
           res
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         response,
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
 
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
@@ -623,9 +629,10 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
       else:
         forSlot(vslot.get(), vindex, res)
 
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         res,
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
 
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
@@ -703,10 +710,11 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
             offset.inc(length)
           res
 
-      return RestApiResponse.jsonResponseWOpt(
+      return RestApiResponse.jsonResponseFinalized(
         RestEpochSyncCommittee(validators: indices,
                                validator_aggregates: aggregates),
-        node.getStateOptimistic(state)
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
       )
 
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
@@ -762,9 +770,10 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
     # Fall back to full state computation
     node.withStateForBlockSlotId(bslot):
       withState(state):
-        return RestApiResponse.jsonResponseWOpt(
+        return RestApiResponse.jsonResponseFinalized(
           RestEpochRandao(randao: get_randao_mix(forkyState.data, qepoch)),
-          node.getStateOptimistic(state)
+          node.getStateOptimistic(state),
+          node.dag.isFinalized(bslot.bid)
         )
 
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
@@ -796,19 +805,20 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
 
     return
       withBlck(bdata):
-        RestApiResponse.jsonResponseWOpt(
+        let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)
+        RestApiResponse.jsonResponseFinalized(
           [
             (
               root: forkyBlck.root,
-              canonical: node.dag.isCanonical(
-                BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)),
+              canonical: node.dag.isCanonical(bid),
               header: (
                 message: forkyBlck.toBeaconBlockHeader,
                 signature: forkyBlck.signature
               )
             )
           ],
-          node.getBlockOptimistic(bdata)
+          node.getBlockOptimistic(bdata),
+          node.dag.isFinalized(bid)
         )
 
   # https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeader
@@ -824,17 +834,18 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
 
     return
       withBlck(bdata):
-        RestApiResponse.jsonResponseWOpt(
+        let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)
+        RestApiResponse.jsonResponseFinalized(
           (
             root: forkyBlck.root,
-            canonical: node.dag.isCanonical(
-              BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)),
+            canonical: node.dag.isCanonical(bid),
             header: (
               message: forkyBlck.toBeaconBlockHeader,
               signature: forkyBlck.signature
             )
           ),
-          node.getBlockOptimistic(bdata)
+          node.getBlockOptimistic(bdata),
+          node.dag.isFinalized(bid)
         )
 
   # https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlock
@@ -850,8 +861,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
         var
           restBlock = decodeBody(RestPublishedSignedBlockContents, body,
                                  version).valueOr:
-            return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
-                                             $error)
+            return RestApiResponse.jsonError(error)
           forked = ForkedSignedBeaconBlock.init(restBlock)
 
         if restBlock.kind != node.dag.cfg.consensusForkAtEpoch(
@@ -912,8 +922,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
         var
           restBlock = decodeBodyJsonOrSsz(RestPublishedSignedBlockContents,
                                           body, version).valueOr:
-            return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
-                                             $error)
+            return RestApiResponse.jsonError(error)
           forked = ForkedSignedBeaconBlock.init(restBlock)
 
         # TODO (henridf): handle broadcast_validation flag
@@ -978,19 +987,38 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
       version = request.headers.getString("eth-consensus-version")
       body = contentBody.get()
 
-    if  body.contentType == OctetStreamMediaType and
-        currentEpochFork.toString != version:
+    if (body.contentType == OctetStreamMediaType) and
+       (currentEpochFork.toString != version):
       return RestApiResponse.jsonError(Http400, BlockIncorrectFork)
 
     case currentEpochFork
     of ConsensusFork.Deneb:
-      return RestApiResponse.jsonError(Http500, $denebImplementationMissing)
+      let
+        restBlock = decodeBodyJsonOrSsz(deneb_mev.SignedBlindedBeaconBlock,
+                                        body).valueOr:
+          return RestApiResponse.jsonError(error)
+
+        payloadBuilderClient = node.getPayloadBuilderClient(
+            restBlock.message.proposer_index).valueOr:
+          return RestApiResponse.jsonError(
+            Http400, "Unable to initialize payload builder client: " & $error)
+        res = await node.unblindAndRouteBlockMEV(
+          payloadBuilderClient, restBlock)
+
+      if res.isErr():
+        return RestApiResponse.jsonError(
+          Http503, BeaconNodeInSyncError, $res.error())
+      if res.get().isNone():
+        return RestApiResponse.jsonError(Http202, BlockValidationError)
+
+      return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
     of ConsensusFork.Capella:
       let
-        restBlock = decodeBodyJsonOrSsz(
-            capella_mev.SignedBlindedBeaconBlock, body).valueOr:
-          return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
-                                           $error)
+        restBlock =
+          decodeBodyJsonOrSsz(capella_mev.SignedBlindedBeaconBlock,
+                              body).valueOr:
+          return RestApiResponse.jsonError(error)
+
         payloadBuilderClient = node.getPayloadBuilderClient(
             restBlock.message.proposer_index).valueOr:
           return RestApiResponse.jsonError(
@@ -1006,7 +1034,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
 
       return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
     of ConsensusFork.Bellatrix:
-      return RestApiResponse.jsonError(Http400, "FOO")
+      return RestApiResponse.jsonError(Http400,
+                                       "Bellatrix builder API unsupported")
     of ConsensusFork.Altair, ConsensusFork.Phase0:
       # Pre-Bellatrix, this endpoint will accept a `SignedBeaconBlock`.
       #
@@ -1015,8 +1044,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
       var
         restBlock = decodeBody(RestPublishedSignedBeaconBlock, body,
                                version).valueOr:
-          return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
-                                           $error)
+          return RestApiResponse.jsonError(error)
         forked = ForkedSignedBeaconBlock(restBlock)
 
       if forked.kind != node.dag.cfg.consensusForkAtEpoch(
@@ -1076,7 +1104,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
 
         RestApiResponse.jsonResponseBlock(
           bdata.asSigned(),
-          node.getBlockOptimistic(bdata)
+          node.getBlockOptimistic(bdata),
+          node.dag.isFinalized(bid)
         )
       else:
         RestApiResponse.jsonError(Http500, InvalidAcceptError)
@@ -1095,28 +1124,30 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
       bdata = node.dag.getForkedBlock(bid).valueOr:
         return RestApiResponse.jsonError(Http404, BlockNotFoundError)
 
-    return RestApiResponse.jsonResponseWOpt(
+    return RestApiResponse.jsonResponseFinalized(
       (root: bid.root),
-      node.getBlockOptimistic(bdata)
+      node.getBlockOptimistic(bdata),
+      node.dag.isFinalized(bid)
     )
 
   # https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockAttestations
   router.api(MethodGet,
              "/eth/v1/beacon/blocks/{block_id}/attestations") do (
     block_id: BlockIdent) -> RestApiResponse:
     let
-      bid = block_id.valueOr:
+      blockIdent = block_id.valueOr:
         return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
                                          $error)
-
-      bdata = node.getForkedBlock(bid).valueOr:
+      bdata = node.getForkedBlock(blockIdent).valueOr:
         return RestApiResponse.jsonError(Http404, BlockNotFoundError)
 
     return
       withBlck(bdata):
-        RestApiResponse.jsonResponseWOpt(
+        let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)
+        RestApiResponse.jsonResponseFinalized(
           forkyBlck.message.body.attestations.asSeq(),
-          node.getBlockOptimistic(bdata)
+          node.getBlockOptimistic(bdata),
+          node.dag.isFinalized(bid)
         )
 
   # https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttestations

beacon_chain/rpc/rest_key_management_api.nim

@@ -123,7 +123,7 @@ proc handleRemoveValidatorReq(host: KeymanagerHost,
         return RemoteKeystoreStatus(status: KeystoreStatus.notFound)
     else:
       return RemoteKeystoreStatus(status: KeystoreStatus.error,
-                                  message: some($res.error()))
+                                  message: Opt.some($res.error()))
 
 proc handleAddRemoteValidatorReq(host: KeymanagerHost,
                                  keystore: RemoteKeystore): RequestItemStatus =