GetBlobSidecars API update

CHANGELOG.md

@@ -12,5 +12,6 @@ the [releases page](https://github.com/Consensys/teku/releases).
 ### Breaking Changes
 
 ### Additions and Improvements
+- Added metadata fields to getBlobSidecars response.  
 
 ### Bug Fixes

data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/GetBlobSidecarsResponse.json

@@ -3,6 +3,16 @@
   "type" : "object",
   "required" : [ "data" ],
   "properties" : {
+    "version" : {
+      "type" : "string",
+      "enum" : [ "phase0", "altair", "bellatrix", "capella", "deneb", "electra" ]
+    },
+    "execution_optimistic" : {
+      "type" : "boolean"
+    },
+    "finalized" : {
+      "type" : "boolean"
+    },
     "data" : {
       "type" : "array",
       "items" : {

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecars.java

@@ -15,15 +15,18 @@
 
 import static tech.pegasys.teku.beaconrestapi.BeaconRestApiTypes.BLOB_INDICES_PARAMETER;
 import static tech.pegasys.teku.beaconrestapi.BeaconRestApiTypes.PARAMETER_BLOCK_ID;
+import static tech.pegasys.teku.ethereum.json.types.EthereumTypes.MILESTONE_TYPE;
 import static tech.pegasys.teku.infrastructure.http.HttpStatusCodes.SC_OK;
+import static tech.pegasys.teku.infrastructure.http.RestApiConstants.EXECUTION_OPTIMISTIC;
+import static tech.pegasys.teku.infrastructure.http.RestApiConstants.FINALIZED;
 import static tech.pegasys.teku.infrastructure.http.RestApiConstants.TAG_BEACON;
+import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.BOOLEAN_TYPE;
 import static tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition.listOf;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
-import java.util.function.Function;
 import tech.pegasys.teku.api.ChainDataProvider;
 import tech.pegasys.teku.api.DataProvider;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
@@ -38,6 +41,7 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
+import tech.pegasys.teku.spec.datastructures.metadata.BlobSidecarsAndMetaData;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionCache;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 
@@ -75,7 +79,7 @@ private static EndpointMetadata createEndpointMetadata(final SchemaDefinitionCac
   @Override
   public void handleRequest(final RestApiRequest request) throws JsonProcessingException {
     final List<UInt64> indices = request.getQueryParameterList(BLOB_INDICES_PARAMETER);
-    final SafeFuture<Optional<List<BlobSidecar>>> future =
+    final SafeFuture<Optional<BlobSidecarsAndMetaData>> future =
         chainDataProvider.getBlobSidecars(request.getPathParameter(PARAMETER_BLOCK_ID), indices);
 
     request.respondAsync(
@@ -86,21 +90,26 @@ public void handleRequest(final RestApiRequest request) throws JsonProcessingExc
                     .orElse(AsyncApiResponse.respondNotFound())));
   }
 
-  private static SerializableTypeDefinition<List<BlobSidecar>> getResponseType(
+  private static SerializableTypeDefinition<BlobSidecarsAndMetaData> getResponseType(
       final SchemaDefinitionCache schemaCache) {
     final DeserializableTypeDefinition<BlobSidecar> blobSidecarType =
         SchemaDefinitionsDeneb.required(schemaCache.getSchemaDefinition(SpecMilestone.DENEB))
             .getBlobSidecarSchema()
             .getJsonTypeDefinition();
-    return SerializableTypeDefinition.<List<BlobSidecar>>object()
+    return SerializableTypeDefinition.<BlobSidecarsAndMetaData>object()
         .name("GetBlobSidecarsResponse")
-        .withField("data", listOf(blobSidecarType), Function.identity())
+        .withOptionalField("version", MILESTONE_TYPE, (b) -> Optional.of(b.getMilestone()))
+        .withOptionalField(
+            EXECUTION_OPTIMISTIC, BOOLEAN_TYPE, (b) -> Optional.of(b.isExecutionOptimistic()))
+        .withOptionalField(FINALIZED, BOOLEAN_TYPE, (b) -> Optional.of(b.isFinalized()))
+        .withField("data", listOf(blobSidecarType), BlobSidecarsAndMetaData::getData)
         .build();
   }
 
-  private static ResponseContentTypeDefinition<List<BlobSidecar>> getSszResponseType() {
-    final OctetStreamResponseContentTypeDefinition.OctetStreamSerializer<List<BlobSidecar>>
-        serializer = (data, out) -> data.forEach(blobSidecar -> blobSidecar.sszSerialize(out));
+  private static ResponseContentTypeDefinition<BlobSidecarsAndMetaData> getSszResponseType() {
+    final OctetStreamResponseContentTypeDefinition.OctetStreamSerializer<BlobSidecarsAndMetaData>
+        serializer =
+            (data, out) -> data.getData().forEach(blobSidecar -> blobSidecar.sszSerialize(out));
 
     return new OctetStreamResponseContentTypeDefinition<>(serializer, __ -> Collections.emptyMap());
   }

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecarsTest.java

@@ -34,6 +34,7 @@
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
+import tech.pegasys.teku.spec.datastructures.metadata.BlobSidecarsAndMetaData;
 import tech.pegasys.teku.spec.datastructures.metadata.ObjectAndMetaData;
 
 class GetBlobSidecarsTest extends AbstractMigratedBeaconHandlerWithChainDataProviderTest {
@@ -61,7 +62,8 @@ void shouldReturnBlobSidecars()
     handler.handleRequest(request);
 
     assertThat(request.getResponseCode()).isEqualTo(SC_OK);
-    assertThat(request.getResponseBody()).isEqualTo(blobSidecars);
+    assertThat(((BlobSidecarsAndMetaData) request.getResponseBody()).getData())
+        .isEqualTo(blobSidecars);
   }
 
   @Test
@@ -82,8 +84,9 @@ void metadata_shouldHandle500() throws JsonProcessingException {
   @Test
   void metadata_shouldHandle200() throws IOException {
     final List<BlobSidecar> blobSidecars = dataStructureUtil.randomBlobSidecars(3);
-
-    final String data = getResponseStringFromMetadata(handler, SC_OK, blobSidecars);
+    final BlobSidecarsAndMetaData blobSidecarsAndMetaData =
+        new BlobSidecarsAndMetaData(blobSidecars, SpecMilestone.DENEB, true, false, false);
+    final String data = getResponseStringFromMetadata(handler, SC_OK, blobSidecarsAndMetaData);
     final String expected =
         Resources.toString(
             Resources.getResource(GetBlobSidecarsTest.class, "getBlobSidecars.json"), UTF_8);

data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java

@@ -65,6 +65,7 @@
 import tech.pegasys.teku.spec.datastructures.forkchoice.ProtoNodeData;
 import tech.pegasys.teku.spec.datastructures.forkchoice.ReadOnlyForkChoiceStrategy;
 import tech.pegasys.teku.spec.datastructures.lightclient.LightClientBootstrap;
+import tech.pegasys.teku.spec.datastructures.metadata.BlobSidecarsAndMetaData;
 import tech.pegasys.teku.spec.datastructures.metadata.BlockAndMetaData;
 import tech.pegasys.teku.spec.datastructures.metadata.ObjectAndMetaData;
 import tech.pegasys.teku.spec.datastructures.metadata.StateAndMetaData;
@@ -100,7 +101,7 @@ public ChainDataProvider(
         combinedChainDataClient,
         new BlockSelectorFactory(spec, combinedChainDataClient),
         new StateSelectorFactory(spec, combinedChainDataClient),
-        new BlobSidecarSelectorFactory(combinedChainDataClient),
+        new BlobSidecarSelectorFactory(spec, combinedChainDataClient),
         rewardCalculator);
   }
 
@@ -179,7 +180,7 @@ public SafeFuture<Optional<ObjectAndMetaData<SignedBeaconBlock>>> getBlock(
     return fromBlock(blockIdParam, Function.identity());
   }
 
-  public SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
+  public SafeFuture<Optional<BlobSidecarsAndMetaData>> getBlobSidecars(
       final String blockIdParam, final List<UInt64> indices) {
     return blobSidecarSelectorFactory
         .createSelectorForBlockId(blockIdParam)
@@ -188,7 +189,11 @@ public SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
 
   public SafeFuture<Optional<List<BlobSidecar>>> getAllBlobSidecarsAtSlot(
       final UInt64 slot, final List<UInt64> indices) {
-    return blobSidecarSelectorFactory.slotSelectorForAll(slot).getBlobSidecars(indices);
+    return blobSidecarSelectorFactory
+        .slotSelectorForAll(slot)
+        .getBlobSidecars(indices)
+        .thenApply(
+            maybeBlobSideCarsMetaData -> maybeBlobSideCarsMetaData.map(ObjectAndMetaData::getData));
   }
 
   public SafeFuture<Optional<ObjectAndMetaData<Bytes32>>> getBlockRoot(final String blockIdParam) {

data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelector.java

@@ -17,8 +17,8 @@
 import java.util.Optional;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
-import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
+import tech.pegasys.teku.spec.datastructures.metadata.BlobSidecarsAndMetaData;
 
 public interface BlobSidecarSelector {
-  SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(List<UInt64> indices);
+  SafeFuture<Optional<BlobSidecarsAndMetaData>> getBlobSidecars(List<UInt64> indices);
 }

data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java

@@ -22,16 +22,22 @@
 import tech.pegasys.teku.api.AbstractSelectorFactory;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb;
+import tech.pegasys.teku.spec.datastructures.metadata.BlobSidecarsAndMetaData;
+import tech.pegasys.teku.storage.client.ChainHead;
 import tech.pegasys.teku.storage.client.CombinedChainDataClient;
 
 public class BlobSidecarSelectorFactory extends AbstractSelectorFactory<BlobSidecarSelector> {
 
-  public BlobSidecarSelectorFactory(final CombinedChainDataClient client) {
+  private final Spec spec;
+
+  public BlobSidecarSelectorFactory(final Spec spec, final CombinedChainDataClient client) {
     super(client);
+    this.spec = spec;
   }
 
   @Override
@@ -44,11 +50,23 @@ public BlobSidecarSelector blockRootSelector(final Bytes32 blockRoot) {
                   if (maybeSlot.isPresent()) {
                     final SlotAndBlockRoot slotAndBlockRoot =
                         new SlotAndBlockRoot(maybeSlot.get(), blockRoot);
-                    return getBlobSidecars(slotAndBlockRoot, indices);
+                    return getBlobSidecars(slotAndBlockRoot, indices)
+                        .thenApply(blobSidecars -> addMetaData(blobSidecars, slotAndBlockRoot));
                   }
                   return client
                       .getBlockByBlockRoot(blockRoot)
-                      .thenCompose(maybeBlock -> getBlobSidecarsForBlock(maybeBlock, indices));
+                      .thenCompose(
+                          maybeBlock -> {
+                            if (maybeBlock.isEmpty()) {
+                              return SafeFuture.completedFuture(Optional.empty());
+                            }
+                            final SignedBeaconBlock block = maybeBlock.get();
+                            final SlotAndBlockRoot slotAndBlockRoot =
+                                new SlotAndBlockRoot(block.getSlot(), blockRoot);
+                            return getBlobSidecarsForBlock(maybeBlock, indices)
+                                .thenApply(
+                                    blobSidecars -> addMetaData(blobSidecars, slotAndBlockRoot));
+                          });
                 });
   }
 
@@ -57,7 +75,13 @@ public BlobSidecarSelector headSelector() {
     return indices ->
         client
             .getChainHead()
-            .map(head -> getBlobSidecars(head.getSlotAndBlockRoot(), indices))
+            .map(
+                head ->
+                    getBlobSidecars(head.getSlotAndBlockRoot(), indices)
+                        .thenApply(
+                            blobSideCars ->
+                                addMetaData(
+                                    blobSideCars, head.getSlotAndBlockRoot(), head.isOptimistic())))
             .orElse(SafeFuture.completedFuture(Optional.empty()));
   }
 
@@ -66,27 +90,58 @@ public BlobSidecarSelector genesisSelector() {
     return indices ->
         client
             .getBlockAtSlotExact(GENESIS_SLOT)
-            .thenCompose(maybeGenesisBlock -> getBlobSidecarsForBlock(maybeGenesisBlock, indices));
+            .thenCompose(
+                maybeGenesisBlock ->
+                    getBlobSidecarsForBlock(maybeGenesisBlock, indices)
+                        .thenApply(
+                            blobSidecars ->
+                                addMetaData(
+                                    blobSidecars,
+                                    GENESIS_SLOT,
+                                    false,
+                                    true,
+                                    client.isFinalized(GENESIS_SLOT))));
   }
 
   @Override
   public BlobSidecarSelector finalizedSelector() {
     return indices ->
         client
             .getLatestFinalized()
-            .map(anchorPoint -> getBlobSidecars(anchorPoint.getSlotAndBlockRoot(), indices))
+            .map(
+                anchorPoint ->
+                    getBlobSidecars(anchorPoint.getSlotAndBlockRoot(), indices)
+                        .thenApply(
+                            blobSideCars ->
+                                addMetaData(
+                                    blobSideCars,
+                                    anchorPoint.getSlotAndBlockRoot(),
+                                    client.isChainHeadOptimistic())))
             .orElse(SafeFuture.completedFuture(Optional.empty()));
   }
 
   @Override
   public BlobSidecarSelector slotSelector(final UInt64 slot) {
     return indices -> {
       if (client.isFinalized(slot)) {
-        return getBlobSidecars(slot, indices);
+        return getBlobSidecars(slot, indices)
+            .thenApply(
+                blobSidecars ->
+                    addMetaData(blobSidecars, slot, client.isChainHeadOptimistic(), true, true));
       }
       return client
           .getBlockAtSlotExact(slot)
-          .thenCompose(maybeBlock -> getBlobSidecarsForBlock(maybeBlock, indices));
+          .thenCompose(
+              maybeBlock ->
+                  getBlobSidecarsForBlock(maybeBlock, indices)
+                      .thenApply(
+                          blobSidecars ->
+                              addMetaData(
+                                  blobSidecars,
+                                  slot,
+                                  client.isChainHeadOptimistic(),
+                                  false,
+                                  client.isFinalized(slot))));
     };
   }
 
@@ -96,7 +151,12 @@ public BlobSidecarSelector slotSelectorForAll(final UInt64 slot) {
             .getAllBlobSidecars(slot, indices)
             .thenApply(
                 blobSidecars ->
-                    blobSidecars.isEmpty() ? Optional.empty() : Optional.of(blobSidecars));
+                    blobSidecars.isEmpty()
+                        ? Optional.empty()
+                        : addMetaData(
+                            // We don't care about metadata since the api (teku only) that
+                            // consumes the return value doesn't use it
+                            Optional.of(blobSidecars), new SlotAndBlockRoot(slot, Bytes32.ZERO)));
   }
 
   private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecarsForBlock(
@@ -125,4 +185,60 @@ private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
       final UInt64 slot, final List<UInt64> indices) {
     return client.getBlobSidecars(slot, indices).thenApply(Optional::of);
   }
+
+  private Optional<BlobSidecarsAndMetaData> addMetaData(
+      final Optional<List<BlobSidecar>> maybeBlobSidecarList,
+      final SlotAndBlockRoot slotAndBlockRoot) {
+    if (maybeBlobSidecarList.isEmpty()) {
+      return Optional.empty();
+    }
+
+    final UInt64 slot = slotAndBlockRoot.getSlot();
+    final Bytes32 blockRoot = slotAndBlockRoot.getBlockRoot();
+    final Optional<ChainHead> maybeChainHead = client.getChainHead();
+    final boolean isFinalized = client.isFinalized(slot);
+    boolean isOptimistic;
+    boolean isCanonical = false;
+
+    if (maybeChainHead.isPresent()) {
+      ChainHead chainHead = maybeChainHead.get();
+      isOptimistic = chainHead.isOptimistic() || client.isOptimisticBlock(blockRoot);
+      isCanonical = client.isCanonicalBlock(slot, blockRoot, chainHead.getRoot());
+    } else {
+      // If there's no chain head, we assume the block is not optimistic and not canonical
+      isOptimistic = client.isOptimisticBlock(blockRoot);
+    }
+    return addMetaData(maybeBlobSidecarList, slot, isOptimistic, isCanonical, isFinalized);
+  }
+
+  private Optional<BlobSidecarsAndMetaData> addMetaData(
+      final Optional<List<BlobSidecar>> maybeBlobSidecarList,
+      final SlotAndBlockRoot slotAndBlockRoot,
+      final boolean isOptimistic) {
+    if (maybeBlobSidecarList.isEmpty()) {
+      return Optional.empty();
+    }
+    return addMetaData(
+        maybeBlobSidecarList,
+        slotAndBlockRoot.getSlot(),
+        isOptimistic,
+        true,
+        client.isFinalized(slotAndBlockRoot.getSlot()));
+  }
+
+  private Optional<BlobSidecarsAndMetaData> addMetaData(
+      final Optional<List<BlobSidecar>> maybeBlobSidecarList,
+      final UInt64 blockSlot,
+      final boolean executionOptimistic,
+      final boolean canonical,
+      final boolean finalized) {
+    return maybeBlobSidecarList.map(
+        blobSidecarList ->
+            new BlobSidecarsAndMetaData(
+                blobSidecarList,
+                spec.atSlot(blockSlot).getMilestone(),
+                executionOptimistic,
+                canonical,
+                finalized));
+  }
 }

data/provider/src/test/java/tech/pegasys/teku/api/AbstractChainDataProviderTest.java

@@ -104,7 +104,7 @@ protected ChainDataProvider setupBySpec(
     this.blockSelectorFactory = spy(new BlockSelectorFactory(spec, mockCombinedChainDataClient));
     this.stateSelectorFactory = spy(new StateSelectorFactory(spec, mockCombinedChainDataClient));
     this.blobSidecarSelectorFactory =
-        spy(new BlobSidecarSelectorFactory(mockCombinedChainDataClient));
+        spy(new BlobSidecarSelectorFactory(spec, mockCombinedChainDataClient));
     final ChainDataProvider provider =
         new ChainDataProvider(
             spec,