From dc8aa88e84c621bed03ce2223c91439ea0a8c086 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 27 Apr 2023 23:39:46 +0400 Subject: [PATCH 1/5] BlobSidecar pruning + tests + fixes + removal of several BlobsSidecar traces --- .../lookup/BlobsSidecarProvider.java | 31 -- .../forkchoice/ReadOnlyStore.java | 4 - .../forkchoice/TestStoreImpl.java | 7 - .../teku/spec/util/DataStructureUtil.java | 24 +- .../blobs/BlobSidecarManager.java | 11 - .../blobs/BlobSidecarManagerImpl.java | 34 -- .../blobs/BlobSidecarManagerTest.java | 54 +-- .../services/chainstorage/StorageService.java | 11 +- .../teku/storage/api/StorageQueryChannel.java | 3 - .../storage/api/StorageUpdateChannel.java | 5 - .../storage/server/kvstore/DatabaseTest.java | 317 +++++++++-------- .../teku/storage/client/RecentChainData.java | 5 - .../client/StorageBackedRecentChainData.java | 5 - .../teku/storage/server/ChainStorage.java | 17 - .../CombinedStorageChannelSplitter.java | 17 - .../pegasys/teku/storage/server/Database.java | 51 +-- .../server/RetryingStorageUpdateChannel.java | 11 - .../server/kvstore/KvStoreDatabase.java | 161 ++------- .../dataaccess/CombinedKvStoreDao.java | 90 +---- .../dataaccess/KvStoreCombinedDao.java | 25 +- .../dataaccess/KvStoreCombinedDaoAdapter.java | 56 +-- .../dataaccess/V4FinalizedKvStoreDao.java | 87 +---- .../server/kvstore/schema/SchemaCombined.java | 4 - .../SchemaFinalizedSnapshotStateAdapter.java | 14 - .../kvstore/schema/V6SchemaCombined.java | 5 +- .../schema/V6SchemaCombinedSnapshot.java | 31 +- .../schema/V6SchemaCombinedTreeState.java | 30 +- .../storage/server/noop/NoOpDatabase.java | 55 +-- ...ecarPruner.java => BlobSidecarPruner.java} | 77 +--- .../pegasys/teku/storage/store/Store.java | 72 ---- .../teku/storage/store/StoreBuilder.java | 9 - .../teku/storage/store/StoreTransaction.java | 7 - .../StorageBackedRecentChainDataTest.java | 3 - ...erTest.java => BlobSidecarPrunerTest.java} | 41 +-- .../teku/storage/store/AbstractStoreTest.java | 7 - .../pegasys/teku/storage/store/StoreTest.java | 98 ------ .../storage/api/StubStorageQueryChannel.java | 7 - .../storage/api/StubStorageUpdateChannel.java | 11 - .../StubStorageUpdateChannelWithDelays.java | 11 - .../teku/storage/client/ChainUpdater.java | 15 +- .../client/MemoryOnlyRecentChainData.java | 2 - .../cli/subcommand/debug/DebugDbCommand.java | 333 +----------------- 42 files changed, 297 insertions(+), 1561 deletions(-) delete mode 100644 ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/lookup/BlobsSidecarProvider.java rename storage/src/main/java/tech/pegasys/teku/storage/server/pruner/{BlobsSidecarPruner.java => BlobSidecarPruner.java} (66%) rename storage/src/test/java/tech/pegasys/teku/storage/server/pruner/{BlobsSidecarPrunerTest.java => BlobSidecarPrunerTest.java} (74%) diff --git a/ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/lookup/BlobsSidecarProvider.java b/ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/lookup/BlobsSidecarProvider.java deleted file mode 100644 index 8f1372d3b83..00000000000 --- a/ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/lookup/BlobsSidecarProvider.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright ConsenSys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.dataproviders.lookup; - -import java.util.Optional; -import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; -import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; - -@FunctionalInterface -public interface BlobsSidecarProvider { - BlobsSidecarProvider NOOP = (block) -> SafeFuture.completedFuture(Optional.empty()); - - default SafeFuture> getBlobsSidecar(SignedBeaconBlock block) { - return getBlobsSidecar(new SlotAndBlockRoot(block.getSlot(), block.getRoot())); - } - - SafeFuture> getBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); -} diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java index 4bb4bf58160..0f94fcec534 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java @@ -26,7 +26,6 @@ import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.blocks.StateAndBlockSummary; -import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.SignedBeaconBlockAndBlobsSidecar; import tech.pegasys.teku.spec.datastructures.execution.SlotAndExecutionPayloadSummary; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; import tech.pegasys.teku.spec.datastructures.state.Checkpoint; @@ -113,9 +112,6 @@ default SafeFuture> retrieveBlock(Bytes32 blockRoot) { SafeFuture> retrieveSignedBlock(Bytes32 blockRoot); - SafeFuture> retrieveSignedBlockAndBlobsSidecar( - Bytes32 blockRoot); - SafeFuture> retrieveBlockAndState(Bytes32 blockRoot); SafeFuture> retrieveStateAndBlockSummary(Bytes32 blockRoot); diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java index e28e1f791ff..57d56b8e480 100644 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java @@ -31,7 +31,6 @@ import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.blocks.StateAndBlockSummary; -import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.SignedBeaconBlockAndBlobsSidecar; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.datastructures.execution.SlotAndExecutionPayloadSummary; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; @@ -196,12 +195,6 @@ public SafeFuture> retrieveSignedBlock(Bytes32 block return SafeFuture.completedFuture(getBlockIfAvailable(blockRoot)); } - @Override - public SafeFuture> retrieveSignedBlockAndBlobsSidecar( - Bytes32 blockRoot) { - return SafeFuture.failedFuture(new UnsupportedOperationException("Not yet implemented")); - } - @Override public SafeFuture> retrieveBlockAndState(Bytes32 blockRoot) { return SafeFuture.completedFuture(getBlockAndState(blockRoot)); diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java index 3f80c592bc0..94fc27314b2 100644 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java @@ -2034,6 +2034,11 @@ public List randomBlobSidecars(int count) { return blobSidecars; } + public BlobSidecar randomBlobSidecar( + final UInt64 slot, final Bytes32 blockRoot, final UInt64 index) { + return new RandomBlobSidecarBuilder().slot(slot).index(index).blockRoot(blockRoot).build(); + } + public BlobSidecar randomBlobSidecar(final Bytes32 blockRoot, final UInt64 index) { return new RandomBlobSidecarBuilder().index(index).blockRoot(blockRoot).build(); } @@ -2311,25 +2316,6 @@ private SchemaDefinitionsDeneb getSchemaDefinitionsDeneb() { } } - public SignedBeaconBlockAndBlobsSidecar randomConsistentSignedBeaconBlockAndBlobsSidecar() { - return randomConsistentSignedBeaconBlockAndBlobsSidecar(randomUInt64()); - } - - public SignedBeaconBlockAndBlobsSidecar randomConsistentSignedBeaconBlockAndBlobsSidecar( - final UInt64 slot) { - final SignedBeaconBlock randomBlock = randomSignedBeaconBlock(slot); - return SchemaDefinitionsDeneb.required(spec.atSlot(slot).getSchemaDefinitions()) - .getSignedBeaconBlockAndBlobsSidecarSchema() - .create( - randomBlock, - randomBlobsSidecar( - randomBlock.getRoot(), - randomBlock.getSlot(), - BeaconBlockBodyDeneb.required(randomBlock.getBeaconBlock().orElseThrow().getBody()) - .getBlobKzgCommitments() - .size())); - } - public SszList randomSszKzgCommitmentList() { final SszListSchema kzgCommitmentsSchema = BeaconBlockBodySchemaDeneb.required( diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java index 26dd50efb4a..66a196eaad5 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java @@ -16,7 +16,6 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.SignedBlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -47,18 +46,12 @@ public boolean isAvailabilityRequiredAtSlot(final UInt64 slot) { return false; } - @Override - public void storeUnconfirmedValidatedBlobsSidecar(final BlobsSidecar blobsSidecar) {} - @Override public void storeNoBlobsSlot(SlotAndBlockRoot slotAndBlockRoot) {} @Override public void storeBlobSidecar(final BlobSidecar blobSidecar) {} - @Override - public void storeUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) {} - @Override public void discardBlobSidecarsByBlock(final SignedBeaconBlock block) {} @@ -78,14 +71,10 @@ SafeFuture validateAndImportBlobSidecar( boolean isAvailabilityRequiredAtSlot(UInt64 slot); - void storeUnconfirmedValidatedBlobsSidecar(BlobsSidecar blobsSidecar); - void storeNoBlobsSlot(SlotAndBlockRoot slotAndBlockRoot); void storeBlobSidecar(BlobSidecar blobSidecar); - void storeUnconfirmedBlobsSidecar(BlobsSidecar blobsSidecar); - void discardBlobSidecarsByBlock(SignedBeaconBlock block); BlobSidecarsAvailabilityChecker createAvailabilityChecker(SignedBeaconBlock block); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index 5ee35858d70..04e86865797 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -26,7 +26,6 @@ import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.collections.LimitedMap; import tech.pegasys.teku.infrastructure.subscribers.Subscribers; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; @@ -45,7 +44,6 @@ import tech.pegasys.teku.storage.client.RecentChainData; public class BlobSidecarManagerImpl implements BlobSidecarManager, SlotEventsChannel { - private static final int MAX_CACHED_VALIDATED_BLOBS_SIDECARS_PER_SLOT = 10; private static final Logger LOG = LogManager.getLogger(); private final Spec spec; @@ -116,16 +114,6 @@ public boolean isAvailabilityRequiredAtSlot(final UInt64 slot) { return spec.isAvailabilityOfBlobSidecarsRequiredAtSlot(recentChainData.getStore(), slot); } - @Override - public void storeUnconfirmedValidatedBlobsSidecar(final BlobsSidecar blobsSidecar) { - // cache already validated blobs - validatedPendingBlobs - .computeIfAbsent(blobsSidecar.getBeaconBlockSlot(), __ -> createNewMap()) - .put(blobsSidecar.getBeaconBlockRoot(), blobsSidecar); - - internalStoreUnconfirmedBlobsSidecar(blobsSidecar); - } - @Override public void storeNoBlobsSlot(final SlotAndBlockRoot slotAndBlockRoot) { storageUpdateChannel @@ -150,11 +138,6 @@ public void storeBlobSidecar(final BlobSidecar blobSidecar) { .ifExceptionGetsHereRaiseABug(); } - @Override - public void storeUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) { - internalStoreUnconfirmedBlobsSidecar(blobsSidecar); - } - @Override public void discardBlobSidecarsByBlock(final SignedBeaconBlock block) { storageUpdateChannel @@ -179,23 +162,6 @@ public void onSlot(final UInt64 slot) { validatedPendingBlobs.headMap(slot.minusMinZero(1)).clear(); } - private void internalStoreUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) { - storageUpdateChannel - .onBlobsSidecar(blobsSidecar) - .thenRun( - () -> - LOG.debug( - "Unconfirmed BlobsSidecar stored for {}", - () -> - new SlotAndBlockRoot( - blobsSidecar.getBeaconBlockSlot(), blobsSidecar.getBeaconBlockRoot()))) - .ifExceptionGetsHereRaiseABug(); - } - - private Map createNewMap() { - return LimitedMap.createSynchronized(MAX_CACHED_VALIDATED_BLOBS_SIDECARS_PER_SLOT); - } - @VisibleForTesting Map getValidatedPendingBlobsForSlot(final UInt64 slot) { diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 42b9851356e..ede5200ce41 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -13,7 +13,6 @@ package tech.pegasys.teku.statetransition.blobs; -import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -28,7 +27,8 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecVersion; import tech.pegasys.teku.spec.TestSpecFactory; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.statetransition.validation.BlobSidecarValidator; @@ -57,53 +57,25 @@ public class BlobSidecarManagerTest { @BeforeEach void setUp() { - when(storageUpdateChannel.onBlobsSidecar(any())).thenReturn(SafeFuture.COMPLETE); + when(storageUpdateChannel.onBlobSidecar(any())).thenReturn(SafeFuture.COMPLETE); + when(storageUpdateChannel.onNoBlobsSlot(any())).thenReturn(SafeFuture.COMPLETE); when(mockedSpec.atSlot(any())).thenReturn(mockedSpecVersion); when(mockedSpecVersion.miscHelpers()).thenReturn(mockedMiscHelpers); when(mockedSpec.getForkSchedule()).thenReturn(mockedForkSchedule); } @Test - void shouldStoreUnconfirmedValidatedBlobsSidecar() { - final BlobsSidecar blobsSidecar = dataStructureUtil.randomBlobsSidecar(); - blobSidecarManager.storeUnconfirmedValidatedBlobsSidecar(blobsSidecar); - - verify(storageUpdateChannel).onBlobsSidecar(blobsSidecar); - } - - @Test - void shouldStoreUnconfirmedBlobsSidecar() { - final BlobsSidecar blobsSidecar = dataStructureUtil.randomBlobsSidecar(); - blobSidecarManager.storeUnconfirmedBlobsSidecar(blobsSidecar); - - verify(storageUpdateChannel).onBlobsSidecar(blobsSidecar); + void shouldStoreBlobSidecar() { + final BlobSidecar blobSidecar = dataStructureUtil.randomBlobSidecar(); + blobSidecarManager.storeBlobSidecar(blobSidecar); + verify(storageUpdateChannel).onBlobSidecar(blobSidecar); } @Test - void shouldDiscardCachedValidatedBlobsOnSlot() { - final Bytes32 blockRoot = dataStructureUtil.randomBytes32(); - - final BlobsSidecar blobs1 = dataStructureUtil.randomBlobsSidecar(blockRoot, UInt64.ONE); - final BlobsSidecar blobs2 = dataStructureUtil.randomBlobsSidecar(blockRoot, UInt64.valueOf(2)); - - blobSidecarManager.storeUnconfirmedValidatedBlobsSidecar(blobs1); - blobSidecarManager.storeUnconfirmedValidatedBlobsSidecar(blobs2); - - assertThat(blobSidecarManager.getValidatedPendingBlobsForSlot(UInt64.ONE)) - .containsEntry(blockRoot, blobs1); - assertThat(blobSidecarManager.getValidatedPendingBlobsForSlot(UInt64.valueOf(2))) - .containsEntry(blockRoot, blobs2); - - blobSidecarManager.onSlot(UInt64.valueOf(2)); - - assertThat(blobSidecarManager.getValidatedPendingBlobsForSlot(UInt64.ONE)) - .containsEntry(blockRoot, blobs1); - assertThat(blobSidecarManager.getValidatedPendingBlobsForSlot(UInt64.valueOf(2))) - .containsEntry(blockRoot, blobs2); - - blobSidecarManager.onSlot(UInt64.valueOf(4)); - - assertThat(blobSidecarManager.getValidatedPendingBlobsForSlot(UInt64.ONE)).isEmpty(); - assertThat(blobSidecarManager.getValidatedPendingBlobsForSlot(UInt64.valueOf(2))).isEmpty(); + void shouldStoreNoBlobsSlot() { + final SlotAndBlockRoot noBlobsSlotAndRoot = + new SlotAndBlockRoot(UInt64.valueOf(3), Bytes32.ZERO); + blobSidecarManager.storeNoBlobsSlot(noBlobsSlotAndRoot); + verify(storageUpdateChannel).onNoBlobsSlot(noBlobsSlotAndRoot); } } diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java index 86e6d53d0b9..3dd3a0ef51c 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java @@ -27,7 +27,6 @@ import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.storage.api.CombinedStorageChannel; import tech.pegasys.teku.storage.api.Eth1DepositStorageChannel; -import tech.pegasys.teku.storage.api.FinalizedCheckpointChannel; import tech.pegasys.teku.storage.api.VoteUpdateChannel; import tech.pegasys.teku.storage.server.BatchingVoteUpdateChannel; import tech.pegasys.teku.storage.server.ChainStorage; @@ -37,7 +36,7 @@ import tech.pegasys.teku.storage.server.RetryingStorageUpdateChannel; import tech.pegasys.teku.storage.server.StorageConfiguration; import tech.pegasys.teku.storage.server.VersionedDatabaseFactory; -import tech.pegasys.teku.storage.server.pruner.BlobsSidecarPruner; +import tech.pegasys.teku.storage.server.pruner.BlobSidecarPruner; import tech.pegasys.teku.storage.server.pruner.BlockPruner; public class StorageService extends Service implements StorageServiceFacade { @@ -47,7 +46,7 @@ public class StorageService extends Service implements StorageServiceFacade { private volatile Database database; private volatile BatchingVoteUpdateChannel batchingVoteUpdateChannel; private volatile Optional blockPruner = Optional.empty(); - private volatile Optional blobsPruner = Optional.empty(); + private volatile Optional blobsPruner = Optional.empty(); private final boolean depositSnapshotStorageEnabled; private final boolean blobsSidecarStorageCountersEnabled; @@ -93,7 +92,7 @@ protected SafeFuture doStart() { if (config.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) { blobsPruner = Optional.of( - new BlobsSidecarPruner( + new BlobSidecarPruner( config.getSpec(), database, serviceConfig.getMetricsSystem(), @@ -130,8 +129,6 @@ protected SafeFuture doStart() { .subscribe(Eth1DepositStorageChannel.class, depositStorage) .subscribe(Eth1EventsChannel.class, depositStorage) .subscribe(VoteUpdateChannel.class, batchingVoteUpdateChannel); - blobsPruner.ifPresent( - pruner -> eventChannels.subscribe(FinalizedCheckpointChannel.class, pruner)); }) .thenCompose( __ -> @@ -141,7 +138,7 @@ protected SafeFuture doStart() { .thenCompose( __ -> blobsPruner - .map(BlobsSidecarPruner::start) + .map(BlobSidecarPruner::start) .orElseGet(() -> SafeFuture.completedFuture(null))); } diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java index bf4dfb60eb5..dfc0a5351e9 100644 --- a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java +++ b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java @@ -23,7 +23,6 @@ import tech.pegasys.teku.infrastructure.events.ChannelInterface; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -87,6 +86,4 @@ SafeFuture> getHotStateAndBlockSummaryByBlockRoot SafeFuture> getBlobSidecarKeys( UInt64 startSlot, UInt64 endSlot, UInt64 limit); - - SafeFuture> getBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); } diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageUpdateChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageUpdateChannel.java index 2eb197a1dca..2fd226ca772 100644 --- a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageUpdateChannel.java +++ b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageUpdateChannel.java @@ -22,7 +22,6 @@ import tech.pegasys.teku.infrastructure.events.ChannelInterface; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; @@ -48,11 +47,7 @@ SafeFuture onFinalizedBlocks( SafeFuture onBlobSidecar(BlobSidecar blobSidecar); - SafeFuture onBlobsSidecar(BlobsSidecar blobsSidecar); - SafeFuture onBlobSidecarsRemoval(UInt64 slot); - SafeFuture onBlobsSidecarRemoval(SlotAndBlockRoot blobsSidecarKey); - void onChainInitialized(AnchorPoint initialAnchor); } diff --git a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java index c06a2e9922a..dcf9af0659a 100644 --- a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java +++ b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java @@ -35,9 +35,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -63,7 +65,6 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -192,151 +193,133 @@ public void verifyBlobsLifecycle(final DatabaseContext context) throws IOExcepti initialize(context); // no blobs, no early slot - assertThat(database.getEarliestBlobsSidecarSlot()).isEmpty(); - - final BlobsSidecar blobsSidecar1 = - dataStructureUtil.randomBlobsSidecar(dataStructureUtil.randomBytes32(), UInt64.valueOf(1)); - final BlobsSidecar blobsSidecar2 = - dataStructureUtil.randomBlobsSidecar(Bytes32.ZERO, UInt64.valueOf(2)); - final BlobsSidecar blobsSidecar2bis = - dataStructureUtil.randomBlobsSidecar(Bytes32.fromHexString("0x01"), UInt64.valueOf(2)); - final BlobsSidecar blobsSidecar3 = - dataStructureUtil.randomBlobsSidecar(dataStructureUtil.randomBytes32(), UInt64.valueOf(3)); - final BlobsSidecar blobsSidecar4 = - dataStructureUtil.randomBlobsSidecar(dataStructureUtil.randomBytes32(), UInt64.valueOf(4)); - final BlobsSidecar blobsSidecarNotAdded = dataStructureUtil.randomBlobsSidecar(); + assertThat(database.getEarliestBlobSidecarSlot()).isEmpty(); + + final BlobSidecar blobSidecar1 = + dataStructureUtil.randomBlobSidecar( + UInt64.valueOf(1), dataStructureUtil.randomBytes32(), UInt64.valueOf(0)); + final BlobSidecar blobSidecar2 = + dataStructureUtil.randomBlobSidecar(UInt64.valueOf(2), Bytes32.ZERO, UInt64.valueOf(0)); + final BlobSidecar blobSidecar2bis = + dataStructureUtil.randomBlobSidecar( + UInt64.valueOf(2), Bytes32.fromHexString("0x01"), UInt64.valueOf(1)); + final BlobSidecar blobSidecar3 = + dataStructureUtil.randomBlobSidecar( + UInt64.valueOf(3), dataStructureUtil.randomBytes32(), UInt64.valueOf(0)); + final SlotAndBlockRoot slotAndBlockRootNoBlobs = + new SlotAndBlockRoot(UInt64.valueOf(4), dataStructureUtil.randomBytes32()); + final BlobSidecar blobSidecar5 = + dataStructureUtil.randomBlobSidecar( + UInt64.valueOf(5), dataStructureUtil.randomBytes32(), UInt64.valueOf(0)); + final BlobSidecar blobSidecarNotAdded = dataStructureUtil.randomBlobSidecar(); // add blobs out of order - database.storeUnconfirmedBlobsSidecar(blobsSidecar2); - database.storeUnconfirmedBlobsSidecar(blobsSidecar1); - database.storeUnconfirmedBlobsSidecar(blobsSidecar2bis); - database.storeUnconfirmedBlobsSidecar(blobsSidecar4); - database.storeUnconfirmedBlobsSidecar(blobsSidecar3); + database.storeBlobSidecar(blobSidecar2); + database.storeBlobSidecar(blobSidecar1); + database.storeBlobSidecar(blobSidecar2bis); + database.storeBlobSidecar(blobSidecar5); + database.storeBlobSidecar(blobSidecar3); + database.storeNoBlobsSlot(slotAndBlockRootNoBlobs); - assertThat(database.getBlobsSidecarColumnCounts()) - .containsExactlyInAnyOrderEntriesOf( - Map.of( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", 5L, - "BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", 5L)); - - assertThat(database.getEarliestBlobsSidecarSlot()).contains(ONE); + assertThat(database.getBlobSidecarColumnCount()).isEqualTo(6L); + assertThat(database.getEarliestBlobSidecarSlot()).contains(ONE); // all added blobs must be there - List.of(blobsSidecar1, blobsSidecar2, blobsSidecar2bis, blobsSidecar3, blobsSidecar4) + List.of(blobSidecar1, blobSidecar2, blobSidecar2bis, blobSidecar3, blobSidecar5) .forEach( blobsSidecar -> - assertThat(database.getBlobsSidecar(blobsSidecarToSlotAndBlockRoot(blobsSidecar))) + assertThat(database.getBlobSidecar(blobSidecarToKey(blobsSidecar))) .contains(blobsSidecar)); + // empty blobSidecar not visible directly, only with stream keys or getEarliestBlobSidecarSlot + assertThat( + database.getBlobSidecar( + SlotAndBlockRootAndBlobIndex.createNoBlobsKey(slotAndBlockRootNoBlobs))) + .isEmpty(); // non added blobs must not be there - assertThat(database.getBlobsSidecar(blobsSidecarToSlotAndBlockRoot(blobsSidecarNotAdded))) - .isEmpty(); + assertThat(database.getBlobSidecar(blobSidecarToKey(blobSidecarNotAdded))).isEmpty(); // all blobs must be streamed ordered by slot - assertBlobsStream(blobsSidecar1, blobsSidecar2, blobsSidecar2bis, blobsSidecar3, blobsSidecar4); - - // a subset of blobs must be streamed ordered by slot - assertBlobsStream( - UInt64.valueOf(2), UInt64.valueOf(3), blobsSidecar2, blobsSidecar2bis, blobsSidecar3); - - // all blobs must be unconfirmed - assertUnconfirmedBlobsStream( - blobsSidecarToSlotAndBlockRoot(blobsSidecar1), - blobsSidecarToSlotAndBlockRoot(blobsSidecar2), - blobsSidecarToSlotAndBlockRoot(blobsSidecar2bis), - blobsSidecarToSlotAndBlockRoot(blobsSidecar3), - blobsSidecarToSlotAndBlockRoot(blobsSidecar4)); - - database.confirmBlobsSidecar(blobsSidecarToSlotAndBlockRoot(blobsSidecar4)); - - assertThat(database.getBlobsSidecarColumnCounts()) - .containsExactlyInAnyOrderEntriesOf( - Map.of( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", 4L, - "BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", 5L)); - - // only 1 and 3 blobs must be unconfirmed - assertUnconfirmedBlobsStream( - blobsSidecarToSlotAndBlockRoot(blobsSidecar1), - blobsSidecarToSlotAndBlockRoot(blobsSidecar2), - blobsSidecarToSlotAndBlockRoot(blobsSidecar2bis), - blobsSidecarToSlotAndBlockRoot(blobsSidecar3)); - // we still have all blobs - assertBlobsStream(blobsSidecar1, blobsSidecar2, blobsSidecar2bis, blobsSidecar3, blobsSidecar4); - - // let's prune unconfirmed with limit to 1 - assertThat(database.pruneOldestUnconfirmedBlobsSidecars(UInt64.MAX_VALUE, 1)).isTrue(); - assertUnconfirmedBlobsStream( - blobsSidecarToSlotAndBlockRoot(blobsSidecar2), - blobsSidecarToSlotAndBlockRoot(blobsSidecar2bis), - blobsSidecarToSlotAndBlockRoot(blobsSidecar3)); - // we have all blobs except the first - assertBlobsStream(blobsSidecar2, blobsSidecar2bis, blobsSidecar3, blobsSidecar4); - - // let's prune unconfirmed up to slot 1 (nothing will be pruned) - assertThat(database.pruneOldestUnconfirmedBlobsSidecars(ONE, 10)).isFalse(); - assertUnconfirmedBlobsStream( - blobsSidecarToSlotAndBlockRoot(blobsSidecar2), - blobsSidecarToSlotAndBlockRoot(blobsSidecar2bis), - blobsSidecarToSlotAndBlockRoot(blobsSidecar3)); - // we still have all blobs except the first - assertBlobsStream(blobsSidecar2, blobsSidecar2bis, blobsSidecar3, blobsSidecar4); - - // let's prune all unconfirmed - assertThat(database.pruneOldestUnconfirmedBlobsSidecars(UInt64.valueOf(3), 10)).isFalse(); - assertUnconfirmedBlobsStream(); - // we have blobsSidecar4 - assertBlobsStream(blobsSidecar4); - - // let's prune all up to a too old slot (nothing will be pruned) - assertThat(database.pruneOldestBlobsSidecar(UInt64.valueOf(3), 10)).isFalse(); - assertBlobsStream(blobsSidecar4); - - // let's prune all up slot 4 - assertThat(database.pruneOldestBlobsSidecar(UInt64.valueOf(4), 1)).isTrue(); - // all empty now - assertUnconfirmedBlobsStream(); - assertBlobsStream(); - - assertThat(database.getBlobsSidecarColumnCounts()) - .containsExactlyInAnyOrderEntriesOf( - Map.of( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", 0L, - "BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", 0L)); - } - - @TestTemplate - void pruneOldestBlobsSidecar_shouldPruneUnconfirmedBlobsToo(final DatabaseContext context) - throws IOException { - initialize(context); - - // no blobs, no early slot - assertThat(database.getEarliestBlobsSidecarSlot()).isEmpty(); - - final BlobsSidecar blobsSidecar1 = - dataStructureUtil.randomBlobsSidecar(dataStructureUtil.randomBytes32(), UInt64.valueOf(1)); - final BlobsSidecar blobsSidecar2 = - dataStructureUtil.randomBlobsSidecar(dataStructureUtil.randomBytes32(), UInt64.valueOf(2)); - final BlobsSidecar blobsSidecar3 = - dataStructureUtil.randomBlobsSidecar(dataStructureUtil.randomBytes32(), UInt64.valueOf(3)); - final BlobsSidecar blobsSidecar4 = - dataStructureUtil.randomBlobsSidecar(dataStructureUtil.randomBytes32(), UInt64.valueOf(4)); - - // add unconfirmed blobs - database.storeUnconfirmedBlobsSidecar(blobsSidecar2); - database.storeUnconfirmedBlobsSidecar(blobsSidecar1); - database.storeUnconfirmedBlobsSidecar(blobsSidecar4); - database.storeUnconfirmedBlobsSidecar(blobsSidecar3); - - assertThat(database.pruneOldestBlobsSidecar(UInt64.MAX_VALUE, 2)).isTrue(); - assertUnconfirmedBlobsStream( - blobsSidecarToSlotAndBlockRoot(blobsSidecar3), - blobsSidecarToSlotAndBlockRoot(blobsSidecar4)); - assertBlobsStream(blobsSidecar3, blobsSidecar4); + assertBlobSidecarKeys( + blobSidecar1.getSlot(), + blobSidecar5.getSlot(), + blobSidecarToKey(blobSidecar1), + blobSidecarToKey(blobSidecar2), + blobSidecarToKey(blobSidecar2bis), + blobSidecarToKey(blobSidecar3), + SlotAndBlockRootAndBlobIndex.createNoBlobsKey(slotAndBlockRootNoBlobs), + blobSidecarToKey(blobSidecar5)); + assertBlobSidecars( + Map.of( + blobSidecar1.getSlot(), + List.of(blobSidecar1), + blobSidecar2.getSlot(), + List.of(blobSidecar2, blobSidecar2bis), + blobSidecar3.getSlot(), + List.of(blobSidecar3), + slotAndBlockRootNoBlobs.getSlot(), + Collections.emptyList(), + blobSidecar5.getSlot(), + List.of(blobSidecar5))); + + // let's prune with limit to 1 + assertThat(database.pruneOldestBlobSidecars(UInt64.MAX_VALUE, 1)).isTrue(); + assertBlobSidecarKeys( + blobSidecar2.getSlot(), + blobSidecar5.getSlot(), + blobSidecarToKey(blobSidecar2), + blobSidecarToKey(blobSidecar2bis), + blobSidecarToKey(blobSidecar3), + SlotAndBlockRootAndBlobIndex.createNoBlobsKey(slotAndBlockRootNoBlobs), + blobSidecarToKey(blobSidecar5)); + assertBlobSidecars( + Map.of( + blobSidecar2.getSlot(), List.of(blobSidecar2, blobSidecar2bis), + blobSidecar3.getSlot(), List.of(blobSidecar3), + slotAndBlockRootNoBlobs.getSlot(), Collections.emptyList(), + blobSidecar5.getSlot(), List.of(blobSidecar5))); + assertThat(database.getEarliestBlobSidecarSlot()).contains(UInt64.valueOf(2)); + assertThat(database.getBlobSidecarColumnCount()).isEqualTo(5L); + + // let's prune up to slot 1 (nothing will be pruned) + assertThat(database.pruneOldestBlobSidecars(ONE, 10)).isFalse(); + assertBlobSidecarKeys( + blobSidecar2.getSlot(), + blobSidecar5.getSlot(), + blobSidecarToKey(blobSidecar2), + blobSidecarToKey(blobSidecar2bis), + blobSidecarToKey(blobSidecar3), + SlotAndBlockRootAndBlobIndex.createNoBlobsKey(slotAndBlockRootNoBlobs), + blobSidecarToKey(blobSidecar5)); + assertBlobSidecars( + Map.of( + blobSidecar2.getSlot(), List.of(blobSidecar2, blobSidecar2bis), + blobSidecar3.getSlot(), List.of(blobSidecar3), + slotAndBlockRootNoBlobs.getSlot(), Collections.emptyList(), + blobSidecar5.getSlot(), List.of(blobSidecar5))); + assertThat(database.getEarliestBlobSidecarSlot()).contains(UInt64.valueOf(2)); + assertThat(database.getBlobSidecarColumnCount()).isEqualTo(5L); + + // let's prune all from slot 4 excluded + assertThat(database.pruneOldestBlobSidecars(UInt64.valueOf(3), 10)).isFalse(); + assertBlobSidecarKeys( + slotAndBlockRootNoBlobs.getSlot(), + blobSidecar5.getSlot(), + SlotAndBlockRootAndBlobIndex.createNoBlobsKey(slotAndBlockRootNoBlobs), + blobSidecarToKey(blobSidecar5)); + assertBlobSidecars( + Map.of( + slotAndBlockRootNoBlobs.getSlot(), Collections.emptyList(), + blobSidecar5.getSlot(), List.of(blobSidecar5))); + assertThat(database.getEarliestBlobSidecarSlot()).contains(UInt64.valueOf(4)); + assertThat(database.getBlobSidecarColumnCount()).isEqualTo(2L); - assertThat(database.pruneOldestBlobsSidecar(UInt64.MAX_VALUE, 100)).isFalse(); - assertUnconfirmedBlobsStream(); - assertBlobsStream(); + // let's prune all + assertThat(database.pruneOldestBlobSidecars(UInt64.valueOf(5), 2)).isTrue(); + // all empty now + assertBlobSidecarKeys(ZERO, UInt64.valueOf(10)); + assertThat(database.getEarliestBlobSidecarSlot()).isEmpty(); + assertThat(database.getBlobSidecarColumnCount()).isEqualTo(0L); } @TestTemplate @@ -2216,11 +2199,17 @@ private void assertBlobsSidecarAvailabilityExceptPruned( } }); prunedBlocksSidecars.forEach( - block -> - assertThat( - database.getBlobsSidecar( - new SlotAndBlockRoot(block.getSlot(), block.getRoot()))) - .isEmpty()); + block -> { + assertThat( + database.getBlobSidecar( + new SlotAndBlockRootAndBlobIndex(block.getSlot(), block.getRoot(), ZERO))) + .isEmpty(); + assertThat( + database.getBlobSidecar( + SlotAndBlockRootAndBlobIndex.createNoBlobsKey( + block.getSlot(), block.getRoot()))) + .isEmpty(); + }); } private void addBlocks(final SignedBlockAndState... blocks) { @@ -2345,29 +2334,39 @@ private void setDefaultStorage(final StorageSystem storageSystem) { storageSystems.add(storageSystem); } - private void assertUnconfirmedBlobsStream(SlotAndBlockRoot... slotAndBlockRoots) { - assertUnconfirmedBlobsStream(ZERO, UInt64.MAX_VALUE, slotAndBlockRoots); - } - - private void assertUnconfirmedBlobsStream( - UInt64 startSlot, UInt64 endSlot, SlotAndBlockRoot... slotAndBlockRoots) { - try (Stream blobsSidecarStream = - database.streamUnconfirmedBlobsSidecars(startSlot, endSlot)) { - final List allSlotAndBlockRoots = blobsSidecarStream.collect(toList()); - assertThat(allSlotAndBlockRoots).containsExactly(slotAndBlockRoots); + private void assertBlobSidecarKeys( + final UInt64 from, final UInt64 to, SlotAndBlockRootAndBlobIndex... keys) { + try (final Stream blobsSidecarStream = + database.streamBlobSidecarKeys(from, to)) { + final List keysFromDb = blobsSidecarStream.collect(toList()); + assertThat(keysFromDb).isEqualTo(Arrays.asList(keys)); } } - private void assertBlobsStream(BlobsSidecar... blobsSidecars) { - assertBlobsStream(ZERO, UInt64.MAX_VALUE, blobsSidecars); - } + private void assertBlobSidecars(final Map> blobSidecars) { + final List slots = blobSidecars.keySet().stream().sorted().collect(toList()); + if (slots.isEmpty()) { + return; + } - private void assertBlobsStream(UInt64 startSlot, UInt64 endSlot, BlobsSidecar... blobsSidecars) { - try (Stream blobsSidecarStream = - database.streamBlobsSidecars(startSlot, endSlot)) { - final List allBlobs = blobsSidecarStream.collect(toList()); - assertThat(allBlobs).containsExactly(blobsSidecars); + final Map> blobSidecarsDb = new HashMap<>(); + try (final Stream blobsSidecarStream = + database.streamBlobSidecarKeys(slots.get(0), slots.get(slots.size() - 1))) { + + for (final Iterator iterator = blobsSidecarStream.iterator(); + iterator.hasNext(); ) { + final SlotAndBlockRootAndBlobIndex key = iterator.next(); + if (key.isNoBlobsKey()) { + blobSidecarsDb.put(key.getSlot(), Collections.emptyList()); + } else { + final List currentSlotBlobs = + blobSidecarsDb.computeIfAbsent(key.getSlot(), __ -> new ArrayList<>()); + currentSlotBlobs.add(database.getBlobSidecar(key).orElseThrow()); + } + } } + + assertThat(blobSidecarsDb).isEqualTo(blobSidecars); } public static class CreateForkChainResult { @@ -2388,8 +2387,8 @@ public UInt64 getFirstHotBlockSlot() { } } - private static SlotAndBlockRoot blobsSidecarToSlotAndBlockRoot(final BlobsSidecar blobsSidecar) { - return new SlotAndBlockRoot( - blobsSidecar.getBeaconBlockSlot(), blobsSidecar.getBeaconBlockRoot()); + private static SlotAndBlockRootAndBlobIndex blobSidecarToKey(final BlobSidecar blobSidecar) { + return new SlotAndBlockRootAndBlobIndex( + blobSidecar.getSlot(), blobSidecar.getBlockRoot(), blobSidecar.getIndex()); } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java b/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java index 3972bf791a7..049f7769c79 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java @@ -28,7 +28,6 @@ import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; -import tech.pegasys.teku.dataproviders.lookup.BlobsSidecarProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.AsyncRunner; @@ -74,7 +73,6 @@ public abstract class RecentChainData implements StoreUpdateHandler { private static final Logger LOG = LogManager.getLogger(); private final BlockProvider blockProvider; - private final BlobsSidecarProvider blobsSidecarProvider; private final StateAndBlockSummaryProvider stateProvider; protected final FinalizedCheckpointChannel finalizedCheckpointChannel; protected final StorageUpdateChannel storageUpdateChannel; @@ -102,7 +100,6 @@ public abstract class RecentChainData implements StoreUpdateHandler { final MetricsSystem metricsSystem, final StoreConfig storeConfig, final BlockProvider blockProvider, - final BlobsSidecarProvider blobsSidecarProvider, final StateAndBlockSummaryProvider stateProvider, final StorageUpdateChannel storageUpdateChannel, final VoteUpdateChannel voteUpdateChannel, @@ -113,7 +110,6 @@ public abstract class RecentChainData implements StoreUpdateHandler { this.metricsSystem = metricsSystem; this.storeConfig = storeConfig; this.blockProvider = blockProvider; - this.blobsSidecarProvider = blobsSidecarProvider; this.stateProvider = stateProvider; this.voteUpdateChannel = voteUpdateChannel; this.chainHeadChannel = chainHeadChannel; @@ -148,7 +144,6 @@ public void initializeFromAnchorPoint(final AnchorPoint anchorPoint, final UInt6 .metricsSystem(metricsSystem) .specProvider(spec) .blockProvider(blockProvider) - .blobsSidecarProvider(blobsSidecarProvider) .stateProvider(stateProvider) .storeConfig(storeConfig) .build(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java index ff7105c87c6..c57ff385044 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java @@ -22,7 +22,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hyperledger.besu.plugin.services.MetricsSystem; -import tech.pegasys.teku.dataproviders.lookup.BlobsSidecarProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.AsyncRunner; @@ -42,7 +41,6 @@ public class StorageBackedRecentChainData extends RecentChainData { private static final Logger LOG = LogManager.getLogger(); private final BlockProvider blockProvider; - private final BlobsSidecarProvider blobsSidecarProvider; private final StateAndBlockSummaryProvider stateProvider; private final StorageQueryChannel storageQueryChannel; private final StoreConfig storeConfig; @@ -62,7 +60,6 @@ public StorageBackedRecentChainData( metricsSystem, storeConfig, storageQueryChannel::getHotBlocksByRoot, - storageQueryChannel::getBlobsSidecar, storageQueryChannel::getHotStateAndBlockSummaryByBlockRoot, storageUpdateChannel, voteUpdateChannel, @@ -73,7 +70,6 @@ public StorageBackedRecentChainData( this.storageQueryChannel = storageQueryChannel; this.blockProvider = storageQueryChannel::getHotBlocksByRoot; this.stateProvider = storageQueryChannel::getHotStateAndBlockSummaryByBlockRoot; - this.blobsSidecarProvider = storageQueryChannel::getBlobsSidecar; } public static SafeFuture create( @@ -156,7 +152,6 @@ private SafeFuture processStoreFuture( .onDiskStoreData(data) .asyncRunner(asyncRunner) .blockProvider(blockProvider) - .blobsSidecarProvider(blobsSidecarProvider) .stateProvider(stateProvider) .storeConfig(storeConfig) .build(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java index 69528fe56f5..1c7a36cfdc6 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java @@ -27,7 +27,6 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.config.SpecConfig; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockHeader; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockSummary; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; @@ -168,21 +167,11 @@ public SafeFuture onBlobSidecar(final BlobSidecar blobSidecar) { return SafeFuture.fromRunnable(() -> database.storeBlobSidecar(blobSidecar)); } - @Override - public SafeFuture onBlobsSidecar(final BlobsSidecar blobsSidecar) { - return SafeFuture.fromRunnable(() -> database.storeUnconfirmedBlobsSidecar(blobsSidecar)); - } - @Override public SafeFuture onBlobSidecarsRemoval(final UInt64 slot) { return SafeFuture.fromRunnable(() -> database.removeBlobSidecars(slot)); } - @Override - public SafeFuture onBlobsSidecarRemoval(final SlotAndBlockRoot blobsSidecarKey) { - return SafeFuture.fromRunnable(() -> database.removeBlobsSidecar(blobsSidecarKey)); - } - @Override public SafeFuture> getLatestFinalizedBlockAtSlot(final UInt64 slot) { return SafeFuture.of(() -> database.getLatestFinalizedBlockAtSlot(slot)); @@ -313,10 +302,4 @@ public SafeFuture> getBlobSidecarKeys( return result; }); } - - @Override - public SafeFuture> getBlobsSidecar( - final SlotAndBlockRoot slotAndBlockRoot) { - return SafeFuture.of(() -> database.getBlobsSidecar(slotAndBlockRoot)); - } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java b/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java index b691c4d7f6c..90a023ae985 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java @@ -24,7 +24,6 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -115,21 +114,11 @@ public SafeFuture onBlobSidecar(final BlobSidecar blobSidecar) { return updateDelegate.onBlobSidecar(blobSidecar); } - @Override - public SafeFuture onBlobsSidecar(final BlobsSidecar blobsSidecar) { - return updateDelegate.onBlobsSidecar(blobsSidecar); - } - @Override public SafeFuture onBlobSidecarsRemoval(final UInt64 slot) { return updateDelegate.onBlobSidecarsRemoval(slot); } - @Override - public SafeFuture onBlobsSidecarRemoval(final SlotAndBlockRoot blobsSidecarKey) { - return updateDelegate.onBlobsSidecarRemoval(blobsSidecarKey); - } - @Override public SafeFuture> onStoreRequest() { return asyncRunner.runAsync(queryDelegate::onStoreRequest); @@ -240,10 +229,4 @@ public SafeFuture> getBlobSidecarKeys( UInt64 startSlot, UInt64 endSlot, UInt64 limit) { return asyncRunner.runAsync(() -> queryDelegate.getBlobSidecarKeys(startSlot, endSlot, limit)); } - - @Override - public SafeFuture> getBlobsSidecar( - final SlotAndBlockRoot slotAndBlockRoot) { - return asyncRunner.runAsync(() -> queryDelegate.getBlobsSidecar(slotAndBlockRoot)); - } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java index 7c21c3158cb..5e5be552721 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java @@ -27,7 +27,6 @@ import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BlockCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -64,60 +63,28 @@ void storeFinalizedBlocks( void storeNoBlobsSlot(SlotAndBlockRoot slotAndBlockRoot); - void storeUnconfirmedBlobsSidecar(BlobsSidecar blobsSidecar); - - void confirmBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); - Optional getBlobSidecar(SlotAndBlockRootAndBlobIndex key); - Optional getBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); - void removeBlobSidecars(UInt64 slot); - void removeBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); - - /** - * this prune method will delete BlobsSidecars (including the unconfirmed ones) starting from the - * oldest BlobsSidecars (by slot) up to BlobsSidecars at {@code lastSlotToPrune} (inclusive). The - * pruning process will be limited to maximum {@code pruneLimit} BlobsSidecars - * - * @param lastSlotToPrune - * @param pruneLimit - * @return true if number of pruned blobs reached the pruneLimit, false otherwise - */ - boolean pruneOldestBlobsSidecar(UInt64 lastSlotToPrune, int pruneLimit); - /** - * this prune method will delete unconfirmed BlobsSidecars starting from the oldest BlobsSidecars - * (by slot) up to BlobsSidecars at {@code lastSlotToPrune} (inclusive). The pruning process will - * be limited to maximum {@code pruneLimit} BlobsSidecars + * This prune method will delete BlobSidecars starting from the oldest BlobSidecars (by slot) up + * to BlobSidecars at {@code lastSlotToPrune} (inclusive). The pruning process will be stopped if + * {@code pruneLimit} reached completing the current slot. So, if pruneLimit happened at index#0 + * BlobSidecar and there are 2 BlobSidecars in a slot, index#1 will be removed too. Main purpose + * of pruneLimit is to softly cap DB operation time. * - * @param lastSlotToPrune - * @param pruneLimit + * @param lastSlotToPrune inclusive, not reached if limit happens first + * @param pruneLimit soft BlobSidecars (not slots) limit * @return true if number of pruned blobs reached the pruneLimit, false otherwise */ - boolean pruneOldestUnconfirmedBlobsSidecars(UInt64 lastSlotToPrune, int pruneLimit); + boolean pruneOldestBlobSidecars(UInt64 lastSlotToPrune, int pruneLimit); @MustBeClosed Stream streamBlobSidecarKeys(UInt64 startSlot, UInt64 endSlot); - @MustBeClosed - Stream streamBlobsSidecars(UInt64 startSlot, UInt64 endSlot); - - @MustBeClosed - Stream> streamBlobsSidecarsAsSsz( - UInt64 startSlot, UInt64 endSlot); - - @MustBeClosed - Stream streamBlobsSidecarKeys(UInt64 startSlot, UInt64 endSlot); - - @MustBeClosed - Stream streamUnconfirmedBlobsSidecars(UInt64 startSlot, UInt64 endSlot); - Optional getEarliestBlobSidecarSlot(); - Optional getEarliestBlobsSidecarSlot(); - Optional createMemoryStore(); WeakSubjectivityState getWeakSubjectivityState(); @@ -221,7 +188,7 @@ Stream> streamBlobsSidecarsAsSsz( Map getColumnCounts(); - Map getBlobsSidecarColumnCounts(); + long getBlobSidecarColumnCount(); void migrate(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/RetryingStorageUpdateChannel.java b/storage/src/main/java/tech/pegasys/teku/storage/server/RetryingStorageUpdateChannel.java index 695d7d318d2..d4ccb1c3d18 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/RetryingStorageUpdateChannel.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/RetryingStorageUpdateChannel.java @@ -29,7 +29,6 @@ import tech.pegasys.teku.infrastructure.time.TimeProvider; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; @@ -137,21 +136,11 @@ public SafeFuture onBlobSidecar(final BlobSidecar blobSidecar) { return retry(() -> delegate.onBlobSidecar(blobSidecar)); } - @Override - public SafeFuture onBlobsSidecar(final BlobsSidecar blobsSidecar) { - return retry(() -> delegate.onBlobsSidecar(blobsSidecar)); - } - @Override public SafeFuture onBlobSidecarsRemoval(final UInt64 slot) { return retry(() -> delegate.onBlobSidecarsRemoval(slot)); } - @Override - public SafeFuture onBlobsSidecarRemoval(final SlotAndBlockRoot blobsSidecarKey) { - return retry(() -> delegate.onBlobsSidecarRemoval(blobsSidecarKey)); - } - private SafeFuture retry(final Supplier> method) { final UInt64 startTime = timeProvider.getTimeInMillis(); while (!aborting.get()) { diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java index 55fd199a19b..85ae706fe3b 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java @@ -17,6 +17,7 @@ import static tech.pegasys.teku.infrastructure.logging.DbLogger.DB_LOGGER; import static tech.pegasys.teku.infrastructure.logging.StatusLogger.STATUS_LOG; import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ONE; +import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ZERO; import com.google.common.annotations.VisibleForTesting; import com.google.errorprone.annotations.MustBeClosed; @@ -48,7 +49,6 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockHeader; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockSummary; import tech.pegasys.teku.spec.datastructures.blocks.BlockAndCheckpoints; @@ -335,8 +335,8 @@ public Map getColumnCounts() { } @Override - public Map getBlobsSidecarColumnCounts() { - return dao.getBlobsSidecarColumnCounts(); + public long getBlobSidecarColumnCount() { + return dao.getBlobSidecarColumnCount(); } @Override @@ -773,36 +773,16 @@ public void storeNoBlobsSlot(final SlotAndBlockRoot slotAndBlockRoot) { } } - @Override - public void storeUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) { - try (final FinalizedUpdater updater = finalizedUpdater()) { - updater.addBlobsSidecar(blobsSidecar); - updater.addUnconfirmedBlobsSidecar(blobsSidecar); - updater.commit(); - } - } - - @Override - public void confirmBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - try (final FinalizedUpdater updater = finalizedUpdater()) { - updater.confirmBlobsSidecar(slotAndBlockRoot); - updater.commit(); - } - } - @Override public Optional getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { final Optional maybePayload = dao.getBlobSidecar(key); + if (maybePayload.filter(Bytes::isEmpty).isPresent()) { + // no BlobSidecars slot entry + return Optional.empty(); + } return maybePayload.map(payload -> spec.deserializeBlobSidecar(payload, key.getSlot())); } - @Override - public Optional getBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - final Optional maybePayload = dao.getBlobsSidecar(slotAndBlockRoot); - return maybePayload.map( - payload -> spec.deserializeBlobsSidecar(payload, slotAndBlockRoot.getSlot())); - } - @Override public void removeBlobSidecars(final UInt64 slot) { try (final FinalizedUpdater updater = finalizedUpdater(); @@ -813,49 +793,23 @@ public void removeBlobSidecars(final UInt64 slot) { } @Override - public void removeBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - try (final FinalizedUpdater updater = finalizedUpdater()) { - updater.removeBlobsSidecar(slotAndBlockRoot); - updater.confirmBlobsSidecar(slotAndBlockRoot); - updater.commit(); - } - } - - @Override - public boolean pruneOldestBlobsSidecar(final UInt64 lastSlotToPrune, final int pruneLimit) { - try (final Stream prunableBlobsKeys = - streamBlobsSidecarKeys(UInt64.ZERO, lastSlotToPrune); + public boolean pruneOldestBlobSidecars(final UInt64 lastSlotToPrune, final int pruneLimit) { + try (final Stream prunableBlobKeys = + streamBlobSidecarKeys(UInt64.ZERO, lastSlotToPrune); final FinalizedUpdater updater = finalizedUpdater()) { - final long pruned = - prunableBlobsKeys - .limit(pruneLimit) - .peek( - slotAndBlockRoot -> { - updater.removeBlobsSidecar(slotAndBlockRoot); - updater.confirmBlobsSidecar(slotAndBlockRoot); - }) - .count(); - updater.commit(); - - return pruned == pruneLimit; - } - } - - @Override - public boolean pruneOldestUnconfirmedBlobsSidecars( - final UInt64 lastSlotToPrune, final int pruneLimit) { - try (final Stream prunableUnconfirmed = - streamUnconfirmedBlobsSidecars(UInt64.ZERO, lastSlotToPrune); - final FinalizedUpdater updater = finalizedUpdater()) { - final long pruned = - prunableUnconfirmed - .limit(pruneLimit) - .peek( - slotAndBlockRoot -> { - updater.removeBlobsSidecar(slotAndBlockRoot); - updater.confirmBlobsSidecar(slotAndBlockRoot); - }) - .count(); + int remaining = pruneLimit; + int pruned = 0; + for (final Iterator it = prunableBlobKeys.iterator(); + it.hasNext(); ) { + --remaining; + final boolean finished = remaining < 0; + final SlotAndBlockRootAndBlobIndex key = it.next(); + if (finished && (key.isNoBlobsKey() || key.getBlobIndex().equals(ZERO))) { + break; + } + updater.removeBlobSidecar(key); + ++pruned; + } updater.commit(); return pruned == pruneLimit; @@ -869,48 +823,11 @@ public Stream streamBlobSidecarKeys( return dao.streamBlobSidecarKeys(startSlot, endSlot); } - @MustBeClosed - @Override - public Stream streamBlobsSidecars(final UInt64 startSlot, final UInt64 endSlot) { - return dao.streamBlobsSidecar(startSlot, endSlot) - .map( - slotAndBlockRootBytesEntry -> - spec.deserializeBlobsSidecar( - slotAndBlockRootBytesEntry.getValue(), - slotAndBlockRootBytesEntry.getKey().getSlot())); - } - - @MustBeClosed - @Override - public Stream> streamBlobsSidecarsAsSsz( - final UInt64 startSlot, final UInt64 endSlot) { - return dao.streamBlobsSidecar(startSlot, endSlot); - } - - @MustBeClosed - @Override - public Stream streamBlobsSidecarKeys( - final UInt64 startSlot, final UInt64 endSlot) { - return dao.streamBlobsSidecarKeys(startSlot, endSlot); - } - - @MustBeClosed - @Override - public Stream streamUnconfirmedBlobsSidecars( - final UInt64 startSlot, final UInt64 endSlot) { - return dao.streamUnconfirmedBlobsSidecar(startSlot, endSlot); - } - @Override public Optional getEarliestBlobSidecarSlot() { return dao.getEarliestBlobSidecarSlot(); } - @Override - public Optional getEarliestBlobsSidecarSlot() { - return dao.getEarliestBlobsSidecarSlot(); - } - @Override public void storeVotes(final Map votes) { try (final HotUpdater hotUpdater = hotUpdater()) { @@ -960,8 +877,7 @@ private UpdateResult doUpdate(final StorageUpdate update) { update.getOptimisticTransitionBlockRoot()); if (update.isBlobsSidecarEnabled()) { - updateBlobsSidecars( - update.getHotBlocks(), update.getFinalizedBlocks(), update.getDeletedHotBlocks()); + updateBlobSidecars(update.getFinalizedBlocks(), update.getDeletedHotBlocks()); } LOG.trace("Applying hot updates"); @@ -1044,26 +960,23 @@ private Optional updateFinalizedOptimisticTransi } } - private void updateBlobsSidecars( - final Map hotBlocks, + private void updateBlobSidecars( final Map finalizedBlocks, final Map deletedHotBlocks) { try (final FinalizedUpdater updater = finalizedUpdater()) { - LOG.trace("Confirming blobs sidecars for new hot blocks"); - hotBlocks.values().stream() - .map( - blockAndCheckpoints -> - new SlotAndBlockRoot( - blockAndCheckpoints.getSlot(), blockAndCheckpoints.getRoot())) - .forEach(updater::confirmBlobsSidecar); - final Set finalizedBlockRoots = finalizedBlocks.keySet(); - - LOG.trace("Removing blobs sidecars for deleted hot blocks"); - deletedHotBlocks.entrySet().stream() - .filter(entry -> !finalizedBlockRoots.contains(entry.getKey())) - .map(entry -> new SlotAndBlockRoot(entry.getValue(), entry.getKey())) - .forEach(updater::removeBlobsSidecar); + LOG.trace("Removing blob sidecars for deleted hot blocks"); + List dropSlots = + deletedHotBlocks.entrySet().stream() + .filter(entry -> !finalizedBlockRoots.contains(entry.getKey())) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + for (final UInt64 slot : dropSlots) { + try (final Stream slotAndBlockRootAndBlobIndexStream = + dao.streamBlobSidecarKeys(slot, slot)) { + slotAndBlockRootAndBlobIndexStream.forEach(updater::removeBlobSidecar); + } + } updater.commit(); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java index bc967cea4e3..de3a15b5432 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java @@ -21,7 +21,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; @@ -34,7 +33,6 @@ import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BlockAndCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.BlockCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; @@ -338,11 +336,6 @@ public Optional getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { new SlotAndBlockRootAndBlobIndex(key.getSlot(), key.getBlockRoot(), key.getBlobIndex())); } - @Override - public Optional getBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - return db.get(schema.getColumnBlobsSidecarBySlotAndBlockRoot(), slotAndBlockRoot); - } - @MustBeClosed @Override public Stream streamBlobSidecarKeys( @@ -353,49 +346,12 @@ public Stream streamBlobSidecarKeys( new SlotAndBlockRootAndBlobIndex(endSlot, MAX_BLOCK_ROOT, UInt64.MAX_VALUE)); } - @MustBeClosed - @Override - public Stream> streamBlobsSidecar( - final UInt64 startSlot, final UInt64 endSlot) { - return db.stream( - schema.getColumnBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot(startSlot, MIN_BLOCK_ROOT), - new SlotAndBlockRoot(endSlot, MAX_BLOCK_ROOT)) - .map(entry -> entry); - } - - @MustBeClosed - @Override - public Stream streamBlobsSidecarKeys( - final UInt64 startSlot, final UInt64 endSlot) { - return db.streamKeys( - schema.getColumnBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot(startSlot, MIN_BLOCK_ROOT), - new SlotAndBlockRoot(endSlot, MAX_BLOCK_ROOT)); - } - - @MustBeClosed - @Override - public Stream streamUnconfirmedBlobsSidecar( - final UInt64 startSlot, final UInt64 endSlot) { - return db.streamKeys( - schema.getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot(startSlot, MIN_BLOCK_ROOT), - new SlotAndBlockRoot(endSlot, MAX_BLOCK_ROOT)); - } - @Override public Optional getEarliestBlobSidecarSlot() { return db.getFirstEntry(schema.getColumnBlobSidecarBySlotRootBlobIndex()) .map(entry -> entry.getKey().getSlot()); } - @Override - public Optional getEarliestBlobsSidecarSlot() { - return db.getFirstEntry(schema.getColumnBlobsSidecarBySlotAndBlockRoot()) - .map(entry -> entry.getKey().getSlot()); - } - @Override public Map getColumnCounts() { final Map columnCounts = new LinkedHashMap<>(); @@ -404,19 +360,10 @@ public Map getColumnCounts() { } @Override - public Map getBlobsSidecarColumnCounts() { - final Map columnCounts = new LinkedHashMap<>(); - - schema.getColumnMap().entrySet().stream() - .filter( - stringKvStoreColumnEntry -> - List.of( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", - "BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT") - .contains(stringKvStoreColumnEntry.getKey())) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)) - .forEach((k, v) -> columnCounts.put(k, db.size(v))); - return columnCounts; + public long getBlobSidecarColumnCount() { + final KvStoreColumn column = + schema.getColumnMap().get("BLOB_SIDECAR_BY_SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX"); + return db.size(column); } @Override @@ -712,38 +659,9 @@ public void addNoBlobsSlot(final SlotAndBlockRoot slotAndBlockRoot) { Bytes.EMPTY); } - @Override - public void addBlobsSidecar(final BlobsSidecar blobsSidecar) { - transaction.put( - schema.getColumnBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot( - blobsSidecar.getBeaconBlockSlot(), blobsSidecar.getBeaconBlockRoot()), - blobsSidecar.sszSerialize()); - } - - @Override - public void addUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) { - transaction.put( - schema.getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot( - blobsSidecar.getBeaconBlockSlot(), blobsSidecar.getBeaconBlockRoot()), - null); - } - @Override public void removeBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { transaction.delete(schema.getColumnBlobSidecarBySlotRootBlobIndex(), key); } - - @Override - public void removeBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - transaction.delete(schema.getColumnBlobsSidecarBySlotAndBlockRoot(), slotAndBlockRoot); - } - - @Override - public void confirmBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - transaction.delete( - schema.getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(), slotAndBlockRoot); - } } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java index facb0b4631f..2e87699399f 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java @@ -16,7 +16,6 @@ import com.google.errorprone.annotations.MustBeClosed; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; @@ -28,7 +27,6 @@ import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BlockAndCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.BlockCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; @@ -131,27 +129,14 @@ public interface KvStoreCombinedDao extends AutoCloseable { Optional getBlobSidecar(SlotAndBlockRootAndBlobIndex key); - Optional getBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); - @MustBeClosed Stream streamBlobSidecarKeys(UInt64 startSlot, UInt64 endSlot); - @MustBeClosed - Stream> streamBlobsSidecar(UInt64 startSlot, UInt64 endSlot); - - @MustBeClosed - Stream streamBlobsSidecarKeys(UInt64 startSlot, UInt64 endSlot); - - @MustBeClosed - Stream streamUnconfirmedBlobsSidecar(UInt64 startSlot, UInt64 endSlot); - Optional getEarliestBlobSidecarSlot(); - Optional getEarliestBlobsSidecarSlot(); - Map getColumnCounts(); - Map getBlobsSidecarColumnCounts(); + long getBlobSidecarColumnCount(); @MustBeClosed Stream streamFinalizedStateSlots(final UInt64 startSlot, final UInt64 endSlot); @@ -241,16 +226,8 @@ interface FinalizedUpdater extends AutoCloseable { void addNoBlobsSlot(SlotAndBlockRoot slotAndBlockRoot); - void addBlobsSidecar(BlobsSidecar blobsSidecar); - - void addUnconfirmedBlobsSidecar(BlobsSidecar blobsSidecar); - void removeBlobSidecar(SlotAndBlockRootAndBlobIndex key); - void removeBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); - - void confirmBlobsSidecar(SlotAndBlockRoot slotAndBlockRoot); - void commit(); void cancel(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java index 4abfd2261a1..23afac50c4d 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java @@ -29,7 +29,6 @@ import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BlockAndCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.BlockCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; @@ -219,8 +218,8 @@ public Map getColumnCounts() { } @Override - public Map getBlobsSidecarColumnCounts() { - return new LinkedHashMap<>(finalizedDao.getBlobsSidecarColumnCounts()); + public long getBlobSidecarColumnCount() { + return finalizedDao.getBlobSidecarColumnCount(); } @Override @@ -239,11 +238,6 @@ public Optional getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { return finalizedDao.getBlobSidecar(key); } - @Override - public Optional getBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - return finalizedDao.getBlobsSidecar(slotAndBlockRoot); - } - @Override @MustBeClosed public Stream streamBlobSidecarKeys( @@ -251,37 +245,11 @@ public Stream streamBlobSidecarKeys( return finalizedDao.streamBlobSidecarKeys(startSlot, endSlot); } - @Override - @MustBeClosed - public Stream> streamBlobsSidecar( - final UInt64 startSlot, final UInt64 endSlot) { - return finalizedDao.streamBlobsSidecar(startSlot, endSlot); - } - - @Override - @MustBeClosed - public Stream streamBlobsSidecarKeys( - final UInt64 startSlot, final UInt64 endSlot) { - return finalizedDao.streamBlobsSidecarKeys(startSlot, endSlot); - } - - @Override - @MustBeClosed - public Stream streamUnconfirmedBlobsSidecar( - final UInt64 startSlot, final UInt64 endSlot) { - return finalizedDao.streamUnconfirmedBlobsSidecar(startSlot, endSlot); - } - @Override public Optional getEarliestBlobSidecarSlot() { return finalizedDao.getEarliestBlobSidecarSlot(); } - @Override - public Optional getEarliestBlobsSidecarSlot() { - return finalizedDao.getEarliestBlobsSidecarSlot(); - } - @Override @MustBeClosed public Stream> getFinalizedStateRoots() { @@ -446,31 +414,11 @@ public void addNoBlobsSlot(final SlotAndBlockRoot slotAndBlockRoot) { finalizedUpdater.addNoBlobsSlot(slotAndBlockRoot); } - @Override - public void addBlobsSidecar(final BlobsSidecar blobsSidecar) { - finalizedUpdater.addBlobsSidecar(blobsSidecar); - } - - @Override - public void addUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) { - finalizedUpdater.addUnconfirmedBlobsSidecar(blobsSidecar); - } - @Override public void removeBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { finalizedUpdater.removeBlobSidecar(key); } - @Override - public void removeBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - finalizedUpdater.removeBlobsSidecar(slotAndBlockRoot); - } - - @Override - public void confirmBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - finalizedUpdater.confirmBlobsSidecar(slotAndBlockRoot); - } - @Override public void addHotState(final Bytes32 blockRoot, final BeaconState state) { hotUpdater.addHotState(blockRoot, state); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4FinalizedKvStoreDao.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4FinalizedKvStoreDao.java index 90f098d7315..8cab258cbfb 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4FinalizedKvStoreDao.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/V4FinalizedKvStoreDao.java @@ -20,10 +20,8 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -32,7 +30,6 @@ import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; @@ -128,10 +125,6 @@ public Optional getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { return db.get(schema.getColumnBlobSidecarBySlotRootBlobIndex(), key); } - public Optional getBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - return db.get(schema.getColumnBlobsSidecarBySlotAndBlockRoot(), slotAndBlockRoot); - } - @MustBeClosed public Stream streamBlobSidecarKeys( final UInt64 startSlot, final UInt64 endSlot) { @@ -141,46 +134,12 @@ public Stream streamBlobSidecarKeys( new SlotAndBlockRootAndBlobIndex(endSlot, MAX_BLOCK_ROOT, UInt64.MAX_VALUE)); } - @MustBeClosed - public Stream> streamBlobsSidecar( - final UInt64 startSlot, final UInt64 endSlot) { - return db.stream( - schema.getColumnBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot(startSlot, MIN_BLOCK_ROOT), - new SlotAndBlockRoot(endSlot, MAX_BLOCK_ROOT)) - .map(entry -> entry); - } - - @MustBeClosed - public Stream streamBlobsSidecarKeys( - final UInt64 startSlot, final UInt64 endSlot) { - return db.streamKeys( - schema.getColumnBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot(startSlot, MIN_BLOCK_ROOT), - new SlotAndBlockRoot(endSlot, MAX_BLOCK_ROOT)); - } - - @MustBeClosed - public Stream streamUnconfirmedBlobsSidecar( - final UInt64 startSlot, final UInt64 endSlot) { - return db.streamKeys( - schema.getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot(startSlot, MIN_BLOCK_ROOT), - new SlotAndBlockRoot(endSlot, MAX_BLOCK_ROOT)); - } - public Optional getEarliestBlobSidecarSlot() { return db.getFirstEntry(schema.getColumnBlobSidecarBySlotRootBlobIndex()) .map(ColumnEntry::getKey) .map(SlotAndBlockRootAndBlobIndex::getSlot); } - public Optional getEarliestBlobsSidecarSlot() { - return db.getFirstEntry(schema.getColumnBlobsSidecarBySlotAndBlockRoot()) - .map(ColumnEntry::getKey) - .map(SlotAndBlockRoot::getSlot); - } - public Optional getRawVariable(final KvStoreVariable var) { return db.getRaw(var); } @@ -233,19 +192,10 @@ public Map getColumnCounts() { return columnCounts; } - public Map getBlobsSidecarColumnCounts() { - final Map columnCounts = new LinkedHashMap<>(); - - schema.getColumnMap().entrySet().stream() - .filter( - stringKvStoreColumnEntry -> - List.of( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", - "BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT") - .contains(stringKvStoreColumnEntry.getKey())) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)) - .forEach((k, v) -> columnCounts.put(k, db.size(v))); - return columnCounts; + public long getBlobSidecarColumnCount() { + final KvStoreColumn column = + schema.getColumnMap().get("BLOB_SIDECAR_BY_SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX"); + return db.size(column); } @MustBeClosed @@ -346,40 +296,11 @@ public void addNoBlobsSlot(final SlotAndBlockRoot slotAndBlockRoot) { Bytes.EMPTY); } - @Override - public void addBlobsSidecar(final BlobsSidecar blobsSidecar) { - transaction.put( - schema.getColumnBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot( - blobsSidecar.getBeaconBlockSlot(), blobsSidecar.getBeaconBlockRoot()), - blobsSidecar.sszSerialize()); - } - - @Override - public void addUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) { - transaction.put( - schema.getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(), - new SlotAndBlockRoot( - blobsSidecar.getBeaconBlockSlot(), blobsSidecar.getBeaconBlockRoot()), - null); - } - @Override public void removeBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { transaction.delete(schema.getColumnBlobSidecarBySlotRootBlobIndex(), key); } - @Override - public void removeBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - transaction.delete(schema.getColumnBlobsSidecarBySlotAndBlockRoot(), slotAndBlockRoot); - } - - @Override - public void confirmBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - transaction.delete( - schema.getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(), slotAndBlockRoot); - } - @Override public void commit() { // Commit db updates diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaCombined.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaCombined.java index 77a5ba3b5dc..79e79ee821e 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaCombined.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaCombined.java @@ -59,10 +59,6 @@ public interface SchemaCombined extends Schema { KvStoreColumn getColumnBlobSidecarBySlotRootBlobIndex(); - KvStoreColumn getColumnBlobsSidecarBySlotAndBlockRoot(); - - KvStoreColumn getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(); - // Variables KvStoreVariable getVariableGenesisTime(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaFinalizedSnapshotStateAdapter.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaFinalizedSnapshotStateAdapter.java index cc31fce4b66..afa600a1870 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaFinalizedSnapshotStateAdapter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/SchemaFinalizedSnapshotStateAdapter.java @@ -21,7 +21,6 @@ import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; @@ -45,15 +44,6 @@ public KvStoreColumn getColumnFinalizedStatesBySlot() { return delegate.getColumnBlobSidecarBySlotRootBlobIndex(); } - public KvStoreColumn getColumnBlobsSidecarBySlotAndBlockRoot() { - return delegate.getColumnBlobsSidecarBySlotAndBlockRoot(); - } - - public KvStoreColumn - getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot() { - return delegate.getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot(); - } - public Map> getColumnMap() { return ImmutableMap.>builder() .put("SLOTS_BY_FINALIZED_ROOT", getColumnSlotsByFinalizedRoot()) @@ -62,10 +52,6 @@ public KvStoreColumn getColumnBlobsSidecarBySlotAndBloc .put("SLOTS_BY_FINALIZED_STATE_ROOT", getColumnSlotsByFinalizedStateRoot()) .put("NON_CANONICAL_BLOCKS_BY_ROOT", getColumnNonCanonicalBlocksByRoot()) .put("NON_CANONICAL_BLOCK_ROOTS_BY_SLOT", getColumnNonCanonicalRootsBySlot()) - .put("BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", getColumnBlobsSidecarBySlotAndBlockRoot()) - .put( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", - getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot()) .put( "BLOB_SIDECAR_BY_SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX", getColumnBlobSidecarBySlotRootBlobIndex()) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombined.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombined.java index 22e33995df9..61e98708622 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombined.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombined.java @@ -200,10 +200,9 @@ public KvStoreVariable getVariableFinalizedDepositSnapshot( .put("SLOTS_BY_FINALIZED_STATE_ROOT", getColumnSlotsByFinalizedStateRoot()) .put("NON_CANONICAL_BLOCKS_BY_ROOT", getColumnNonCanonicalBlocksByRoot()) .put("NON_CANONICAL_BLOCK_ROOTS_BY_SLOT", getColumnNonCanonicalRootsBySlot()) - .put("BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", getColumnBlobsSidecarBySlotAndBlockRoot()) .put( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", - getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot()) + "BLOB_SIDECAR_BY_SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX", + getColumnBlobSidecarBySlotRootBlobIndex()) .build(); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedSnapshot.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedSnapshot.java index 7c68a35f958..c610af44e53 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedSnapshot.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedSnapshot.java @@ -18,9 +18,7 @@ import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.BYTES32_SERIALIZER; import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.BYTES_SERIALIZER; import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX_KEY_SERIALIZER; -import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.SLOT_AND_BLOCK_ROOT_KEY_SERIALIZER; import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.UINT64_SERIALIZER; -import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.VOID_SERIALIZER; import com.google.common.collect.ImmutableMap; import java.util.Collection; @@ -32,7 +30,6 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; import tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer; @@ -48,8 +45,6 @@ public class V6SchemaCombinedSnapshot extends V6SchemaCombined private final KvStoreColumn finalizedStatesBySlot; private final KvStoreColumn blobSidecarBySlotRootBlobIndex; - private final KvStoreColumn blobsSidecarBySlotAndBlockRoot; - private final KvStoreColumn unconfirmedBlobsSidecarBySlotAndBlockRoot; private final List deletedColumnIds; private V6SchemaCombinedSnapshot(final Spec spec, final int finalizedOffset) { @@ -73,13 +68,6 @@ private V6SchemaCombinedSnapshot(final Spec spec, final int finalizedOffset) { KvStoreSerializer.createSignedBlockSerializer(spec)); nonCanonicalBlockRootsBySlot = KvStoreColumn.create(finalizedOffset + 6, UINT64_SERIALIZER, BLOCK_ROOTS_SERIALIZER); - - blobsSidecarBySlotAndBlockRoot = - KvStoreColumn.create( - finalizedOffset + 10, SLOT_AND_BLOCK_ROOT_KEY_SERIALIZER, BYTES_SERIALIZER); - unconfirmedBlobsSidecarBySlotAndBlockRoot = - KvStoreColumn.create( - finalizedOffset + 11, SLOT_AND_BLOCK_ROOT_KEY_SERIALIZER, VOID_SERIALIZER); blobSidecarBySlotRootBlobIndex = KvStoreColumn.create( finalizedOffset + 12, @@ -90,7 +78,9 @@ private V6SchemaCombinedSnapshot(final Spec spec, final int finalizedOffset) { List.of( asColumnId(finalizedOffset + 7), asColumnId(finalizedOffset + 8), - asColumnId(finalizedOffset + 9)); + asColumnId(finalizedOffset + 9), + asColumnId(finalizedOffset + 10), + asColumnId(finalizedOffset + 11)); } public static V6SchemaCombinedSnapshot createV4(final Spec spec) { @@ -137,17 +127,6 @@ public KvStoreColumn> getColumnNonCanonicalRootsBySlot() { return blobSidecarBySlotRootBlobIndex; } - @Override - public KvStoreColumn getColumnBlobsSidecarBySlotAndBlockRoot() { - return blobsSidecarBySlotAndBlockRoot; - } - - @Override - public KvStoreColumn - getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot() { - return unconfirmedBlobsSidecarBySlotAndBlockRoot; - } - @Override public Map> getColumnMap() { return ImmutableMap.>builder() @@ -164,10 +143,6 @@ public KvStoreColumn getColumnBlobsSidecarBySlotAndBloc .put("SLOTS_BY_FINALIZED_STATE_ROOT", getColumnSlotsByFinalizedStateRoot()) .put("NON_CANONICAL_BLOCKS_BY_ROOT", getColumnNonCanonicalBlocksByRoot()) .put("NON_CANONICAL_BLOCK_ROOTS_BY_SLOT", getColumnNonCanonicalRootsBySlot()) - .put("BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", getColumnBlobsSidecarBySlotAndBlockRoot()) - .put( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", - getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot()) .put( "BLOB_SIDECAR_BY_SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX", getColumnBlobSidecarBySlotRootBlobIndex()) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedTreeState.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedTreeState.java index 40fb65ab666..c0b76894285 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedTreeState.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedTreeState.java @@ -19,9 +19,7 @@ import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.BYTES_SERIALIZER; import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.COMPRESSED_BRANCH_INFO_KV_STORE_SERIALIZER; import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX_KEY_SERIALIZER; -import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.SLOT_AND_BLOCK_ROOT_KEY_SERIALIZER; import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.UINT64_SERIALIZER; -import static tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer.VOID_SERIALIZER; import com.google.common.collect.ImmutableMap; import java.util.Collection; @@ -34,7 +32,6 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; import tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer; @@ -50,8 +47,6 @@ public class V6SchemaCombinedTreeState extends V6SchemaCombined implements Schem private final KvStoreColumn finalizedStateTreeBranchesByRoot; private final KvStoreColumn blobSidecarBySlotRootBlobIndex; - private final KvStoreColumn blobsSidecarBySlotAndBlockRoot; - private final KvStoreColumn unconfirmedBlobsSidecarBySlotAndBlockRoot; private final List deletedColumnIds; public V6SchemaCombinedTreeState(final Spec spec) { @@ -81,12 +76,6 @@ public V6SchemaCombinedTreeState(final Spec spec) { V6_FINALIZED_OFFSET + 8, BYTES32_SERIALIZER, KvStoreSerializer.createSignedBlockSerializer(spec)); - blobsSidecarBySlotAndBlockRoot = - KvStoreColumn.create( - finalizedOffset + 12, SLOT_AND_BLOCK_ROOT_KEY_SERIALIZER, BYTES_SERIALIZER); - unconfirmedBlobsSidecarBySlotAndBlockRoot = - KvStoreColumn.create( - finalizedOffset + 13, SLOT_AND_BLOCK_ROOT_KEY_SERIALIZER, VOID_SERIALIZER); blobSidecarBySlotRootBlobIndex = KvStoreColumn.create( finalizedOffset + 14, @@ -96,7 +85,9 @@ public V6SchemaCombinedTreeState(final Spec spec) { List.of( asColumnId(finalizedOffset + 9), asColumnId(finalizedOffset + 10), - asColumnId(finalizedOffset + 11)); + asColumnId(finalizedOffset + 11), + asColumnId(finalizedOffset + 12), + asColumnId(finalizedOffset + 13)); } @Override @@ -145,17 +136,6 @@ public KvStoreColumn> getColumnNonCanonicalRootsBySlot() { return blobSidecarBySlotRootBlobIndex; } - @Override - public KvStoreColumn getColumnBlobsSidecarBySlotAndBlockRoot() { - return blobsSidecarBySlotAndBlockRoot; - } - - @Override - public KvStoreColumn - getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot() { - return unconfirmedBlobsSidecarBySlotAndBlockRoot; - } - @Override public Map> getVariableMap() { return Map.of( @@ -189,10 +169,6 @@ public Map> getVariableMap() { .put("SLOTS_BY_FINALIZED_STATE_ROOT", getColumnSlotsByFinalizedStateRoot()) .put("NON_CANONICAL_BLOCKS_BY_ROOT", getColumnNonCanonicalBlocksByRoot()) .put("NON_CANONICAL_BLOCK_ROOTS_BY_SLOT", getColumnNonCanonicalRootsBySlot()) - .put("BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", getColumnBlobsSidecarBySlotAndBlockRoot()) - .put( - "UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT", - getColumnUnconfirmedBlobsSidecarBySlotAndBlockRoot()) .put( "BLOB_SIDECAR_BY_SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX", getColumnBlobSidecarBySlotRootBlobIndex()) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java index 68c108f247c..516e3eab146 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -31,7 +30,6 @@ import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BlockCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -249,8 +247,8 @@ public Map getColumnCounts() { } @Override - public Map getBlobsSidecarColumnCounts() { - return new HashMap<>(); + public long getBlobSidecarColumnCount() { + return 0L; } @Override @@ -275,74 +273,27 @@ public void storeBlobSidecar(final BlobSidecar blobSidecar) {} @Override public void storeNoBlobsSlot(final SlotAndBlockRoot slotAndBlockRoot) {} - @Override - public void storeUnconfirmedBlobsSidecar(final BlobsSidecar blobsSidecar) {} - - @Override - public void confirmBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) {} - @Override public Optional getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { return Optional.empty(); } - @Override - public Optional getBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) { - return Optional.empty(); - } - @Override public void removeBlobSidecars(final UInt64 slot) {} - @Override - public void removeBlobsSidecar(final SlotAndBlockRoot slotAndBlockRoot) {} - @Override public Stream streamBlobSidecarKeys( final UInt64 startSlot, final UInt64 endSlot) { return Stream.empty(); } - @Override - public Stream streamBlobsSidecars(final UInt64 startSlot, final UInt64 endSlot) { - return Stream.empty(); - } - - @Override - public Stream> streamBlobsSidecarsAsSsz( - final UInt64 startSlot, final UInt64 endSlot) { - return Stream.empty(); - } - - @Override - public Stream streamBlobsSidecarKeys(UInt64 startSlot, UInt64 endSlot) { - return Stream.empty(); - } - @Override public Optional getEarliestBlobSidecarSlot() { return Optional.empty(); } @Override - public Optional getEarliestBlobsSidecarSlot() { - return Optional.empty(); - } - - @Override - public boolean pruneOldestBlobsSidecar(final UInt64 lastSlotToPrune, final int pruneLimit) { - return false; - } - - @Override - public Stream streamUnconfirmedBlobsSidecars( - final UInt64 startSlot, final UInt64 endSlot) { - return Stream.empty(); - } - - @Override - public boolean pruneOldestUnconfirmedBlobsSidecars( - final UInt64 lastSlotToPrune, final int pruneLimit) { + public boolean pruneOldestBlobSidecars(final UInt64 lastSlotToPrune, final int pruneLimit) { return false; } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobsSidecarPruner.java b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java similarity index 66% rename from storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobsSidecarPruner.java rename to storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java index 92de4e20522..2d437fc1266 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobsSidecarPruner.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java @@ -16,11 +16,9 @@ import static tech.pegasys.teku.spec.config.Constants.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; import java.time.Duration; -import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicLong; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -33,12 +31,10 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.service.serviceutils.Service; import tech.pegasys.teku.spec.Spec; -import tech.pegasys.teku.spec.datastructures.state.Checkpoint; -import tech.pegasys.teku.storage.api.FinalizedCheckpointChannel; import tech.pegasys.teku.storage.server.Database; import tech.pegasys.teku.storage.server.ShuttingDownException; -public class BlobsSidecarPruner extends Service implements FinalizedCheckpointChannel { +public class BlobSidecarPruner extends Service { private static final Logger LOG = LogManager.getLogger(); private final Spec spec; @@ -47,17 +43,14 @@ public class BlobsSidecarPruner extends Service implements FinalizedCheckpointCh private final Duration pruneInterval; private final int pruneLimit; private final TimeProvider timeProvider; - private final boolean blobsSidecarStorageCountersEnabled; + private final boolean blobSidecarStorageCountersEnabled; private Optional scheduledPruner = Optional.empty(); private Optional genesisTime = Optional.empty(); - private final AtomicReference> latestFinalizedSlot = - new AtomicReference<>(Optional.empty()); + private final AtomicLong blobColumnSize = new AtomicLong(0); - private final Map blobsColumnsSize = new ConcurrentHashMap<>(); - - public BlobsSidecarPruner( + public BlobSidecarPruner( final Spec spec, final Database database, final MetricsSystem metricsSystem, @@ -65,34 +58,24 @@ public BlobsSidecarPruner( final TimeProvider timeProvider, final Duration pruneInterval, final int pruneLimit, - final boolean blobsSidecarStorageCountersEnabled) { + final boolean blobSidecarStorageCountersEnabled) { this.spec = spec; this.database = database; this.asyncRunner = asyncRunner; this.pruneInterval = pruneInterval; this.pruneLimit = pruneLimit; this.timeProvider = timeProvider; - this.blobsSidecarStorageCountersEnabled = blobsSidecarStorageCountersEnabled; + this.blobSidecarStorageCountersEnabled = blobSidecarStorageCountersEnabled; - if (blobsSidecarStorageCountersEnabled) { + if (blobSidecarStorageCountersEnabled) { LabelledGauge labelledGauge = metricsSystem.createLabelledGauge( TekuMetricCategory.STORAGE, - "blobs_sidecar_counts", - "Number of blobs sidecars stored", + "blob_sidecar_counts", + "Number of blob sidecars stored", "type"); - labelledGauge.labels( - () -> - Optional.ofNullable(blobsColumnsSize.get("BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT")) - .orElse(0L), - "total"); - labelledGauge.labels( - () -> - Optional.ofNullable( - blobsColumnsSize.get("UNCONFIRMED_BLOBS_SIDECAR_BY_SLOT_AND_BLOCK_ROOT")) - .orElse(0L), - "unconfirmed"); + labelledGauge.labels(blobColumnSize::get, "total"); } } @@ -116,35 +99,9 @@ protected synchronized SafeFuture doStop() { private void pruneBlobs() { pruneBlobsPriorToAvailabilityWindow(); - pruneUnconfirmedBlobs(); - if (blobsSidecarStorageCountersEnabled) { - blobsColumnsSize.putAll(database.getBlobsSidecarColumnCounts()); - } - } - - private void pruneUnconfirmedBlobs() { - final Optional maybeLatestFinalizedSlot = - latestFinalizedSlot.getAndSet(Optional.empty()); - if (maybeLatestFinalizedSlot.isEmpty()) { - LOG.debug("Not pruning unconfirmed blobs as no new finalized slot is available."); - return; - } - - LOG.debug( - "Pruning unconfirmed blobs up to slot {}, limit {}", - maybeLatestFinalizedSlot.get(), - pruneLimit); - try { - final long start = System.currentTimeMillis(); - final boolean limitReached = - database.pruneOldestUnconfirmedBlobsSidecars(maybeLatestFinalizedSlot.get(), pruneLimit); - LOG.debug( - "Unconfirmed blobs pruning finished in {} ms. Limit reached: {}", - () -> System.currentTimeMillis() - start, - () -> limitReached); - } catch (ShuttingDownException | RejectedExecutionException ex) { - LOG.debug("Shutting down", ex); + if (blobSidecarStorageCountersEnabled) { + blobColumnSize.set(database.getBlobSidecarColumnCount()); } } @@ -167,7 +124,7 @@ private void pruneBlobsPriorToAvailabilityWindow() { LOG.debug("Pruning blobs up to slot {}, limit {}", latestPrunableSlot, pruneLimit); try { final long start = System.currentTimeMillis(); - final boolean limitReached = database.pruneOldestBlobsSidecar(latestPrunableSlot, pruneLimit); + final boolean limitReached = database.pruneOldestBlobSidecars(latestPrunableSlot, pruneLimit); LOG.debug( "Blobs pruning finished in {} ms. Limit reached: {}", () -> System.currentTimeMillis() - start, @@ -177,12 +134,6 @@ private void pruneBlobsPriorToAvailabilityWindow() { } } - @Override - public void onNewFinalizedCheckpoint( - final Checkpoint checkpoint, final boolean fromOptimisticBlock) { - latestFinalizedSlot.set(Optional.of(checkpoint.getEpochStartSlot(spec))); - } - private Optional getGenesisTime() { if (genesisTime.isPresent()) { return genesisTime; diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java b/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java index a2c8bfddbee..9d9cb5302b3 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java @@ -14,7 +14,6 @@ package tech.pegasys.teku.storage.store; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import static tech.pegasys.teku.dataproviders.generators.StateAtSlotTask.AsyncStateProvider.fromAnchor; import static tech.pegasys.teku.dataproviders.lookup.BlockProvider.fromDynamicMap; import static tech.pegasys.teku.dataproviders.lookup.BlockProvider.fromMap; @@ -39,7 +38,6 @@ import tech.pegasys.teku.dataproviders.generators.StateAtSlotTask; import tech.pegasys.teku.dataproviders.generators.StateGenerationTask; import tech.pegasys.teku.dataproviders.generators.StateRegenerationBaseSelector; -import tech.pegasys.teku.dataproviders.lookup.BlobsSidecarProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.AsyncRunner; @@ -49,12 +47,10 @@ import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.blocks.StateAndBlockSummary; -import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.SignedBeaconBlockAndBlobsSidecar; import tech.pegasys.teku.spec.datastructures.execution.SlotAndExecutionPayloadSummary; import tech.pegasys.teku.spec.datastructures.forkchoice.VoteTracker; import tech.pegasys.teku.spec.datastructures.forkchoice.VoteUpdater; @@ -64,7 +60,6 @@ import tech.pegasys.teku.spec.datastructures.state.Checkpoint; import tech.pegasys.teku.spec.datastructures.state.CheckpointState; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; -import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; import tech.pegasys.teku.storage.api.StorageUpdateChannel; import tech.pegasys.teku.storage.api.StoredBlockMetadata; import tech.pegasys.teku.storage.api.VoteUpdateChannel; @@ -88,7 +83,6 @@ class Store implements UpdatableStore { private final Spec spec; private final StateAndBlockSummaryProvider stateProvider; private final BlockProvider blockProvider; - private final BlobsSidecarProvider blobsSidecarProvider; final ForkChoiceStrategy forkChoiceStrategy; private final Optional initialCheckpoint; @@ -110,7 +104,6 @@ private Store( final Spec spec, final int hotStatePersistenceFrequencyInEpochs, final BlockProvider blockProvider, - final BlobsSidecarProvider blobsSidecarProvider, final StateAndBlockSummaryProvider stateProvider, final CachingTaskQueue states, final Optional initialCheckpoint, @@ -169,8 +162,6 @@ private Store( .orElseGet(Collections::emptyMap)), fromMap(this.blocks), blockProvider); - - this.blobsSidecarProvider = blobsSidecarProvider; } public static UpdatableStore create( @@ -178,7 +169,6 @@ public static UpdatableStore create( final MetricsSystem metricsSystem, final Spec spec, final BlockProvider blockProvider, - final BlobsSidecarProvider blobsSidecarProvider, final StateAndBlockSummaryProvider stateAndBlockProvider, final Optional initialCheckpoint, final UInt64 time, @@ -220,7 +210,6 @@ public static UpdatableStore create( spec, config.getHotStatePersistenceFrequencyInEpochs(), blockProvider, - blobsSidecarProvider, stateAndBlockProvider, stateTaskQueue, initialCheckpoint, @@ -471,67 +460,6 @@ public SafeFuture> retrieveSignedBlock(final Bytes32 }); } - @Override - public SafeFuture> retrieveSignedBlockAndBlobsSidecar( - final Bytes32 blockRoot) { - - return retrieveSignedBlock(blockRoot) - .thenCompose( - signedBeaconBlock -> { - if (signedBeaconBlock.isEmpty()) { - return SafeFuture.completedFuture(Optional.empty()); - } - return retrieveSignedBlockAndBlobsSidecar(signedBeaconBlock.get()) - .thenApply(Optional::of); - }); - } - - private SafeFuture retrieveSignedBlockAndBlobsSidecar( - final SignedBeaconBlock signedBeaconBlock) { - return blobsSidecarProvider - .getBlobsSidecar(signedBeaconBlock) - .thenApply( - blobsSidecar -> - createSignedBeaconBlockAndBlobsSidecar(signedBeaconBlock, blobsSidecar)); - } - - private SignedBeaconBlockAndBlobsSidecar createSignedBeaconBlockAndBlobsSidecar( - final SignedBeaconBlock signedBeaconBlock, final Optional maybeBlobsSidecar) { - if (maybeBlobsSidecar.isPresent()) { - checkState( - maybeBlobsSidecar.get().getBeaconBlockRoot().equals(signedBeaconBlock.getRoot()), - "BlobsSidecar belongs to a different BeaconBlock"); - } else { - checkState( - signedBeaconBlock - .getBeaconBlock() - .orElseThrow() - .getBody() - .toVersionDeneb() - .orElseThrow() - .getBlobKzgCommitments() - .isEmpty(), - "BeaconBlock contains kzg commitments but BlobsSidecar is not present"); - } - - final SchemaDefinitionsDeneb schemaDefinitionsDeneb = - spec.atSlot(signedBeaconBlock.getSlot()) - .getSchemaDefinitions() - .toVersionDeneb() - .orElseThrow(); - - final BlobsSidecar blobsSidecar = - maybeBlobsSidecar.orElseGet( - () -> - schemaDefinitionsDeneb - .getBlobsSidecarSchema() - .createEmpty(signedBeaconBlock.getRoot(), signedBeaconBlock.getSlot())); - - return schemaDefinitionsDeneb - .getSignedBeaconBlockAndBlobsSidecarSchema() - .create(signedBeaconBlock, blobsSidecar); - } - @Override public SafeFuture> retrieveBlockAndState(Bytes32 blockRoot) { return getAndCacheBlockAndState(blockRoot); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java index 9c0d0cce75c..542d18824cc 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java @@ -21,7 +21,6 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; -import tech.pegasys.teku.dataproviders.lookup.BlobsSidecarProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.AsyncRunner; @@ -39,7 +38,6 @@ public class StoreBuilder { private MetricsSystem metricsSystem; private Spec spec; private BlockProvider blockProvider; - private BlobsSidecarProvider blobsSidecarProvider; private StateAndBlockSummaryProvider stateAndBlockProvider; private StoreConfig storeConfig = StoreConfig.createDefault(); @@ -109,7 +107,6 @@ public UpdatableStore build() { metricsSystem, spec, blockProvider, - blobsSidecarProvider, stateAndBlockProvider, anchor, time, @@ -167,12 +164,6 @@ public StoreBuilder blockProvider(final BlockProvider blockProvider) { return this; } - public StoreBuilder blobsSidecarProvider(final BlobsSidecarProvider blobsSidecarProvider) { - checkNotNull(blobsSidecarProvider); - this.blobsSidecarProvider = blobsSidecarProvider; - return this; - } - public StoreBuilder stateProvider(final StateAndBlockSummaryProvider stateProvider) { checkNotNull(stateProvider); this.stateAndBlockProvider = stateProvider; diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java index 5d5be43de7b..1139395599c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java @@ -39,7 +39,6 @@ import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.blocks.StateAndBlockSummary; -import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.SignedBeaconBlockAndBlobsSidecar; import tech.pegasys.teku.spec.datastructures.execution.SlotAndExecutionPayloadSummary; import tech.pegasys.teku.spec.datastructures.forkchoice.ReadOnlyForkChoiceStrategy; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; @@ -331,12 +330,6 @@ public SafeFuture> retrieveSignedBlock(Bytes32 block return store.retrieveSignedBlock(blockRoot); } - @Override - public SafeFuture> retrieveSignedBlockAndBlobsSidecar( - Bytes32 blockRoot) { - return store.retrieveSignedBlockAndBlobsSidecar(blockRoot); - } - @Override public SafeFuture> retrieveBlockAndState(Bytes32 blockRoot) { if (blockData.containsKey(blockRoot)) { diff --git a/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java b/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java index d3a8a43a849..334d69f8161 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java @@ -26,7 +26,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Test; -import tech.pegasys.teku.dataproviders.lookup.BlobsSidecarProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -105,7 +104,6 @@ public void storageBackedClient_storeInitializeViaGetStoreRequest() .metricsSystem(new StubMetricsSystem()) .specProvider(spec) .blockProvider(BlockProvider.NOOP) - .blobsSidecarProvider(BlobsSidecarProvider.NOOP) .stateProvider(StateAndBlockSummaryProvider.NOOP) .storeConfig(storeConfig) .build(); @@ -153,7 +151,6 @@ public void storageBackedClient_storeInitializeViaNewGenesisState() .metricsSystem(new StubMetricsSystem()) .specProvider(spec) .blockProvider(BlockProvider.NOOP) - .blobsSidecarProvider(BlobsSidecarProvider.NOOP) .stateProvider(StateAndBlockSummaryProvider.NOOP) .storeConfig(storeConfig) .build(); diff --git a/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobsSidecarPrunerTest.java b/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java similarity index 74% rename from storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobsSidecarPrunerTest.java rename to storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java index 8ccdaf11239..61596af34ed 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobsSidecarPrunerTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java @@ -18,7 +18,6 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static tech.pegasys.teku.spec.config.Constants.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; @@ -33,10 +32,9 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; -import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.storage.server.Database; -public class BlobsSidecarPrunerTest { +public class BlobSidecarPrunerTest { public static final Duration PRUNE_INTERVAL = Duration.ofSeconds(5); public static final int PRUNE_LIMIT = 10; @@ -48,13 +46,12 @@ public class BlobsSidecarPrunerTest { private UInt64 genesisTime = UInt64.valueOf(0); private final StubTimeProvider timeProvider = StubTimeProvider.withTimeInSeconds(0); - private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); private final StubAsyncRunner asyncRunner = new StubAsyncRunner(timeProvider); private final Database database = mock(Database.class); private final StubMetricsSystem stubMetricsSystem = new StubMetricsSystem(); - private final BlobsSidecarPruner blobsPruner = - new BlobsSidecarPruner( + private final BlobSidecarPruner blobsPruner = + new BlobSidecarPruner( spec, database, stubMetricsSystem, @@ -77,7 +74,7 @@ void shouldNotPruneWhenGenesisNotAvailable() { asyncRunner.executeDueActions(); verify(database).getGenesisTime(); - verify(database, never()).pruneOldestBlobsSidecar(any(), anyInt()); + verify(database, never()).pruneOldestBlobSidecars(any(), anyInt()); } @Test @@ -85,7 +82,7 @@ void shouldNotPrunePriorGenesis() { asyncRunner.executeDueActions(); verify(database).getGenesisTime(); - verify(database, never()).pruneOldestBlobsSidecar(any(), anyInt()); + verify(database, never()).pruneOldestBlobSidecars(any(), anyInt()); } @Test @@ -98,7 +95,7 @@ void shouldNotPruneWhenLatestPrunableIncludeGenesis() { timeProvider.advanceTimeBy(Duration.ofSeconds(currentTime.longValue())); asyncRunner.executeDueActions(); - verify(database, never()).pruneOldestBlobsSidecar(any(), anyInt()); + verify(database, never()).pruneOldestBlobSidecars(any(), anyInt()); } @Test @@ -113,30 +110,6 @@ void shouldPruneWhenLatestPrunableSlotIsGreaterThanOldestDAEpoch() { timeProvider.advanceTimeBy(Duration.ofSeconds(currentTime.longValue())); asyncRunner.executeDueActions(); - verify(database).pruneOldestBlobsSidecar(UInt64.valueOf((slotsPerEpoch / 2) - 1), PRUNE_LIMIT); - } - - @Test - void shouldNotPruneUnconfirmedBlobsWithoutFinalizedCheckpoint() { - asyncRunner.executeDueActions(); - - verify(database, never()).pruneOldestUnconfirmedBlobsSidecars(any(), anyInt()); - } - - @Test - void shouldPruneUnconfirmedBlobsWithFinalizedCheckpoint() { - blobsPruner.onNewFinalizedCheckpoint(dataStructureUtil.randomCheckpoint(1), false); - asyncRunner.executeDueActions(); - - final UInt64 expectedLastSlotToPrune = - UInt64.valueOf(spec.getGenesisSpecConfig().getSlotsPerEpoch()); - - verify(database).pruneOldestUnconfirmedBlobsSidecars(expectedLastSlotToPrune, PRUNE_LIMIT); - - timeProvider.advanceTimeBy(PRUNE_INTERVAL); - asyncRunner.executeDueActions(); - - verify(database, times(1)) - .pruneOldestUnconfirmedBlobsSidecars(expectedLastSlotToPrune, PRUNE_LIMIT); + verify(database).pruneOldestBlobSidecars(UInt64.valueOf((slotsPerEpoch / 2) - 1), PRUNE_LIMIT); } } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java b/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java index 7ad7d959669..59aa1593642 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java @@ -25,7 +25,6 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Collectors; -import tech.pegasys.teku.dataproviders.lookup.BlobsSidecarProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -140,7 +139,6 @@ protected UpdatableStore createGenesisStore(final StoreConfig pruningOptions) { .metricsSystem(new StubMetricsSystem()) .specProvider(spec) .blockProvider(blockProviderFromChainBuilder()) - .blobsSidecarProvider(blobsSidecarFromChainBuilder()) .stateProvider(StateAndBlockSummaryProvider.NOOP) .anchor(Optional.empty()) .genesisTime(genesis.getState().getGenesisTime()) @@ -171,9 +169,4 @@ protected BlockProvider blockProviderFromChainBuilder() { .flatMap(Optional::stream) .collect(Collectors.toMap(SignedBeaconBlock::getRoot, Function.identity()))); } - - protected BlobsSidecarProvider blobsSidecarFromChainBuilder() { - return (block) -> - SafeFuture.completedFuture(chainBuilder.getBlobsSidecar(block.getBlockRoot())); - } } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java index b76912cc809..3e6ba7381f7 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java @@ -24,23 +24,19 @@ import java.util.Optional; import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes32; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.async.SafeFutureAssert; import tech.pegasys.teku.infrastructure.metrics.StubMetricsSystem; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; -import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.SignedBeaconBlockAndBlobsSidecar; import tech.pegasys.teku.spec.datastructures.forkchoice.InvalidCheckpointException; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; import tech.pegasys.teku.spec.datastructures.state.Checkpoint; import tech.pegasys.teku.spec.datastructures.state.CheckpointState; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; -import tech.pegasys.teku.spec.generator.ChainBuilder.BlockOptions; import tech.pegasys.teku.storage.api.StubStorageUpdateChannel; import tech.pegasys.teku.storage.api.StubStorageUpdateChannelWithDelays; import tech.pegasys.teku.storage.store.UpdatableStore.StoreTransaction; @@ -60,7 +56,6 @@ public void create_timeLessThanGenesisTime() { new StubMetricsSystem(), spec, blockProviderFromChainBuilder(), - blobsSidecarFromChainBuilder(), StateAndBlockSummaryProvider.NOOP, Optional.empty(), genesisTime.minus(1), @@ -130,99 +125,6 @@ public void retrieveBlockState_withLimitedCache() { }); } - @Test - @Disabled("TODO: Rework for BlobSidecars") - public void retrieveSignedBlockAndBlobsSidecar_withBlobs() { - final UpdatableStore store = createGenesisStore(); - final SignedBlockAndState blockAndState = - chainBuilder.generateBlockAtSlot(1, BlockOptions.create().setGenerateRandomBlobs(true)); - - addBlock(store, blockAndState); - - final SafeFuture> signedBlockAndBlobsSidecar = - store.retrieveSignedBlockAndBlobsSidecar(blockAndState.getRoot()); - - assertThat(signedBlockAndBlobsSidecar) - .isCompletedWithValueMatching(Optional::isPresent, "Result must be present") - .isCompletedWithValueMatching( - signedBeaconBlockAndBlobsSidecar -> - signedBeaconBlockAndBlobsSidecar - .orElseThrow() - .getSignedBeaconBlock() - .equals(blockAndState.getBlock()), - " Block must match") - .isCompletedWithValueMatching( - signedBeaconBlockAndBlobsSidecar -> - !signedBeaconBlockAndBlobsSidecar - .orElseThrow() - .getBlobsSidecar() - .getBlobs() - .isEmpty(), - "Blobs must be present"); - } - - @Test - public void retrieveSignedBlockAndBlobsSidecar_withEmptyBlobs() { - final UpdatableStore store = createGenesisStore(); - final SignedBlockAndState blockAndState = - chainBuilder.generateBlockAtSlot(1, BlockOptions.create().setGenerateRandomBlobs(false)); - - addBlock(store, blockAndState); - - final SafeFuture> signedBlockAndBlobsSidecar = - store.retrieveSignedBlockAndBlobsSidecar(blockAndState.getRoot()); - - assertThat(signedBlockAndBlobsSidecar) - .isCompletedWithValueMatching(Optional::isPresent, "Result must be present") - .isCompletedWithValueMatching( - signedBeaconBlockAndBlobsSidecar -> - signedBeaconBlockAndBlobsSidecar - .orElseThrow() - .getSignedBeaconBlock() - .equals(blockAndState.getBlock()), - " Block must match") - .isCompletedWithValueMatching( - signedBeaconBlockAndBlobsSidecar -> - signedBeaconBlockAndBlobsSidecar - .orElseThrow() - .getBlobsSidecar() - .getBlobs() - .isEmpty(), - "Blobs must be empty"); - } - - @Test - public void retrieveSignedBlockAndBlobsSidecar_withInconsistentSidecar() { - final UpdatableStore store = createGenesisStore(); - - // generate block with commitments but skip blob storage - final SignedBlockAndState blockAndState = - chainBuilder.generateBlockAtSlot( - 1, BlockOptions.create().setGenerateRandomBlobs(true).setStoreBlobsSidecar(false)); - - addBlock(store, blockAndState); - - final SafeFuture> signedBlockAndBlobsSidecar = - store.retrieveSignedBlockAndBlobsSidecar(blockAndState.getRoot()); - - SafeFutureAssert.assertThatSafeFuture(signedBlockAndBlobsSidecar) - .isCompletedExceptionallyWithMessage( - "BeaconBlock contains kzg commitments but BlobsSidecar is not present"); - } - - @Test - public void retrieveSignedBlockAndBlobsSidecar_shouldReturnEmptyIfBlockNotPresent() { - final UpdatableStore store = createGenesisStore(); - final SignedBlockAndState blockAndState = - chainBuilder.generateBlockAtSlot(1, BlockOptions.create().setGenerateRandomBlobs(true)); - - final SafeFuture> signedBlockAndBlobsSidecar = - store.retrieveSignedBlockAndBlobsSidecar(blockAndState.getRoot()); - - assertThat(signedBlockAndBlobsSidecar) - .isCompletedWithValueMatching(Optional::isEmpty, "Result must be empty"); - } - @Test public void retrieveCheckpointState_withLimitedCache() { processCheckpointsWithLimitedCache( diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java index b5f37fe4dbf..65c8ec76789 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java @@ -24,7 +24,6 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -144,10 +143,4 @@ public SafeFuture> getBlobSidecarKeys( final UInt64 startSlot, final UInt64 endSlot, final UInt64 limit) { return SafeFuture.completedFuture(List.of()); } - - @Override - public SafeFuture> getBlobsSidecar( - final SlotAndBlockRoot slotAndBlockRoot) { - return SafeFuture.completedFuture(Optional.empty()); - } } diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannel.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannel.java index 29084e19543..9a3ccf27b0c 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannel.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannel.java @@ -21,7 +21,6 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; @@ -75,21 +74,11 @@ public SafeFuture onBlobSidecar(final BlobSidecar blobSidecar) { return SafeFuture.COMPLETE; } - @Override - public SafeFuture onBlobsSidecar(final BlobsSidecar blobsSidecar) { - return SafeFuture.COMPLETE; - } - @Override public SafeFuture onBlobSidecarsRemoval(final UInt64 slot) { return SafeFuture.COMPLETE; } - @Override - public SafeFuture onBlobsSidecarRemoval(final SlotAndBlockRoot blobsSidecarKey) { - return SafeFuture.COMPLETE; - } - @Override public void onChainInitialized(final AnchorPoint initialAnchor) {} } diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannelWithDelays.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannelWithDelays.java index 586998a805a..073908d6f09 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannelWithDelays.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageUpdateChannelWithDelays.java @@ -22,7 +22,6 @@ import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; @@ -81,21 +80,11 @@ public SafeFuture onBlobSidecar(final BlobSidecar blobSidecar) { return asyncRunner.runAsync(() -> SafeFuture.COMPLETE); } - @Override - public SafeFuture onBlobsSidecar(final BlobsSidecar blobsSidecar) { - return asyncRunner.runAsync(() -> SafeFuture.COMPLETE); - } - @Override public SafeFuture onBlobSidecarsRemoval(final UInt64 slot) { return asyncRunner.runAsync(() -> SafeFuture.COMPLETE); } - @Override - public SafeFuture onBlobsSidecarRemoval(final SlotAndBlockRoot blobsSidecarKey) { - return asyncRunner.runAsync(() -> SafeFuture.COMPLETE); - } - @Override public void onChainInitialized(final AnchorPoint initialAnchor) {} } diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java index 3c1a1146b4b..6ce3c04a01a 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java @@ -22,8 +22,9 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; +import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadHeader; import tech.pegasys.teku.spec.datastructures.state.Checkpoint; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; @@ -234,8 +235,16 @@ public SignedBlockAndState advanceChainUntil(final UInt64 slot) { public SignedBlockAndState advanceChain(final UInt64 slot) { final SignedBlockAndState block = chainBuilder.generateBlockAtSlot(slot); - Optional maybeBlobsSideCar = chainBuilder.getBlobsSidecar(block.getRoot()); - maybeBlobsSideCar.ifPresent(blobSidecarManager::storeUnconfirmedBlobsSidecar); + final Optional> maybeBlobSidecars = + chainBuilder.getBlobSidecars(block.getRoot()); + maybeBlobSidecars.ifPresent( + blobSidecars -> { + if (blobSidecars.isEmpty()) { + blobSidecarManager.storeNoBlobsSlot(new SlotAndBlockRoot(slot, block.getRoot())); + } else { + blobSidecars.forEach(blobSidecarManager::storeBlobSidecar); + } + }); saveBlock(block); return block; } diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java index 8d1d59130a7..0a86dec6b22 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; -import tech.pegasys.teku.dataproviders.lookup.BlobsSidecarProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.AsyncRunner; @@ -49,7 +48,6 @@ private MemoryOnlyRecentChainData( metricsSystem, storeConfig, BlockProvider.NOOP, - BlobsSidecarProvider.NOOP, StateAndBlockSummaryProvider.NOOP, storageUpdateChannel, voteUpdateChannel, diff --git a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java index cf55963050d..0ae56a40965 100644 --- a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java +++ b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java @@ -13,14 +13,10 @@ package tech.pegasys.teku.cli.subcommand.debug; -import static tech.pegasys.teku.spec.config.Constants.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; - import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.Map; @@ -49,14 +45,13 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.service.serviceutils.layout.DataDirLayout; import tech.pegasys.teku.spec.Spec; -import tech.pegasys.teku.spec.SpecMilestone; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobsSidecar; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockInvariants; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.state.AnchorPoint; import tech.pegasys.teku.spec.datastructures.state.Checkpoint; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; import tech.pegasys.teku.storage.server.Database; import tech.pegasys.teku.storage.server.DatabaseStorageException; import tech.pegasys.teku.storage.server.DepositStorage; @@ -407,325 +402,29 @@ public int dumpBlobsSidecar( Files.newOutputStream( outputFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) { - try (final Stream> streamBlobsSidecarAsSsz = - database.streamBlobsSidecarsAsSsz(from, to)) { - for (final Iterator> iterator = - streamBlobsSidecarAsSsz.iterator(); - iterator.hasNext(); ) { - final Map.Entry entryData = iterator.next(); + try (final Stream keyStream = + database.streamBlobSidecarKeys(from, to)) { + + for (final Iterator it = keyStream.iterator(); + it.hasNext(); ) { + final SlotAndBlockRootAndBlobIndex key = it.next(); + if (key.isNoBlobsKey()) { + continue; + } + final BlobSidecar blobSidecar = database.getBlobSidecar(key).orElseThrow(); out.putNextEntry( new ZipEntry( - entryData.getKey().getSlot() + blobSidecar.getSlot() + "_" - + entryData.getKey().getBlockRoot().toUnprefixedHexString() + + blobSidecar.getBlockRoot().toUnprefixedHexString() + ".ssz")); - out.write(entryData.getValue().toArrayUnsafe()); + blobSidecar.sszSerialize(out); index++; } } - System.out.println("Wrote " + index + " blobs sidecars to " + outputFile.toAbsolutePath()); - - index = 0; - try (final Stream unconfirmedBlobs = - database.streamUnconfirmedBlobsSidecars(from, to)) { - out.putNextEntry(new ZipEntry("unconfirmed.txt")); - for (final Iterator iterator = unconfirmedBlobs.iterator(); - iterator.hasNext(); ) { - final SlotAndBlockRoot entryData = iterator.next(); - out.write(entryData.toLogString().getBytes(StandardCharsets.UTF_8)); - if (iterator.hasNext()) { - out.write("\n".getBytes(StandardCharsets.UTF_8)); - } - index++; - } - } - } - - System.out.println( - "Wrote " + index + " unconfirmed blobs sidecars to " + outputFile.toAbsolutePath()); - return 0; - } - - @Command( - name = "validate-blobs-sidecars", - description = "Validate the blobs sidecars", - mixinStandardHelpOptions = true, - showDefaultValues = true, - abbreviateSynopsis = true, - versionProvider = PicoCliVersionProvider.class, - synopsisHeading = "%n", - descriptionHeading = "%nDescription:%n%n", - optionListHeading = "%nOptions:%n", - footerHeading = "%n", - footer = "Teku is licensed under the Apache License 2.0") - public int validateBlobsSidecars( - @Mixin final BeaconNodeDataOptions beaconNodeDataOptions, - @Mixin final Eth2NetworkOptions eth2NetworkOptions, - @Option( - required = true, - names = {"--current-slot", "-s"}, - description = - "Provide the current slot from which calculate Data Availability window") - final Long currentSlot) - throws Exception { - - final Spec spec = eth2NetworkOptions.getNetworkConfiguration().getSpec(); - - final UInt64 denebActivationEpoch = - spec.forMilestone(SpecMilestone.DENEB) - .getConfig() - .toVersionDeneb() - .orElseThrow() - .getDenebForkEpoch(); - - final UInt64 currentEpoch = - eth2NetworkOptions - .getNetworkConfiguration() - .getSpec() - .computeEpochAtSlot(UInt64.valueOf(currentSlot)); - - final UInt64 lowerEpochBoundDataAvailability = - currentEpoch.minusMinZero(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS).max(denebActivationEpoch); - - final UInt64 lowerSlotBoundDataAvailability = - eth2NetworkOptions - .getNetworkConfiguration() - .getSpec() - .computeStartSlotAtEpoch(lowerEpochBoundDataAvailability); - - long missingFinalizedBlobsSidecars = 0; - long missingHotBlobsSidecars = 0; - long orphanBlobsSidecars = 0; - long unconfirmedBlobsSidecarsWithBlocks = 0; - long unprunedBlobsSidecars = 0; - long unprunedUnconfirmedBlobsSidecarsNonFinalized = 0; - long unprunedUnconfirmedBlobsSidecarsFinalized = 0; - long orphanUnconfirmations = 0; - long blocksToSidecarsCheckCounter = 0; - long sidecarsToBlocksCheckCounter = 0; - - final long startTimeMillis = System.currentTimeMillis(); - - Optional maybeEarliestBlobsSidecarSlot = Optional.empty(); - Optional maybeLatestFinalizedSlot = Optional.empty(); - Optional maybeEarliestBlock = Optional.empty(); - try (final Database database = createDatabase(beaconNodeDataOptions, eth2NetworkOptions)) { - - // checking blobs for finalized blocks - - Optional maybeFinalizedBlock = database.getLastAvailableFinalizedBlock(); - maybeEarliestBlock = database.getEarliestAvailableBlock(); - maybeEarliestBlobsSidecarSlot = database.getEarliestBlobsSidecarSlot(); - maybeLatestFinalizedSlot = maybeFinalizedBlock.map(SignedBeaconBlock::getSlot); - while (maybeFinalizedBlock.isPresent()) { - final SignedBeaconBlock currentBlock = maybeFinalizedBlock.get(); - if (blocksToSidecarsCheckCounter == 0) { - System.out.printf( - "Checking blobs sidecars tracing chain from best finalized block %s (%s)%n", - currentBlock.getRoot(), currentBlock.getSlot()); - } - - // check blobs sidecar existence - final SlotAndBlockRoot slotAndBlockRoot = - new SlotAndBlockRoot(currentBlock.getSlot(), currentBlock.getRoot()); - - final Optional maybeCurrentBlobsSidecar; - if (maybeEarliestBlobsSidecarSlot.isEmpty() - || maybeEarliestBlobsSidecarSlot.get().isGreaterThan(currentBlock.getSlot())) { - maybeCurrentBlobsSidecar = Optional.empty(); - } else { - maybeCurrentBlobsSidecar = database.getBlobsSidecar(slotAndBlockRoot); - } - - if (maybeCurrentBlobsSidecar.isEmpty() - && currentBlock.getSlot().isGreaterThanOrEqualTo(lowerSlotBoundDataAvailability)) { - // inside DA window but missing - missingFinalizedBlobsSidecars++; - System.err.printf( - "ERROR: Unable to locate finalized blobs sidecar %s%n", - slotAndBlockRoot.toLogString()); - } - - final Optional maybeParent = - database.getSignedBlock(currentBlock.getParentRoot()); - if (maybeParent.isEmpty()) { - if (maybeEarliestBlock.isPresent() - && maybeEarliestBlock.get().getRoot().equals(currentBlock.getRoot())) { - System.out.printf( - "Found back to earliest block %s (%s)%n", - currentBlock.getRoot(), currentBlock.getSlot()); - break; - } - } else { - checkFinalizedIndices(database, maybeParent.get()); - maybeFinalizedBlock = maybeParent; - } - blocksToSidecarsCheckCounter++; - if (blocksToSidecarsCheckCounter % 5_000 == 0) { - System.out.printf("%s (%s)...%n", currentBlock.getRoot(), currentBlock.getSlot()); - } - } - - // checking blobs sidecars for hot blocks - - try (final Stream> blockStream = database.streamHotBlocksAsSsz()) { - // Iterate through hot blocks - System.out.printf( - "Checking blobs sidecars excess with respect to finalized or hot blocks block...%n"); - - for (final Iterator> iterator = blockStream.iterator(); - iterator.hasNext(); ) { - blocksToSidecarsCheckCounter++; - - final Map.Entry rootAndBlock = iterator.next(); - final Bytes blockData = rootAndBlock.getValue(); - final UInt64 blockSlot = BeaconBlockInvariants.extractSignedBeaconBlockSlot(blockData); - - // check blobs sidecar existence - final SlotAndBlockRoot slotAndBlockRoot = - new SlotAndBlockRoot(blockSlot, Bytes32.wrap(rootAndBlock.getKey())); - final Optional maybeCurrentBlobsSidecar = - database.getBlobsSidecar(slotAndBlockRoot); - if (maybeCurrentBlobsSidecar.isEmpty() - && blockSlot.isGreaterThanOrEqualTo(lowerSlotBoundDataAvailability)) { - // inside DA window but missing - missingFinalizedBlobsSidecars++; - System.err.printf( - "ERROR: Unable to locate hot blobs sidecar %s%n", slotAndBlockRoot.toLogString()); - } - } - } - - final ArrayList unconfirmedBlobsSidecars = new ArrayList<>(); - - // collect unconfirmed blobs sidecar - try (final Stream slotAndBlockRootStream = - database.streamUnconfirmedBlobsSidecars(UInt64.ZERO, UInt64.MAX_VALUE)) { - slotAndBlockRootStream.forEach(unconfirmedBlobsSidecars::add); - } - - // calculate non-pruned or orphaned blobs sidecars - try (final Stream blobsSidecarsKeys = - database.streamBlobsSidecarKeys(UInt64.ZERO, UInt64.MAX_VALUE)) { - // Iterate through hot blocks - for (final Iterator iterator = blobsSidecarsKeys.iterator(); - iterator.hasNext(); ) { - final SlotAndBlockRoot blobsSidecarsKey = iterator.next(); - - sidecarsToBlocksCheckCounter++; - - final boolean isUnconfirmed = unconfirmedBlobsSidecars.contains(blobsSidecarsKey); - - final Optional maybeFinalizedBlockFound = - database - .getFinalizedBlockAtSlot(blobsSidecarsKey.getSlot()) - .filter(block -> block.getRoot().equals(blobsSidecarsKey.getBlockRoot())); - - final Optional maybeHotBlockFound = - database.getHotBlock(blobsSidecarsKey.getBlockRoot()); - - if (isUnconfirmed) { - // unconfirmed, check a corresponding hot or finalized block doesn't exist - - if (maybeHotBlockFound.isPresent()) { - unconfirmedBlobsSidecarsWithBlocks++; - System.err.printf( - "ERROR: unconfirmed blobs sidecar has a corresponding hot block %s%n", - blobsSidecarsKey.toLogString()); - } else if (maybeFinalizedBlockFound.isPresent()) { - unconfirmedBlobsSidecarsWithBlocks++; - System.err.printf( - "ERROR: unconfirmed blobs sidecar has a corresponding finalized block %s%n", - blobsSidecarsKey.toLogString()); - } - } else { - // is confirmed, hot or finalized blocks must exist - - if (maybeHotBlockFound.isEmpty() && maybeFinalizedBlockFound.isEmpty()) { - - orphanBlobsSidecars++; - - System.err.printf( - "ERROR: confirmed blobs sidecar has no corresponding finalized or hot block %s%n", - blobsSidecarsKey.toLogString()); - } - } - - // check Data Availability - if (blobsSidecarsKey.getSlot().isLessThan(lowerSlotBoundDataAvailability)) { - unprunedBlobsSidecars++; - System.err.printf( - "WARNING: non pruned blobs sidecar %s%n", blobsSidecarsKey.toLogString()); - } - } - } - - // check that unconfirmed blobs sidecar exists in the main blobs sidecar table - System.out.printf("Checking unconfirmed blobs sidecar in main blobs sidecar table...%n"); - - for (final SlotAndBlockRoot slotAndBlockRoot : unconfirmedBlobsSidecars) { - if (maybeLatestFinalizedSlot - .map(slot -> slotAndBlockRoot.getSlot().isGreaterThan(slot)) - .orElse(false)) { - // unconfirmed sidecars for a non finalized slot - unprunedUnconfirmedBlobsSidecarsNonFinalized++; - System.err.printf( - "WARNING: non pruned unconfirmed hot blobs sidecar %s%n", - slotAndBlockRoot.toLogString()); - - } else { - // unconfirmed sidecars for finalized slot - unprunedUnconfirmedBlobsSidecarsFinalized++; - System.err.printf( - "WARNING: non pruned unconfirmed finalized blobs sidecar %s%n", - slotAndBlockRoot.toLogString()); - } - - if (database.getBlobsSidecar(slotAndBlockRoot).isEmpty()) { - orphanUnconfirmations++; - System.err.printf( - "ERROR: Unable to locate unconfirmed blobs sidecar in main blobs sidecar table %s%n", - slotAndBlockRoot.toLogString()); - } - } - } catch (DatabaseStorageException ex) { - System.out.println("Failed to open database"); + System.out.println("Wrote " + index + " blob sidecars to " + outputFile.toAbsolutePath()); } - - final long endTimeMillis = System.currentTimeMillis(); - System.out.printf( - "Done. Performed %s block->blobs_sidecars and %s blobs_sidecars->block checks in %s ms%n", - blocksToSidecarsCheckCounter, - sidecarsToBlocksCheckCounter, - endTimeMillis - startTimeMillis); - System.out.printf( - "\tCalculated Data Availability window (slots): %s - %s%n", - lowerSlotBoundDataAvailability, currentSlot); - System.out.printf("\tEarliest BlobsSidecar slot: %s%n", maybeEarliestBlobsSidecarSlot); - System.out.printf( - "\tEarliest finalized block slot: %s%n", - maybeEarliestBlock.map(SignedBeaconBlock::getSlot)); - System.out.printf("\tLatest finalized block slot: %s%n", maybeLatestFinalizedSlot); - System.out.printf("\tMissing finalized BlobsSidecars: %s%n", missingFinalizedBlobsSidecars); - System.out.printf("\tMissing hot BlobsSidecars: %s%n", missingHotBlobsSidecars); - System.out.printf( - "\tUnconfirmed BlobsSidecars with blocks (should have been confirmed): %s%n", - unconfirmedBlobsSidecarsWithBlocks); - System.out.printf( - "\tOrphaned BlobsSidecars (corresponding block is missing): %s%n", orphanBlobsSidecars); - System.out.printf( - "\tNon-pruned BlobsSidecars (older than DA lower bound): %s%n", unprunedBlobsSidecars); - System.out.printf( - "\tNon-pruned unconfirmed Hot BlobsSidecars (related to non-imported blocks): %s%n", - unprunedUnconfirmedBlobsSidecarsNonFinalized); - System.out.printf( - "\tNon-pruned unconfirmed Finalized BlobsSidecars (related to non-imported blocks): %s%n", - unprunedUnconfirmedBlobsSidecarsFinalized); - System.out.printf( - "\tOrphaned unconfirmations of BlobsSidecars (corresponding BlobsSidecars is missing): %s%n", - orphanUnconfirmations); - return 0; } From 26b55cec86ced9d5aa1948a1b78c942365be305d Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 28 Apr 2023 15:08:46 +0400 Subject: [PATCH 2/5] Fix pruned could be > pruneLimit by design --- .../pegasys/teku/storage/server/kvstore/KvStoreDatabase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java index 85ae706fe3b..3711be9d75c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java @@ -812,7 +812,7 @@ public boolean pruneOldestBlobSidecars(final UInt64 lastSlotToPrune, final int p } updater.commit(); - return pruned == pruneLimit; + return pruned >= pruneLimit; } } From 52306bb5871125a96c629acfec09892a07644550 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 28 Apr 2023 15:24:06 +0400 Subject: [PATCH 3/5] Clarify some logic complexity --- .../pegasys/teku/storage/server/kvstore/KvStoreDatabase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java index 3711be9d75c..e73a9a9cfc2 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java @@ -804,6 +804,7 @@ public boolean pruneOldestBlobSidecars(final UInt64 lastSlotToPrune, final int p --remaining; final boolean finished = remaining < 0; final SlotAndBlockRootAndBlobIndex key = it.next(); + // Before we finish we should check that there are no BlobSidecars left in the same slot if (finished && (key.isNoBlobsKey() || key.getBlobIndex().equals(ZERO))) { break; } @@ -812,6 +813,7 @@ public boolean pruneOldestBlobSidecars(final UInt64 lastSlotToPrune, final int p } updater.commit(); + // `pruned` will be greater when we reach pruneLimit not on the latest BlobSidecar in a slot return pruned >= pruneLimit; } } From a6bc68d8f8aa63a46959678041e096573a2561ba Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 28 Apr 2023 15:33:21 +0400 Subject: [PATCH 4/5] Simplify and skip db lookup --- .../pegasys/teku/storage/server/kvstore/KvStoreDatabase.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java index e73a9a9cfc2..fca83457deb 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java @@ -775,11 +775,10 @@ public void storeNoBlobsSlot(final SlotAndBlockRoot slotAndBlockRoot) { @Override public Optional getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { - final Optional maybePayload = dao.getBlobSidecar(key); - if (maybePayload.filter(Bytes::isEmpty).isPresent()) { - // no BlobSidecars slot entry + if (key.isNoBlobsKey()) { return Optional.empty(); } + final Optional maybePayload = dao.getBlobSidecar(key); return maybePayload.map(payload -> spec.deserializeBlobSidecar(payload, key.getSlot())); } From 16c80bba9323bab7234283dc52081c85ea6956c8 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 28 Apr 2023 17:30:53 +0400 Subject: [PATCH 5/5] add earliest blob sidecar slot metrics --- .../teku/storage/server/pruner/BlobSidecarPruner.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java index 2d437fc1266..c28107f6d4d 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java @@ -49,6 +49,7 @@ public class BlobSidecarPruner extends Service { private Optional genesisTime = Optional.empty(); private final AtomicLong blobColumnSize = new AtomicLong(0); + private final AtomicLong earliestBlobSidecarSlot = new AtomicLong(-1); public BlobSidecarPruner( final Spec spec, @@ -71,11 +72,12 @@ public BlobSidecarPruner( LabelledGauge labelledGauge = metricsSystem.createLabelledGauge( TekuMetricCategory.STORAGE, - "blob_sidecar_counts", - "Number of blob sidecars stored", + "blob_sidecars", + "Statistics for BlobSidecars stored", "type"); labelledGauge.labels(blobColumnSize::get, "total"); + labelledGauge.labels(earliestBlobSidecarSlot::get, "earliest_slot"); } } @@ -102,6 +104,8 @@ private void pruneBlobs() { if (blobSidecarStorageCountersEnabled) { blobColumnSize.set(database.getBlobSidecarColumnCount()); + earliestBlobSidecarSlot.set( + database.getEarliestBlobSidecarSlot().map(UInt64::longValue).orElse(-1L)); } }