Merge branch 'dev' into lc-prooftest

README.md

@@ -2,7 +2,7 @@
 
 [![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA) [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-To learn more about proof-of-stake and sharding, see the [PoS FAQ](https://eth.wiki/en/concepts/proof-of-stake-faqs), [sharding FAQ](https://eth.wiki/sharding/Sharding-FAQs) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm).
+To learn more about proof-of-stake and sharding, see the [PoS documentation](https://ethereum.org/en/developers/docs/consensus-mechanisms/pos/), [sharding documentation](https://ethereum.org/en/upgrades/sharding/) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm).
 
 This repository hosts the current Ethereum proof-of-stake specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed-upon changes to the spec can be made through pull requests.
 

setup.py

@@ -232,7 +232,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->
 
                     if not _is_constant_id(name):
                         # Check for short type declarations
-                        if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List")):
+                        if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List", "ByteVector")):
                             custom_types[name] = value
                         continue
 
@@ -590,7 +590,6 @@ def imports(cls, preset_name: str):
         return super().imports(preset_name) + f'''
 from eth2spec.utils import kzg
 from eth2spec.bellatrix import {preset_name} as bellatrix
-from eth2spec.utils.ssz.ssz_impl import serialize as ssz_serialize
 '''
 
 
@@ -617,12 +616,13 @@ def sundry_functions(cls) -> str:
 ROOTS_OF_UNITY = kzg.compute_roots_of_unity(TESTING_FIELD_ELEMENTS_PER_BLOB)
 
 
-def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> BlobsSidecar:
-    pass'''
+def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> Optional[BlobsSidecar]:
+    return "TEST"'''
 
     @classmethod
     def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
         constants = {
+            'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
             'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
             'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
         }
@@ -1131,7 +1131,7 @@ def run(self):
         "pycryptodome==3.15.0",
         "py_ecc==6.0.0",
         "milagro_bls_binding==1.9.0",
-        "remerkleable==0.1.24",
+        "remerkleable==0.1.25",
         RUAMEL_YAML_VERSION,
         "lru-dict==1.1.8",
         MARKO_VERSION,

specs/altair/light-client/full-node.md

@@ -84,13 +84,13 @@ def create_light_client_update(state: BeaconState,
     header = state.latest_block_header.copy()
     header.state_root = hash_tree_root(state)
     assert hash_tree_root(header) == hash_tree_root(block.message)
-    update_signature_period = compute_sync_committee_period(compute_epoch_at_slot(block.message.slot))
+    update_signature_period = compute_sync_committee_period_at_slot(block.message.slot)
 
     assert attested_state.slot == attested_state.latest_block_header.slot
     attested_header = attested_state.latest_block_header.copy()
     attested_header.state_root = hash_tree_root(attested_state)
     assert hash_tree_root(attested_header) == block.message.parent_root
-    update_attested_period = compute_sync_committee_period(compute_epoch_at_slot(attested_header.slot))
+    update_attested_period = compute_sync_committee_period_at_slot(attested_header.slot)
 
     # `next_sync_committee` is only useful if the message is signed by the current sync committee
     if update_attested_period == update_signature_period:
@@ -133,7 +133,7 @@ def create_light_client_update(state: BeaconState,
 Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to `is_better_update`) for each sync committee period covering any epochs in range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientUpdate` for other sync committee periods.
 
 - `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.slot`
-- `LightClientUpdate` are only considered if `compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) == compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot))`
+- `LightClientUpdate` are only considered if `compute_sync_committee_period_at_slot(update.attested_header.slot) == compute_sync_committee_period_at_slot(update.signature_slot)`
 - Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time.
 
 ### `create_light_client_finality_update`

specs/altair/light-client/p2p-interface.md

@@ -71,6 +71,17 @@ For light clients, the following validations MUST additionally pass before forwa
 
 Light clients SHOULD call `process_light_client_finality_update` even if the message is ignored.
 
+The gossip `ForkDigest`-context is determined based on `compute_fork_version(compute_epoch_at_slot(finality_update.attested_header.slot))`.
+
+Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:
+
+[0]: # (eth2spec: skip)
+
+| `fork_version`                  | Message SSZ type                     |
+| ------------------------------- | ------------------------------------ |
+| `GENESIS_FORK_VERSION`          | n/a                                  |
+| `ALTAIR_FORK_VERSION` and later | `altair.LightClientFinalityUpdate`   |
+
 ###### `light_client_optimistic_update`
 
 This topic is used to propagate the latest `LightClientOptimisticUpdate` to light clients, allowing them to keep track of the latest `optimistic_header`.
@@ -88,6 +99,17 @@ For light clients, the following validations MUST additionally pass before forwa
 
 Light clients SHOULD call `process_light_client_optimistic_update` even if the message is ignored.
 
+The gossip `ForkDigest`-context is determined based on `compute_fork_version(compute_epoch_at_slot(optimistic_update.attested_header.slot))`.
+
+Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:
+
+[0]: # (eth2spec: skip)
+
+| `fork_version`                  | Message SSZ type                     |
+| ------------------------------- | ------------------------------------ |
+| `GENESIS_FORK_VERSION`          | n/a                                  |
+| `ALTAIR_FORK_VERSION` and later | `altair.LightClientOptimisticUpdate` |
+
 ### The Req/Resp domain
 
 #### Messages

specs/altair/light-client/sync-protocol.md

@@ -77,9 +77,9 @@ Additional documents describe how the light client sync protocol can be used:
 
 ```python
 class LightClientBootstrap(Container):
-    # The requested beacon block header
+    # Header matching the requested beacon block root
     header: BeaconBlockHeader
-    # Current sync committee corresponding to `header`
+    # Current sync committee corresponding to `header.state_root`
     current_sync_committee: SyncCommittee
     current_sync_committee_branch: Vector[Bytes32, floorlog2(CURRENT_SYNC_COMMITTEE_INDEX)]
 ```
@@ -88,12 +88,12 @@ class LightClientBootstrap(Container):
 
 ```python
 class LightClientUpdate(Container):
-    # The beacon block header that is attested to by the sync committee
+    # Header attested to by the sync committee
     attested_header: BeaconBlockHeader
-    # Next sync committee corresponding to `attested_header`
+    # Next sync committee corresponding to `attested_header.state_root`
     next_sync_committee: SyncCommittee
     next_sync_committee_branch: Vector[Bytes32, floorlog2(NEXT_SYNC_COMMITTEE_INDEX)]
-    # The finalized beacon block header attested to by Merkle branch
+    # Finalized header corresponding to `attested_header.state_root`
     finalized_header: BeaconBlockHeader
     finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)]
     # Sync committee aggregate signature
@@ -106,9 +106,9 @@ class LightClientUpdate(Container):
 
 ```python
 class LightClientFinalityUpdate(Container):
-    # The beacon block header that is attested to by the sync committee
+    # Header attested to by the sync committee
     attested_header: BeaconBlockHeader
-    # The finalized beacon block header attested to by Merkle branch
+    # Finalized header corresponding to `attested_header.state_root`
     finalized_header: BeaconBlockHeader
     finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)]
     # Sync committee aggregate signature
@@ -121,7 +121,7 @@ class LightClientFinalityUpdate(Container):
 
 ```python
 class LightClientOptimisticUpdate(Container):
-    # The beacon block header that is attested to by the sync committee
+    # Header attested to by the sync committee
     attested_header: BeaconBlockHeader
     # Sync committee aggregate signature
     sync_aggregate: SyncAggregate
@@ -134,9 +134,9 @@ class LightClientOptimisticUpdate(Container):
 ```python
 @dataclass
 class LightClientStore(object):
-    # Beacon block header that is finalized
+    # Header that is finalized
     finalized_header: BeaconBlockHeader
-    # Sync committees corresponding to the header
+    # Sync committees corresponding to the finalized header
     current_sync_committee: SyncCommittee
     next_sync_committee: SyncCommittee
     # Best available header to switch finalized head to if we see nothing else

specs/bellatrix/p2p-interface.md

@@ -110,7 +110,7 @@ The following gossip validation from prior specifications MUST NOT be applied if
 ### Transitioning the gossip
 
 See gossip transition details found in the [Altair document](../altair/p2p-interface.md#transitioning-the-gossip) for
-details on how to handle transitioning gossip topics for EIP-4844.
+details on how to handle transitioning gossip topics.
 
 ## The Req/Resp domain
 

specs/capella/p2p-interface.md

@@ -0,0 +1,110 @@
+# Capella -- Networking
+
+This document contains the networking specification for Capella.
+
+The specification of these changes continues in the same format as the network specifications of previous upgrades, and assumes them as pre-requisite.
+
+## Table of contents
+
+<!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+- [Modifications in Capella](#modifications-in-capella)
+  - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub)
+    - [Topics and messages](#topics-and-messages)
+      - [Global topics](#global-topics)
+        - [`beacon_block`](#beacon_block)
+        - [`bls_to_execution_change`](#bls_to_execution_change)
+    - [Transitioning the gossip](#transitioning-the-gossip)
+  - [The Req/Resp domain](#the-reqresp-domain)
+    - [Messages](#messages)
+      - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2)
+      - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+<!-- /TOC -->
+
+
+# Modifications in Capella
+
+## The gossip domain: gossipsub
+
+A new topic is added to support the gossip of withdrawal credential change messages. And an existing topic is upgraded for updated types in Capella.
+
+### Topics and messages
+
+Topics follow the same specification as in prior upgrades. All existing topics remain stable except the beacon block topic which is updated with the modified type.
+
+The new topics along with the type of the `data` field of a gossipsub message are given in this table:
+
+| Name | Message Type |
+| - | - |
+| `beacon_block` | `SignedBeaconBlock` (modified) |
+| `bls_to_execution_change` | `SignedBLSToExecutionChange` |
+
+Note that the `ForkDigestValue` path segment of the topic separates the old and the new `beacon_block` topics.
+
+#### Global topics
+
+Capella changes the type of the global beacon block topic and adds one global topic to propagate withdrawal credential change messages to all potential proposers of beacon blocks.
+
+##### `beacon_block`
+
+The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in Capella.
+Specifically, this type changes with the addition of `bls_to_execution_changes` to the inner `BeaconBlockBody`.
+See Capella [state transition document](./beacon-chain.md#beaconblockbody) for further details.
+
+##### `bls_to_execution_change`
+
+This topic is used to propagate signed bls to execution change messages to be included in future blocks.
+
+The following validations MUST pass before forwarding the `signed_bls_to_execution_change` on the network:
+
+- _[IGNORE]_ The `signed_bls_to_execution_change` is the first valid signed bls to execution change received
+  for the validator with index `signed_bls_to_execution_change.message.validator_index`.
+- _[REJECT]_ All of the conditions within `process_bls_to_execution_change` pass validation.
+
+### Transitioning the gossip
+
+See gossip transition details found in the [Altair document](../altair/p2p-interface.md#transitioning-the-gossip) for
+details on how to handle transitioning gossip topics for Capella.
+
+## The Req/Resp domain
+
+### Messages
+
+#### BeaconBlocksByRange v2
+
+**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/2/`
+
+The Capella fork-digest is introduced to the `context` enum to specify Capella block type.
+
+Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:
+
+[0]: # (eth2spec: skip)
+
+| `fork_version`           | Chunk SSZ type             |
+| ------------------------ | -------------------------- |
+| `GENESIS_FORK_VERSION`   | `phase0.SignedBeaconBlock` |
+| `ALTAIR_FORK_VERSION`    | `altair.SignedBeaconBlock` |
+| `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` |
+| `CAPELLA_FORK_VERSION`   | `capella.SignedBeaconBlock` |
+
+#### BeaconBlocksByRoot v2
+
+**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/`
+
+The Capella fork-digest is introduced to the `context` enum to specify Capella block type.
+
+Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:
+
+[1]: # (eth2spec: skip)
+
+| `fork_version`           | Chunk SSZ type             |
+| ------------------------ | -------------------------- |
+| `GENESIS_FORK_VERSION`   | `phase0.SignedBeaconBlock` |
+| `ALTAIR_FORK_VERSION`    | `altair.SignedBeaconBlock` |
+| `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` |
+| `CAPELLA_FORK_VERSION`   | `capella.SignedBeaconBlock` |
+

specs/capella/validator.md

@@ -18,6 +18,7 @@
   - [Block proposal](#block-proposal)
     - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody)
       - [ExecutionPayload](#executionpayload)
+      - [BLS to execution changes](#bls-to-execution-changes)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->
@@ -106,3 +107,7 @@ def prepare_execution_payload(state: BeaconState,
         payload_attributes=payload_attributes,
     )
 ```
+
+##### BLS to execution changes
+
+Up to `MAX_BLS_TO_EXECUTION_CHANGES`, [`BLSToExecutionChange`](./beacon-chain.md#blstoexecutionchange) objects can be included in the `block`. The BLS to execution changes must satisfy the verification conditions found in [BLS to execution change processing](./beacon-chain.md#new-process_bls_to_execution_change).

specs/eip4844/beacon-chain.md

@@ -23,6 +23,8 @@
     - [`ExecutionPayloadHeader`](#executionpayloadheader)
 - [Helper functions](#helper-functions)
   - [Misc](#misc)
+    - [`validate_blobs_sidecar`](#validate_blobs_sidecar)
+    - [`is_data_available`](#is_data_available)
     - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash)
     - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes)
     - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions)
@@ -44,9 +46,7 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844.
 
 | Name | SSZ equivalent | Description |
 | - | - | - |
-| `Blob` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_BLOB]` | |
 | `VersionedHash` | `Bytes32` | |
-| `KZGCommitment` | `Bytes48` | Same as BLS standard "is valid pubkey" check but also allows `0x00..00` for point-at-infinity |
 
 ## Constants
 
@@ -55,7 +55,6 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844.
 | Name | Value |
 | - | - |
 | `BLOB_TX_TYPE` | `uint8(0x05)` |
-| `FIELD_ELEMENTS_PER_BLOB` | `uint64(4096)` |
 | `VERSIONED_HASH_VERSION_KZG` | `Bytes1(0x01)` | 
 
 ### Domain types
@@ -150,6 +149,43 @@ class ExecutionPayloadHeader(Container):
 
 ### Misc
 
+#### `validate_blobs_sidecar`
+
+```python
+def validate_blobs_sidecar(slot: Slot,
+                           beacon_block_root: Root,
+                           expected_kzg_commitments: Sequence[KZGCommitment],
+                           blobs_sidecar: BlobsSidecar) -> None:
+    assert slot == blobs_sidecar.beacon_block_slot
+    assert beacon_block_root == blobs_sidecar.beacon_block_root
+    blobs = blobs_sidecar.blobs
+    kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof
+    assert len(expected_kzg_commitments) == len(blobs)
+
+    assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof)
+```
+
+#### `is_data_available`
+
+The implementation of `is_data_available` is meant to change with later sharding upgrades.
+Initially, it requires every verifying actor to retrieve the matching `BlobsSidecar`,
+and validate the sidecar with `validate_blobs_sidecar`.
+
+Without the sidecar the block may be processed further optimistically,
+but MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded.
+
+```python
+def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool:
+    # `retrieve_blobs_sidecar` is implementation dependent, raises an exception if not available.
+    sidecar = retrieve_blobs_sidecar(slot, beacon_block_root)
+    if sidecar == "TEST":
+        return True  # For testing; remove once we have a way to inject `BlobsSidecar` into tests
+    validate_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar)
+
+    return True
+```
+
+
 #### `kzg_commitment_to_versioned_hash`
 
 ```python
@@ -204,6 +240,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None:
     process_operations(state, block.body)
     process_sync_aggregate(state, block.body.sync_aggregate)
     process_blob_kzg_commitments(state, block.body)  # [New in EIP-4844]
+
+    # New in EIP-4844, note: Can sync optimistically without this condition, see note on `is_data_available`
+    assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments)
 ```
 
 #### Execution payload