Add Bitlist and Bitvector

scripts/build_spec.py

@@ -25,8 +25,8 @@
     signing_root,
 )
 from eth2spec.utils.ssz.ssz_typing import (
-    Bit, Bool, Container, List, Vector, Bytes, uint64,
-    Bytes4, Bytes32, Bytes48, Bytes96,
+    bit, boolean, Container, List, Vector, uint64,
+    Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector,
 )
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
@@ -52,8 +52,8 @@
     is_empty,
 )
 from eth2spec.utils.ssz.ssz_typing import (
-    Bit, Bool, Container, List, Vector, Bytes, uint64,
-    Bytes4, Bytes32, Bytes48, Bytes96,
+    bit, boolean, Container, List, Vector, Bytes, uint64,
+    Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector,
 )
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
@@ -174,8 +174,8 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st
 
 
 ignored_dependencies = [
-    'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN'
-    'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96',
+    'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN'
+    'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector',
     'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
     'bytes'  # to be removed after updating spec doc
 ]

specs/core/0_beacon-chain.md

@@ -80,8 +80,6 @@
         - [`bytes_to_int`](#bytes_to_int)
         - [`get_total_balance`](#get_total_balance)
         - [`get_domain`](#get_domain)
-        - [`get_bitfield_bit`](#get_bitfield_bit)
-        - [`verify_bitfield`](#verify_bitfield)
         - [`convert_to_indexed`](#convert_to_indexed)
         - [`validate_indexed_attestation`](#validate_indexed_attestation)
         - [`is_slashable_attestation_data`](#is_slashable_attestation_data)
@@ -191,6 +189,7 @@ The following values are (non-configurable) constants used throughout the specif
 | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) |
 | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) |
 | `SHUFFLE_ROUND_COUNT` | `90` |
+| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` |
 
 * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.)
 
@@ -297,7 +296,7 @@ class Validator(Container):
     pubkey: BLSPubkey
     withdrawal_credentials: Hash  # Commitment to pubkey for withdrawals and transfers
     effective_balance: Gwei  # Balance at stake
-    slashed: Bool
+    slashed: boolean
     # Status epochs
     activation_eligibility_epoch: Epoch  # When criteria for activation were met
     activation_epoch: Epoch
@@ -337,7 +336,7 @@ class AttestationData(Container):
 ```python
 class AttestationDataAndCustodyBit(Container):
     data: AttestationData
-    custody_bit: Bit  # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data
+    custody_bit: bit  # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data
 ```
 
 #### `IndexedAttestation`
@@ -354,7 +353,7 @@ class IndexedAttestation(Container):
 
 ```python
 class PendingAttestation(Container):
-    aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8]
+    aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]
     data: AttestationData
     inclusion_delay: Slot
     proposer_index: ValidatorIndex
@@ -421,9 +420,9 @@ class AttesterSlashing(Container):
 
 ```python
 class Attestation(Container):
-    aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8]
+    aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]
     data: AttestationData
-    custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8]
+    custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]
     signature: BLSSignature
 ```
 
@@ -525,7 +524,7 @@ class BeaconState(Container):
     previous_justified_root: Hash  # Previous epoch snapshot
     current_justified_epoch: Epoch
     current_justified_root: Hash
-    justification_bitfield: uint64  # Bit set for every recent justified epoch
+    justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]  # Bit set for every recent justified epoch
     # Finality
     finalized_epoch: Epoch
     finalized_root: Hash
@@ -865,13 +864,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S
 ```python
 def get_attesting_indices(state: BeaconState,
                           attestation_data: AttestationData,
-                          bitfield: bytes) -> Sequence[ValidatorIndex]:
+                          bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]:
     """
     Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``.
     """
     committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard)
-    assert verify_bitfield(bitfield, len(committee))
-    return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1])
+    return sorted([index for i, index in enumerate(committee) if bitfield[i]])
 ```
 
 ### `int_to_bytes`
@@ -912,34 +910,6 @@ def get_domain(state: BeaconState,
     return bls_domain(domain_type, fork_version)
 ```
 
-### `get_bitfield_bit`
-
-```python
-def get_bitfield_bit(bitfield: bytes, i: int) -> int:
-    """
-    Extract the bit in ``bitfield`` at position ``i``.
-    """
-    return (bitfield[i // 8] >> (i % 8)) % 2
-```
-
-### `verify_bitfield`
-
-```python
-def verify_bitfield(bitfield: bytes, committee_size: int) -> bool:
-    """
-    Verify ``bitfield`` against the ``committee_size``.
-    """
-    if len(bitfield) != (committee_size + 7) // 8:
-        return False
-
-    # Check `bitfield` is padded with zero bits only
-    for i in range(committee_size, len(bitfield) * 8):
-        if get_bitfield_bit(bitfield, i) == 0b1:
-            return False
-
-    return True
-```
-
 ### `convert_to_indexed`
 
 ```python
@@ -1322,38 +1292,40 @@ def process_justification_and_finalization(state: BeaconState) -> None:
     # Process justifications
     state.previous_justified_epoch = state.current_justified_epoch
     state.previous_justified_root = state.current_justified_root
-    state.justification_bitfield = (state.justification_bitfield << 1) % 2**64
+    state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \
+        state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1]
+    state.justification_bitfield[0] = 0b0
     previous_epoch_matching_target_balance = get_attesting_balance(
         state, get_matching_target_attestations(state, previous_epoch)
     )
     if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2:
         state.current_justified_epoch = previous_epoch
         state.current_justified_root = get_block_root(state, state.current_justified_epoch)
-        state.justification_bitfield |= (1 << 1)
+        state.justification_bitfield[1] = 0b1
     current_epoch_matching_target_balance = get_attesting_balance(
         state, get_matching_target_attestations(state, current_epoch)
     )
     if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2:
         state.current_justified_epoch = current_epoch
         state.current_justified_root = get_block_root(state, state.current_justified_epoch)
-        state.justification_bitfield |= (1 << 0)
+        state.justification_bitfield[0] = 0b1
 
     # Process finalizations
     bitfield = state.justification_bitfield
     # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source
-    if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch:
+    if all(bitfield[1:4]) and old_previous_justified_epoch + 3 == current_epoch:
         state.finalized_epoch = old_previous_justified_epoch
         state.finalized_root = get_block_root(state, state.finalized_epoch)
     # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source
-    if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch:
+    if all(bitfield[1:3]) and old_previous_justified_epoch + 2 == current_epoch:
         state.finalized_epoch = old_previous_justified_epoch
         state.finalized_root = get_block_root(state, state.finalized_epoch)
     # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source
-    if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch:
+    if all(bitfield[0:3]) and old_current_justified_epoch + 2 == current_epoch:
         state.finalized_epoch = old_current_justified_epoch
         state.finalized_root = get_block_root(state, state.finalized_epoch)
     # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source
-    if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch:
+    if all(bitfield[0:2]) and old_current_justified_epoch + 1 == current_epoch:
         state.finalized_epoch = old_current_justified_epoch
         state.finalized_root = get_block_root(state, state.finalized_epoch)
 ```

specs/core/1_custody-game.md

@@ -272,6 +272,16 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int:
     return crosslink_length * chunks_per_epoch
 ```
 
+### `get_bitfield_bit`
+
+```python
+def get_bitfield_bit(bitfield: bytes, i: int) -> int:
+    """
+    Extract the bit in ``bitfield`` at position ``i``.
+    """
+    return (bitfield[i // 8] >> (i % 8)) % 2
+```
+
 ### `get_custody_chunk_bit`
 
 ```python
@@ -564,9 +574,8 @@ def process_bit_challenge(state: BeaconState,
 
     # Verify the chunk count
     chunk_count = get_custody_chunk_count(attestation.data.crosslink)
-    assert verify_bitfield(challenge.chunk_bits, chunk_count)
     # Verify the first bit of the hash of the chunk bits does not equal the custody bit
-    custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index))
+    custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)]
     assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0)
     # Add new bit challenge record
     new_record = CustodyBitChallengeRecord(

specs/core/1_shard-data-chains.md

@@ -92,7 +92,7 @@ class ShardAttestation(Container):
         slot: Slot
         shard: Shard
         shard_block_root: Bytes32
-    aggregation_bitfield: Bytes[PLACEHOLDER]
+    aggregation_bitfield: Bitlist[PLACEHOLDER]
     aggregate_signature: BLSSignature
 ```
 
@@ -230,10 +230,9 @@ def verify_shard_attestation_signature(state: BeaconState,
                                        attestation: ShardAttestation) -> None:
     data = attestation.data
     persistent_committee = get_persistent_committee(state, data.shard, data.slot)
-    assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee))
     pubkeys = []
     for i, index in enumerate(persistent_committee):
-        if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1:
+        if attestation.aggregation_bitfield[i]:
             validator = state.validators[index]
             assert is_active_validator(validator, get_current_epoch(state))
             pubkeys.append(validator.pubkey)

specs/light_client/sync_protocol.md

@@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl
 {
     'header': BeaconBlockHeader,
     'shard_aggregate_signature': BLSSignature,
-    'shard_bitfield': 'bytes',
+    'shard_bitfield': Bitlist[PLACEHOLDER],
     'shard_parent_block': ShardBlock,
 }
 ```
@@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val
     assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header)
     committee = compute_committee(proof.header, validator_memory)
     # Verify that we have >=50% support
-    support_balance = sum([v.effective_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True])
+    support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]])
     total_balance = sum([v.effective_balance for i, v in enumerate(committee)])
     assert support_balance * 2 > total_balance
     # Verify shard attestations
     group_public_key = bls_aggregate_pubkeys([
         v.pubkey for v, index in enumerate(committee)
-        if get_bitfield_bit(proof.shard_bitfield, index) is True
+        if proof.shard_bitfield[index]
     ])
     assert bls_verify(
         pubkey=group_public_key,