Skip to content

Commit

Permalink
Checkpoint Sync 1/5 - fork/version detection and unmarshaling support (…
Browse files Browse the repository at this point in the history 
…#10380)

* fork/version detection and unmarshaling support

* Update config/params/config.go

Co-authored-by: Radosław Kapka <[email protected]>

* Update proto/detect/configfork.go

Co-authored-by: Radosław Kapka <[email protected]>

* PR feedback

* move ssz initialization into the detect package

* clarify comment

* VersionForEpoch is much simpler/clearer in reverse

* simpler VersionForEpoch; build AllConfigs in init

* use fieldparams for Version

* Update proto/detect/configfork_test.go

Co-authored-by: Radosław Kapka <[email protected]>

* remove custom ForkName type, use runtime/version

* pr cleanup

* random fix from bad gh ui suggestion; privatize

* privatize fieldSpec methods; + unit tests

* Update proto/detect/configfork.go

Co-authored-by: Potuz <[email protected]>

* fix bad github ui suggestion

* ensure unique versions for simpler config match

* fmt & adding unit test for ByState()

* table-driven unit test for ByState

* TestUnmarshalState

* OrderedSchedule -> network/forks per PR feedback

* goimports

* lint fixes

* move proto/detect -> ssz/encoding/detect

* use typeUndefined in String

* backport config tests from e2e PR

* fix config parity test; make debugging it easier

* lint

* fix fork schedule initialization

* cleanup

* fix build

* fix big ole derp

* anything for you, deep source

* goimportsss

* InitializeForkSchedule in LoadChainConfigFile

* PR feedback

Co-authored-by: kasey <[email protected]>
Co-authored-by: Radosław Kapka <[email protected]>
Co-authored-by: Potuz <[email protected]>
  • Loading branch information
@kasey @rkapka @potuz
4 people authored Mar 21, 2022
1 parent df8da80 commit cf0bd63
Show file tree
Hide file tree
Showing 34 changed files with 1,344 additions and 193 deletions.
1 change: 1 addition & 0 deletions beacon-chain/db/kv/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ go_library(
"//config/params:go_default_library",
"//container/slice:go_default_library",
"//encoding/bytesutil:go_default_library",
"//encoding/ssz/detect:go_default_library",
"//io/file:go_default_library",
"//monitoring/progress:go_default_library",
"//monitoring/tracing:go_default_library",
Expand Down
34 changes: 18 additions & 16 deletions beacon-chain/db/kv/wss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,44 +7,46 @@ import (

"github.com/pkg/errors"
types "github.com/prysmaticlabs/eth2-types"
statev2 "github.com/prysmaticlabs/prysm/beacon-chain/state/v2"
"github.com/prysmaticlabs/prysm/config/params"
"github.com/prysmaticlabs/prysm/encoding/ssz/detect"
ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
)

// SaveOrigin loads an ssz serialized Block & BeaconState from an io.Reader
// (ex: an open file) prepares the database so that the beacon node can begin
// syncing, using the provided values as their point of origin. This is an alternative
// to syncing from genesis, and should only be run on an empty database.
func (s *Store) SaveOrigin(ctx context.Context, stateReader, blockReader io.Reader) error {
// unmarshal both block and state before trying to save anything
// so that we fail early if there is any issue with the ssz data
blk := &ethpb.SignedBeaconBlockAltair{}
sb, err := ioutil.ReadAll(stateReader)
if err != nil {
return errors.Wrap(err, "failed to read origin state bytes")
}
bb, err := ioutil.ReadAll(blockReader)
if err != nil {
return errors.Wrap(err, "error reading block given to SaveOrigin")
}
if err := blk.UnmarshalSSZ(bb); err != nil {
return errors.Wrap(err, "could not unmarshal checkpoint block")

cf, err := detect.FromState(sb)
if err != nil {
return errors.Wrap(err, "failed to detect config and fork for origin state")
}
wblk, err := wrapper.WrappedAltairSignedBeaconBlock(blk)
bs, err := cf.UnmarshalBeaconState(sb)
if err != nil {
return errors.Wrap(err, "could not wrap checkpoint block")
return errors.Wrap(err, "could not unmarshal origin state")
}
bs, err := statev2.InitializeFromSSZReader(stateReader)
wblk, err := cf.UnmarshalBeaconBlock(bb)
if err != nil {
return errors.Wrap(err, "could not initialize checkpoint state from reader")
return errors.Wrap(err, "unable to unmarshal origin SignedBeaconBlock")
}

blockRoot, err := wblk.Block().HashTreeRoot()
if err != nil {
return errors.Wrap(err, "could not compute HashTreeRoot of checkpoint block")
}
// save block
if err := s.SaveBlock(ctx, wblk); err != nil {
return errors.Wrap(err, "could not save checkpoint block")
}
blockRoot, err := blk.Block.HashTreeRoot()
if err != nil {
return errors.Wrap(err, "could not compute HashTreeRoot of checkpoint block")
}

// save state
if err = s.SaveState(ctx, bs, blockRoot); err != nil {
Expand All @@ -70,7 +72,7 @@ func (s *Store) SaveOrigin(ctx context.Context, stateReader, blockReader io.Read

// rebuild the checkpoint from the block
// use it to mark the block as justified and finalized
slotEpoch, err := blk.Block.Slot.SafeDivSlot(params.BeaconConfig().SlotsPerEpoch)
slotEpoch, err := wblk.Block().Slot().SafeDivSlot(params.BeaconConfig().SlotsPerEpoch)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion beacon-chain/node/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func configureTracing(cliCtx *cli.Context) error {
func configureChainConfig(cliCtx *cli.Context) {
if cliCtx.IsSet(cmd.ChainConfigFileFlag.Name) {
chainConfigFileName := cliCtx.String(cmd.ChainConfigFileFlag.Name)
params.LoadChainConfigFile(chainConfigFileName)
params.LoadChainConfigFile(chainConfigFileName, nil)
}
}

Expand Down
16 changes: 1 addition & 15 deletions beacon-chain/state/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,7 @@ go_library(
"prometheus.go",
],
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state",
visibility = [
"//beacon-chain:__subpackages__",
"//contracts/deposit:__subpackages__",
"//proto/migration:__pkg__",
"//proto/prysm/v1alpha1:__subpackages__",
"//proto/testing:__subpackages__",
"//slasher/rpc:__subpackages__",
"//testing/benchmark:__pkg__",
"//testing/slasher/simulator:__pkg__",
"//testing/spectest:__subpackages__",
"//testing/util:__pkg__",
"//tools/benchmark-files-gen:__pkg__",
"//tools/exploredb:__pkg__",
"//tools/pcli:__pkg__",
],
visibility = ["//visibility:public"],
deps = [
"//config/fieldparams:go_default_library",
"//proto/prysm/v1alpha1:go_default_library",
Expand Down
23 changes: 0 additions & 23 deletions beacon-chain/state/state-native/v2/state_trie.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package v2

import (
"context"
"io"
"io/ioutil"
"runtime"
"sort"

Expand All @@ -30,27 +28,6 @@ func InitializeFromProto(st *ethpb.BeaconStateAltair) (state.BeaconStateAltair,
return InitializeFromProtoUnsafe(proto.Clone(st).(*ethpb.BeaconStateAltair))
}

// InitializeFromSSZReader can be used when the source for a serialized BeaconState object
// is an io.Reader. This allows client code to remain agnostic about whether the data comes
// from the network or a file without needing to read the entire state into mem as a large byte slice.
func InitializeFromSSZReader(r io.Reader) (state.BeaconStateAltair, error) {
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
return InitializeFromSSZBytes(b)
}

// InitializeFromSSZBytes is a convenience method to obtain a BeaconState by unmarshaling
// a slice of bytes containing the ssz-serialized representation of the state.
func InitializeFromSSZBytes(marshaled []byte) (*BeaconState, error) {
st := &ethpb.BeaconStateAltair{}
if err := st.UnmarshalSSZ(marshaled); err != nil {
return nil, err
}
return InitializeFromProtoUnsafe(st)
}

// InitializeFromProtoUnsafe directly uses the beacon state protobuf fields
// and sets them as fields of the BeaconState type.
func InitializeFromProtoUnsafe(st *ethpb.BeaconStateAltair) (*BeaconState, error) {
Expand Down
15 changes: 1 addition & 14 deletions beacon-chain/state/v1/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,7 @@ go_library(
"unsupported_setters.go",
],
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/v1",
visibility = [
"//beacon-chain:__subpackages__",
"//contracts/deposit:__subpackages__",
"//proto/migration:__subpackages__",
"//proto/prysm/v1alpha1:__subpackages__",
"//proto/testing:__subpackages__",
"//runtime/interop:__subpackages__",
"//slasher/rpc:__subpackages__",
"//testing/benchmark:__pkg__",
"//testing/spectest:__subpackages__",
"//testing/util:__pkg__",
"//tools/benchmark-files-gen:__pkg__",
"//tools/pcli:__pkg__",
],
visibility = ["//visibility:public"],
deps = [
"//beacon-chain/state:go_default_library",
"//beacon-chain/state/fieldtrie:go_default_library",
Expand Down
7 changes: 1 addition & 6 deletions beacon-chain/state/v2/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,7 @@ go_library(
"types.go",
],
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/v2",
visibility = [
"//beacon-chain:__subpackages__",
"//proto/migration:__subpackages__",
"//testing/spectest:__subpackages__",
"//testing/util:__pkg__",
],
visibility = ["//visibility:public"],
deps = [
"//beacon-chain/state:go_default_library",
"//beacon-chain/state/fieldtrie:go_default_library",
Expand Down
29 changes: 0 additions & 29 deletions beacon-chain/state/v2/state_trie.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package v2

import (
"context"
"io"
"io/ioutil"
"runtime"
"sort"

Expand Down Expand Up @@ -33,33 +31,6 @@ func InitializeFromProto(st *ethpb.BeaconStateAltair) (state.BeaconStateAltair,
return InitializeFromProtoUnsafe(proto.Clone(st).(*ethpb.BeaconStateAltair))
}

// InitializeFromSSZReader can be used when the source for a serialized BeaconState object
// is an io.Reader. This allows client code to remain agnostic about whether the data comes
// from the network or a file without needing to read the entire state into mem as a large byte slice.
func InitializeFromSSZReader(r io.Reader) (state.BeaconStateAltair, error) {
if features.Get().EnableNativeState {
return statenative.InitializeFromSSZReader(r)
}
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
return InitializeFromSSZBytes(b)
}

// InitializeFromSSZBytes is a convenience method to obtain a BeaconState by unmarshaling
// a slice of bytes containing the ssz-serialized representation of the state.
func InitializeFromSSZBytes(marshaled []byte) (state.BeaconStateAltair, error) {
if features.Get().EnableNativeState {
return statenative.InitializeFromSSZBytes(marshaled)
}
st := &ethpb.BeaconStateAltair{}
if err := st.UnmarshalSSZ(marshaled); err != nil {
return nil, err
}
return InitializeFromProtoUnsafe(st)
}

// InitializeFromProtoUnsafe directly uses the beacon state protobuf pointer
// and sets it as the inner state of the BeaconState type.
func InitializeFromProtoUnsafe(st *ethpb.BeaconStateAltair) (state.BeaconStateAltair, error) {
Expand Down
6 changes: 1 addition & 5 deletions beacon-chain/state/v3/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,7 @@ go_library(
"types.go",
],
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/v3",
visibility = [
"//beacon-chain:__subpackages__",
"//testing/spectest:__subpackages__",
"//testing/util:__pkg__",
],
visibility = ["//visibility:public"],
deps = [
"//beacon-chain/state:go_default_library",
"//beacon-chain/state/fieldtrie:go_default_library",
Expand Down
1 change: 1 addition & 0 deletions config/fieldparams/mainnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,6 @@ const (
MaxBytesPerTxLength = 1073741824 // MaxBytesPerTxLength defines the maximum number of bytes that can be included in a transaction.
FeeRecipientLength = 20 // FeeRecipientLength defines the byte length of a fee recipient.
LogsBloomLength = 256 // LogsBloomLength defines the byte length of a logs bloom.
VersionLength = 4 // VersionLength defines the byte length of a fork version number.
EthBurnAddressHex = "0x0000000000000000000000000000000000000000" // EthBurnAddressHex defines the hex encoded address of the eth1.0 burn contract.
)
1 change: 1 addition & 0 deletions config/fieldparams/minimal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,6 @@ const (
MaxBytesPerTxLength = 1073741824 // MaxBytesPerTxLength defines the maximum number of bytes that can be included in a transaction.
FeeRecipientLength = 20 // FeeRecipientLength defines the byte length of a fee recipient.
LogsBloomLength = 256 // LogsBloomLength defines the byte length of a logs bloom.
VersionLength = 4 // VersionLength defines the byte length of a fork version number.
EthBurnAddressHex = "0x0000000000000000000000000000000000000000" // EthBurnAddressHex defines the hex encoded address of the eth1.0 burn contract.
)
2 changes: 2 additions & 0 deletions config/params/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ go_library(
importpath = "github.com/prysmaticlabs/prysm/config/params",
visibility = ["//visibility:public"],
deps = [
"//config/fieldparams:go_default_library",
"//encoding/bytesutil:go_default_library",
"//math:go_default_library",
"@com_github_ethereum_go_ethereum//common:go_default_library",
"@com_github_ethereum_go_ethereum//params:go_default_library",
"@com_github_mohae_deepcopy//:go_default_library",
"@com_github_pkg_errors//:go_default_library",
"@com_github_prysmaticlabs_eth2_types//:go_default_library",
"@com_github_sirupsen_logrus//:go_default_library",
"@in_gopkg_yaml_v2//:go_default_library",
Expand Down
30 changes: 18 additions & 12 deletions config/params/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/ethereum/go-ethereum/common"
types "github.com/prysmaticlabs/eth2-types"
fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
"github.com/prysmaticlabs/prysm/encoding/bytesutil"
)

Expand Down Expand Up @@ -135,14 +136,14 @@ type BeaconChainConfig struct {
SlashingProtectionPruningEpochs types.Epoch // SlashingProtectionPruningEpochs defines a period after which all prior epochs are pruned in the validator database.

// Fork-related values.
GenesisForkVersion []byte `yaml:"GENESIS_FORK_VERSION" spec:"true"` // GenesisForkVersion is used to track fork version between state transitions.
AltairForkVersion []byte `yaml:"ALTAIR_FORK_VERSION" spec:"true"` // AltairForkVersion is used to represent the fork version for altair.
AltairForkEpoch types.Epoch `yaml:"ALTAIR_FORK_EPOCH" spec:"true"` // AltairForkEpoch is used to represent the assigned fork epoch for altair.
BellatrixForkVersion []byte `yaml:"BELLATRIX_FORK_VERSION" spec:"true"` // BellatrixForkVersion is used to represent the fork version for bellatrix.
BellatrixForkEpoch types.Epoch `yaml:"BELLATRIX_FORK_EPOCH" spec:"true"` // BellatrixForkEpoch is used to represent the assigned fork epoch for bellatrix.
ShardingForkVersion []byte `yaml:"SHARDING_FORK_VERSION" spec:"true"` // ShardingForkVersion is used to represent the fork version for sharding.
ShardingForkEpoch types.Epoch `yaml:"SHARDING_FORK_EPOCH" spec:"true"` // ShardingForkEpoch is used to represent the assigned fork epoch for sharding.
ForkVersionSchedule map[[4]byte]types.Epoch // Schedule of fork epochs by version.
GenesisForkVersion []byte `yaml:"GENESIS_FORK_VERSION" spec:"true"` // GenesisForkVersion is used to track fork version between state transitions.
AltairForkVersion []byte `yaml:"ALTAIR_FORK_VERSION" spec:"true"` // AltairForkVersion is used to represent the fork version for altair.
AltairForkEpoch types.Epoch `yaml:"ALTAIR_FORK_EPOCH" spec:"true"` // AltairForkEpoch is used to represent the assigned fork epoch for altair.
BellatrixForkVersion []byte `yaml:"BELLATRIX_FORK_VERSION" spec:"true"` // BellatrixForkVersion is used to represent the fork version for bellatrix.
BellatrixForkEpoch types.Epoch `yaml:"BELLATRIX_FORK_EPOCH" spec:"true"` // BellatrixForkEpoch is used to represent the assigned fork epoch for bellatrix.
ShardingForkVersion []byte `yaml:"SHARDING_FORK_VERSION" spec:"true"` // ShardingForkVersion is used to represent the fork version for sharding.
ShardingForkEpoch types.Epoch `yaml:"SHARDING_FORK_EPOCH" spec:"true"` // ShardingForkEpoch is used to represent the assigned fork epoch for sharding.
ForkVersionSchedule map[[fieldparams.VersionLength]byte]types.Epoch // Schedule of fork epochs by version.

// Weak subjectivity values.
SafetyDecay uint64 // SafetyDecay is defined as the loss in the 1/3 consensus safety margin of the casper FFG mechanism.
Expand Down Expand Up @@ -193,11 +194,16 @@ type BeaconChainConfig struct {
// InitializeForkSchedule initializes the schedules forks baked into the config.
func (b *BeaconChainConfig) InitializeForkSchedule() {
// Reset Fork Version Schedule.
b.ForkVersionSchedule = map[[4]byte]types.Epoch{}
b.ForkVersionSchedule = configForkSchedule(b)
}

func configForkSchedule(b *BeaconChainConfig) map[[fieldparams.VersionLength]byte]types.Epoch {
fvs := map[[fieldparams.VersionLength]byte]types.Epoch{}
// Set Genesis fork data.
b.ForkVersionSchedule[bytesutil.ToBytes4(b.GenesisForkVersion)] = b.GenesisEpoch
fvs[bytesutil.ToBytes4(b.GenesisForkVersion)] = b.GenesisEpoch
// Set Altair fork data.
b.ForkVersionSchedule[bytesutil.ToBytes4(b.AltairForkVersion)] = b.AltairForkEpoch
fvs[bytesutil.ToBytes4(b.AltairForkVersion)] = b.AltairForkEpoch
// Set Bellatrix fork data.
b.ForkVersionSchedule[bytesutil.ToBytes4(b.BellatrixForkVersion)] = b.BellatrixForkEpoch
fvs[bytesutil.ToBytes4(b.BellatrixForkVersion)] = b.BellatrixForkEpoch
return fvs
}
Loading

0 comments on commit cf0bd63

Please sign in to comment.