State dump split (#2126)

- Revert binary state dump
- #1922 for separating config and records
- scripts/new-genesis-from-existing-state.sh that dump state, calculate new genesis hash, upload to s3
- testnet genesis records separate from near binary
- download testnet genesis from s3 in python start_testnet
- check genesis hash when run testnet
 
Co-authored-by: Evgeny Kuzyakov <[email protected]>
Co-authored-by: Bo Yao <[email protected]>

Test Plans
-------------
- it can `near init --chain-id=testnet --genesis-config near/res/testnet_genesis_config.json --genesis-records <records download from s3> --genesis-hash <expected-hash>` to initialize testnet config from external genesis records
- When `near run`, with incorrect `~/.near/genesis_hash` it will panic
- It can start testnet with updated start_testnet.py, which download genesis_records from s3
- After stop a node, we can call `state-viewer dump_genesis` to dump genesis_records, config, and genesis_hash.
- If there was no state change happen since genesis block, load the dump genesis_records/config will give you a same genesis_hash, and dump again generate same genesis_records, config & genesis_hash.

.gitattributes

@@ -1,3 +1,2 @@
 Cargo.lock linguist-generated=true -diff
 **/package-lock.json linguist-generated=true -diff
-near/res/testnet.json linguist-generated=true -diff

.gitignore

@@ -53,3 +53,7 @@ pytest/.adversary
 
 # a file indicating that the person understands the risks of running adversarial tests
 pytest/.consent
+
+# testnet genesis
+/near/res/testnet.json
+/near/res/testnet_genesis_records*.json

chain/client/src/test_utils.rs

@@ -102,6 +102,7 @@ pub fn setup(
         &chain_genesis,
         runtime.clone(),
         network_adapter.clone(),
+        None,
     )
     .unwrap();
     let config = ClientConfig::test(

chain/client/src/view_client.rs

@@ -52,9 +52,19 @@ impl ViewClientActor {
         chain_genesis: &ChainGenesis,
         runtime_adapter: Arc<dyn RuntimeAdapter>,
         network_adapter: Arc<dyn NetworkAdapter>,
+        expected_genesis_hash: Option<String>,
     ) -> Result<Self, Error> {
         // TODO: should we create shared ChainStore that is passed to both Client and ViewClient?
-        let chain = Chain::new(store, runtime_adapter.clone(), chain_genesis)?;
+        let mut chain = Chain::new(store, runtime_adapter.clone(), chain_genesis)?;
+        if let Some(expected_genesis_hash) = expected_genesis_hash {
+            let genesis_hash = chain.get_block_by_height(0).unwrap().hash().to_string();
+            if genesis_hash != expected_genesis_hash {
+                panic!(
+                    "Expected genesis hash to be {}, actual {}",
+                    expected_genesis_hash, genesis_hash,
+                );
+            }
+        }
         Ok(ViewClientActor {
             chain,
             runtime_adapter,

chain/network/tests/announce_account.rs

@@ -78,10 +78,15 @@ pub fn setup_network_node(
         .unwrap()
         .start();
 
-        let view_client_actor =
-            ViewClientActor::new(store.clone(), &chain_genesis, runtime.clone(), network_adapter)
-                .unwrap()
-                .start();
+        let view_client_actor = ViewClientActor::new(
+            store.clone(),
+            &chain_genesis,
+            runtime.clone(),
+            network_adapter,
+            None,
+        )
+        .unwrap()
+        .start();
 
         PeerManagerActor::new(
             store.clone(),

chain/network/tests/routing.rs

@@ -96,6 +96,7 @@ pub fn setup_network_node(
             &chain_genesis,
             runtime.clone(),
             network_adapter.clone(),
+            None,
         )
         .unwrap()
         .start();