diff --git a/Cargo.lock b/Cargo.lock index 5a8206098b5..5d8adba14a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1963,6 +1963,7 @@ dependencies = [ "near-pool 0.1.0", "near-primitives 0.1.0", "near-store 0.1.0", + "owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2187,11 +2188,12 @@ dependencies = [ "actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "borsh 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "bs58 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.3 (registry+https://github.com/rust-lang/crates.io-index)", + "easy-ext 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "jemallocator 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2386,7 +2388,6 @@ version = "0.0.1" dependencies = [ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "borsh 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index 73f77224dc3..b84f7b41c78 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -16,6 +16,7 @@ serde = "1.0" serde_derive = "1.0" cached = "0.11.0" lazy_static = "1.4" +owning_ref = "0.4.0" borsh = "0.5.0" diff --git a/chain/chain/src/store.rs b/chain/chain/src/store.rs index 057db16e033..6d73ab50b63 100644 --- a/chain/chain/src/store.rs +++ b/chain/chain/src/store.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use borsh::{BorshDeserialize, BorshSerialize}; use cached::{Cached, SizedCache}; use chrono::Utc; +use owning_ref::OwningRef; use serde::Serialize; use near_primitives::block::{Approval, BlockScore}; @@ -20,8 +21,8 @@ use near_primitives::transaction::{ ExecutionOutcomeWithId, ExecutionOutcomeWithIdAndProof, SignedTransaction, }; use near_primitives::types::{ - AccountId, BlockExtra, BlockHeight, ChunkExtra, EpochId, NumBlocks, ShardId, StateChangeCause, - StateChanges, StateChangesRequest, + AccountId, BlockExtra, BlockHeight, ChunkExtra, EpochId, NumBlocks, RawStateChangesList, + ShardId, StateChanges, StateChangesExt, StateChangesRequest, }; use near_primitives::utils::{index_to_bytes, to_timestamp}; use near_primitives::views::LightClientBlockView; @@ -917,56 +918,124 @@ impl ChainStoreAccess for ChainStore { .map_err(|e| e.into()) } + /// Retrieve the key-value changes from the store and decode them appropriately. + /// + /// We store different types of data, so we need to take care of all the types. That is, the + /// account data and the access keys are internally-serialized and we have to deserialize those + /// values appropriately. Code and data changes are simple blobs of data, so we return them as + /// base64-encoded blobs. fn get_key_value_changes( &self, block_hash: &CryptoHash, state_changes_request: &StateChangesRequest, ) -> Result { use near_primitives::utils; + + // We store the trie changes under a compound key: `block_hash + trie_key`, so when we + // query the changes, we reverse the process by splitting the key using simple slicing of an + // array of bytes, essentially, extracting `trie_key`. + // + // Example: data changes are stored under a key: + // + // block_hash + (col::ACCOUNT + account_id + ACCOUNT_DATA_SEPARATOR + user_specified_key) + // + // Thus, to query all the changes by a user-specified key prefix, we do the following: + // 1. Query RocksDB for + // block_hash + (col::ACCOUNT + account_id + ACCOUNT_DATA_SEPARATOR + user_specified_key_prefix) + // + // 2. In the simplest case, to extract the full key we need to slice the RocksDB key by a length of + // block_hash + (col::ACCOUNT + account_id + ACCOUNT_DATA_SEPARATOR) + // + // In this implementation, however, we decoupled this process into two steps: + // + // 2.1. Split off the `block_hash` (using `common_storage_key_prefix_len`), thus we are + // left working with a key that was used in the trie. + // 2.2. Parse the trie key with a relevant KeyFor* implementation to ensure consistency + let mut storage_key = block_hash.as_ref().to_vec(); - storage_key.extend(match state_changes_request { + let common_storage_key_prefix_len = storage_key.len(); + + let data_key: Vec = match state_changes_request { StateChangesRequest::AccountChanges { account_id } => { - utils::key_for_account(account_id) - } - StateChangesRequest::DataChanges { account_id, key_prefix } => { - utils::key_for_data(account_id, key_prefix) + utils::KeyForAccount::new(account_id).into() } StateChangesRequest::SingleAccessKeyChanges { account_id, access_key_pk } => { - utils::key_for_access_key(account_id, access_key_pk) + utils::KeyForAccessKey::new(account_id, access_key_pk).into() } StateChangesRequest::AllAccessKeyChanges { account_id } => { - utils::key_for_all_access_keys(account_id) + utils::KeyForAccessKey::get_prefix(account_id).into() } - StateChangesRequest::CodeChanges { account_id } => utils::key_for_code(account_id), - StateChangesRequest::SinglePostponedReceiptChanges { account_id, data_id } => { - utils::key_for_postponed_receipt_id(account_id, data_id) + StateChangesRequest::CodeChanges { account_id } => { + utils::KeyForCode::new(account_id).into() } - StateChangesRequest::AllPostponedReceiptChanges { account_id } => { - utils::key_for_all_postponed_receipts(account_id) + StateChangesRequest::DataChanges { account_id, key_prefix } => { + utils::KeyForData::new(account_id, key_prefix.as_ref()).into() } - }); - let common_key_prefix_len = block_hash.as_ref().len() - + match state_changes_request { - StateChangesRequest::AccountChanges { .. } - | StateChangesRequest::SingleAccessKeyChanges { .. } - | StateChangesRequest::AllAccessKeyChanges { .. } - | StateChangesRequest::CodeChanges { .. } - | StateChangesRequest::SinglePostponedReceiptChanges { .. } - | StateChangesRequest::AllPostponedReceiptChanges { .. } => storage_key.len(), - StateChangesRequest::DataChanges { account_id, .. } => { - utils::key_for_data(account_id, b"").len() - } - }; - let mut changes = StateChanges::new(); - let changes_iter = self.store.iter_prefix_ser::>)>>( - ColKeyValueChanges, - &storage_key, - ); - for change in changes_iter { - let (key, value) = change?; - changes.insert(key[common_key_prefix_len..].to_owned(), value); - } - Ok(changes) + }; + storage_key.extend(&data_key); + + let mut changes_per_key_prefix = self + .store + .iter_prefix_ser::(ColKeyValueChanges, &storage_key) + .map(|change| { + // Split off the irrelevant part of the key, so only the original trie_key is left. + let (key, state_changes) = change?; + let key = OwningRef::new(key).map(|key| &key[common_storage_key_prefix_len..]); + Ok((key, state_changes)) + }); + + // It is a lifetime workaround. We cannot return `&mut changes_per_key_prefix.filter_map(...` + // as that is a temporary object created there with `filter_map`, and you cannot leak a + // reference to a temporary object. + let mut changes_per_exact_key; + + let changes_per_key: &mut dyn Iterator = match state_changes_request { + // These request types are expected to match the key exactly: + StateChangesRequest::AccountChanges { .. } + | StateChangesRequest::SingleAccessKeyChanges { .. } + | StateChangesRequest::CodeChanges { .. } => { + changes_per_exact_key = changes_per_key_prefix.filter_map(|change| { + let (key, state_changes) = match change { + Ok(change) => change, + error => { + return Some(error); + } + }; + if key.len() != data_key.len() { + None + } else { + debug_assert_eq!(key.as_ref(), data_key.as_ref() as &[u8]); + Some(Ok((key, state_changes))) + } + }); + &mut changes_per_exact_key + } + + StateChangesRequest::AllAccessKeyChanges { .. } + | StateChangesRequest::DataChanges { .. } => &mut changes_per_key_prefix, + }; + + Ok(match state_changes_request { + StateChangesRequest::AccountChanges { account_id, .. } => { + StateChanges::from_account_changes(changes_per_key, account_id)? + } + StateChangesRequest::SingleAccessKeyChanges { account_id, .. } + | StateChangesRequest::AllAccessKeyChanges { account_id, .. } => { + let access_key_pk = match state_changes_request { + StateChangesRequest::SingleAccessKeyChanges { access_key_pk, .. } => { + Some(access_key_pk) + } + _ => None, + }; + StateChanges::from_access_key_changes(changes_per_key, account_id, access_key_pk)? + } + StateChangesRequest::CodeChanges { account_id, .. } => { + StateChanges::from_code_changes(changes_per_key, account_id)? + } + StateChangesRequest::DataChanges { account_id, .. } => { + StateChanges::from_data_changes(changes_per_key, account_id)? + } + }) } } diff --git a/chain/client/src/types.rs b/chain/client/src/types.rs index 07cbfdfaece..bedbfbcd20f 100644 --- a/chain/client/src/types.rs +++ b/chain/client/src/types.rs @@ -11,12 +11,12 @@ use near_network::PeerInfo; use near_primitives::hash::CryptoHash; use near_primitives::sharding::ChunkHash; use near_primitives::types::{ - AccountId, BlockHeight, BlockId, MaybeBlockId, ShardId, StateChanges, StateChangesRequest, + AccountId, BlockHeight, BlockIdOrFinality, MaybeBlockId, ShardId, StateChangesRequest, }; use near_primitives::utils::generate_random_string; use near_primitives::views::{ - BlockView, ChunkView, EpochValidatorInfo, FinalExecutionOutcomeView, Finality, GasPriceView, - LightClientBlockView, QueryRequest, QueryResponse, + BlockView, ChunkView, EpochValidatorInfo, FinalExecutionOutcomeView, GasPriceView, + LightClientBlockView, QueryRequest, QueryResponse, StateChangesView, }; pub use near_primitives::views::{StatusResponse, StatusSyncInfo}; @@ -142,9 +142,12 @@ impl SyncStatus { } /// Actor message requesting block by id or hash. -pub enum GetBlock { - BlockId(BlockId), - Finality(Finality), +pub struct GetBlock(pub BlockIdOrFinality); + +impl GetBlock { + pub fn latest() -> Self { + Self(BlockIdOrFinality::latest()) + } } impl Message for GetBlock { @@ -166,14 +169,13 @@ impl Message for GetChunk { #[derive(Deserialize, Clone)] pub struct Query { pub query_id: String, - pub block_id: MaybeBlockId, + pub block_id_or_finality: BlockIdOrFinality, pub request: QueryRequest, - pub finality: Finality, } impl Query { - pub fn new(block_id: MaybeBlockId, request: QueryRequest, finality: Finality) -> Self { - Query { query_id: generate_random_string(10), block_id, request, finality } + pub fn new(block_id_or_finality: BlockIdOrFinality, request: QueryRequest) -> Self { + Query { query_id: generate_random_string(10), block_id_or_finality, request } } } @@ -246,5 +248,5 @@ pub struct GetKeyValueChanges { } impl Message for GetKeyValueChanges { - type Result = Result; + type Result = Result; } diff --git a/chain/client/src/view_client.rs b/chain/client/src/view_client.rs index e5f5742bdf2..fbc66c30885 100644 --- a/chain/client/src/view_client.rs +++ b/chain/client/src/view_client.rs @@ -25,10 +25,12 @@ use near_primitives::block::{BlockHeader, BlockScore, GenesisId}; use near_primitives::hash::CryptoHash; use near_primitives::merkle::verify_path; use near_primitives::network::AnnounceAccount; -use near_primitives::types::{AccountId, BlockHeight, BlockId, MaybeBlockId, StateChanges}; +use near_primitives::types::{ + AccountId, BlockHeight, BlockId, BlockIdOrFinality, Finality, MaybeBlockId, +}; use near_primitives::views::{ BlockView, ChunkView, EpochValidatorInfo, FinalExecutionOutcomeView, FinalExecutionStatus, - Finality, GasPriceView, LightClientBlockView, QueryRequest, QueryResponse, + GasPriceView, LightClientBlockView, QueryRequest, QueryResponse, StateChangesView, }; use near_store::Store; @@ -147,12 +149,16 @@ impl ViewClientActor { return response.map(Some); } - let header = match msg.block_id { - Some(BlockId::Height(block_height)) => self.chain.get_header_by_height(block_height), - Some(BlockId::Hash(block_hash)) => self.chain.get_block_header(&block_hash), - None => { + let header = match msg.block_id_or_finality { + BlockIdOrFinality::BlockId(BlockId::Height(block_height)) => { + self.chain.get_header_by_height(block_height) + } + BlockIdOrFinality::BlockId(BlockId::Hash(block_hash)) => { + self.chain.get_block_header(&block_hash) + } + BlockIdOrFinality::Finality(ref finality) => { let block_hash = - self.get_block_hash_by_finality(&msg.finality).map_err(|e| e.to_string())?; + self.get_block_hash_by_finality(&finality).map_err(|e| e.to_string())?; self.chain.get_block_header(&block_hash) } }; @@ -199,9 +205,8 @@ impl ViewClientActor { self.network_adapter.do_send(NetworkRequests::Query { query_id: msg.query_id.clone(), account_id: validator, - block_id: msg.block_id.clone(), + block_id_or_finality: msg.block_id_or_finality.clone(), request: msg.request.clone(), - finality: msg.finality.clone(), }); } @@ -351,16 +356,18 @@ impl Handler for ViewClientActor { type Result = Result; fn handle(&mut self, msg: GetBlock, _: &mut Context) -> Self::Result { - match msg { - GetBlock::Finality(finality) => { + match msg.0 { + BlockIdOrFinality::Finality(finality) => { let block_hash = self.get_block_hash_by_finality(&finality).map_err(|e| e.to_string())?; self.chain.get_block(&block_hash).map(Clone::clone) } - GetBlock::BlockId(BlockId::Height(height)) => { + BlockIdOrFinality::BlockId(BlockId::Height(height)) => { self.chain.get_block_by_height(height).map(Clone::clone) } - GetBlock::BlockId(BlockId::Hash(hash)) => self.chain.get_block(&hash).map(Clone::clone), + BlockIdOrFinality::BlockId(BlockId::Hash(hash)) => { + self.chain.get_block(&hash).map(Clone::clone) + } } .and_then(|block| { self.runtime_adapter @@ -443,12 +450,13 @@ impl Handler for ViewClientActor { /// Returns a list of changes in a store for a given block. impl Handler for ViewClientActor { - type Result = Result; + type Result = Result; fn handle(&mut self, msg: GetKeyValueChanges, _: &mut Context) -> Self::Result { self.chain .store() .get_key_value_changes(&msg.block_hash, &msg.state_changes_request) + .map(|state_changes| state_changes.into_iter().map(Into::into).collect()) .map_err(|e| e.to_string()) } } @@ -545,8 +553,8 @@ impl Handler for ViewClientActor { } NetworkViewClientResponses::NoResponse } - NetworkViewClientMessages::Query { query_id, block_id, request, finality } => { - let query = Query { query_id: query_id.clone(), block_id, request, finality }; + NetworkViewClientMessages::Query { query_id, block_id_or_finality, request } => { + let query = Query { query_id: query_id.clone(), block_id_or_finality, request }; match self.handle_query(query) { Ok(Some(r)) => { NetworkViewClientResponses::QueryResponse { query_id, response: Ok(r) } diff --git a/chain/client/tests/catching_up.rs b/chain/client/tests/catching_up.rs index 55221ef33ab..0735645e747 100644 --- a/chain/client/tests/catching_up.rs +++ b/chain/client/tests/catching_up.rs @@ -22,8 +22,8 @@ mod tests { use near_primitives::sharding::ChunkHash; use near_primitives::test_utils::init_integration_logger; use near_primitives::transaction::SignedTransaction; - use near_primitives::types::{BlockHeight, BlockHeightDelta}; - use near_primitives::views::{Finality, QueryRequest, QueryResponseKind::ViewAccount}; + use near_primitives::types::{BlockHeight, BlockHeightDelta, BlockIdOrFinality}; + use near_primitives::views::{QueryRequest, QueryResponseKind::ViewAccount}; fn get_validators_and_key_pairs() -> (Vec>, Vec) { let validators = vec![ @@ -308,11 +308,10 @@ mod tests { connectors1.write().unwrap()[i] .1 .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: account_to.clone(), }, - Finality::None, )) .then(move |res| { let res_inner = res.unwrap(); @@ -506,12 +505,11 @@ mod tests { connectors1.write().unwrap()[i] .1 .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: flat_validators[j] .to_string(), }, - Finality::None, )) .then(move |res| { let res_inner = res.unwrap(); diff --git a/chain/client/tests/chunks_management.rs b/chain/client/tests/chunks_management.rs index 219b61aa62c..c7b611164a6 100644 --- a/chain/client/tests/chunks_management.rs +++ b/chain/client/tests/chunks_management.rs @@ -20,7 +20,6 @@ use near_primitives::test_utils::init_integration_logger; use near_primitives::test_utils::{heavy_test, init_test_logger}; use near_primitives::transaction::SignedTransaction; use near_primitives::validator_signer::InMemoryValidatorSigner; -use near_primitives::views::Finality; #[test] fn chunks_produced_and_distributed_all_in_all_shards() { @@ -217,7 +216,7 @@ fn chunks_produced_and_distributed_common( *connectors.write().unwrap() = conn; let view_client = connectors.write().unwrap()[0].1.clone(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let header: BlockHeader = res.unwrap().unwrap().header.into(); let block_hash = header.hash; let connectors_ = connectors.write().unwrap(); diff --git a/chain/client/tests/cross_shard_tx.rs b/chain/client/tests/cross_shard_tx.rs index 74ce8cc5d0f..b142cdb383e 100644 --- a/chain/client/tests/cross_shard_tx.rs +++ b/chain/client/tests/cross_shard_tx.rs @@ -8,7 +8,8 @@ use near_client::test_utils::setup_mock_all_validators; use near_client::{ClientActor, Query, ViewClientActor}; use near_network::{NetworkRequests, NetworkResponses, PeerInfo}; use near_primitives::test_utils::init_test_logger; -use near_primitives::views::{Finality, QueryRequest, QueryResponseKind::ViewAccount}; +use near_primitives::types::BlockIdOrFinality; +use near_primitives::views::{QueryRequest, QueryResponseKind::ViewAccount}; /// Tests that the KeyValueRuntime properly sets balances in genesis and makes them queriable #[test] @@ -51,9 +52,8 @@ fn test_keyvalue_runtime_balances() { connectors_[i] .1 .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: flat_validators[i].to_string() }, - Finality::None, )) .then(move |res| { let query_response = res.unwrap().unwrap().unwrap(); @@ -93,9 +93,8 @@ mod tests { use near_primitives::hash::CryptoHash; use near_primitives::test_utils::init_test_logger; use near_primitives::transaction::SignedTransaction; - use near_primitives::types::AccountId; - use near_primitives::views::QueryResponseKind::ViewAccount; - use near_primitives::views::{Finality, QueryRequest, QueryResponse}; + use near_primitives::types::{AccountId, BlockIdOrFinality}; + use near_primitives::views::{QueryRequest, QueryResponse, QueryResponseKind::ViewAccount}; fn send_tx( num_validators: usize, @@ -192,9 +191,8 @@ mod tests { + (*presumable_epoch.read().unwrap() * 8) % 24] .1 .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: account_id.clone() }, - Finality::None, )) .then(move |x| { test_cross_shard_tx_callback( @@ -284,11 +282,10 @@ mod tests { + (*presumable_epoch.read().unwrap() * 8) % 24] .1 .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: validators[i].to_string(), }, - Finality::None, )) .then(move |x| { test_cross_shard_tx_callback( @@ -336,9 +333,8 @@ mod tests { + (*presumable_epoch.read().unwrap() * 8) % 24] .1 .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: account_id.clone() }, - Finality::None, )) .then(move |x| { test_cross_shard_tx_callback( @@ -452,11 +448,10 @@ mod tests { connectors_[i + *presumable_epoch.read().unwrap() * 8] .1 .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: flat_validators[i].to_string(), }, - Finality::None, )) .then(move |x| { test_cross_shard_tx_callback( diff --git a/chain/client/tests/process_blocks.rs b/chain/client/tests/process_blocks.rs index b86f630ff58..f0f56b4dab5 100644 --- a/chain/client/tests/process_blocks.rs +++ b/chain/client/tests/process_blocks.rs @@ -26,7 +26,6 @@ use near_primitives::transaction::{SignedTransaction, Transaction}; use near_primitives::types::{EpochId, MerkleHash}; use near_primitives::utils::to_timestamp; use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner}; -use near_primitives::views::Finality; use near_store::test_utils::create_test_store; /// Runs block producing client and stops after network mock received two blocks. @@ -107,7 +106,7 @@ fn produce_blocks_with_tx() { }), ); near_network::test_utils::wait_or_panic(5000); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let header: BlockHeader = res.unwrap().unwrap().header.into(); let block_hash = header.hash; client @@ -144,7 +143,7 @@ fn receive_network_block() { NetworkResponses::NoResponse }), ); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let last_block = res.unwrap().unwrap(); let signer = InMemoryValidatorSigner::from_seed("test1", KeyType::ED25519, "test1"); let block = Block::produce( @@ -208,7 +207,7 @@ fn receive_network_block_header() { _ => NetworkResponses::NoResponse, }), ); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let last_block = res.unwrap().unwrap(); let signer = InMemoryValidatorSigner::from_seed("test", KeyType::ED25519, "test"); let block = Block::produce( @@ -293,7 +292,7 @@ fn produce_block_with_approvals() { NetworkResponses::NoResponse }), ); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let last_block = res.unwrap().unwrap(); let signer1 = InMemoryValidatorSigner::from_seed("test2", KeyType::ED25519, "test2"); let block = Block::produce( @@ -371,7 +370,7 @@ fn invalid_blocks() { NetworkResponses::NoResponse }), ); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let last_block = res.unwrap().unwrap(); let signer = InMemoryValidatorSigner::from_seed("test", KeyType::ED25519, "test"); // Send block with invalid chunk mask diff --git a/chain/client/tests/query_client.rs b/chain/client/tests/query_client.rs index 6084d711b1a..13f7acde8d6 100644 --- a/chain/client/tests/query_client.rs +++ b/chain/client/tests/query_client.rs @@ -4,7 +4,8 @@ use futures::{future, FutureExt}; use near_client::test_utils::setup_no_network; use near_client::Query; use near_primitives::test_utils::init_test_logger; -use near_primitives::views::{Finality, QueryRequest, QueryResponseKind}; +use near_primitives::types::BlockIdOrFinality; +use near_primitives::views::{QueryRequest, QueryResponseKind}; /// Query account from view client #[test] @@ -15,9 +16,8 @@ fn query_client() { actix::spawn( view_client .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: "test".to_owned() }, - Finality::None, )) .then(|res| { match res.unwrap().unwrap().unwrap().kind { diff --git a/chain/jsonrpc/client/src/lib.rs b/chain/jsonrpc/client/src/lib.rs index 6167b013a8c..38342dcc6ce 100644 --- a/chain/jsonrpc/client/src/lib.rs +++ b/chain/jsonrpc/client/src/lib.rs @@ -6,11 +6,13 @@ use serde::Deserialize; use serde::Serialize; use near_primitives::hash::CryptoHash; -use near_primitives::rpc::{BlockQueryInfo, RpcGenesisRecordsRequest, RpcQueryRequest}; -use near_primitives::types::{BlockId, MaybeBlockId, ShardId}; +use near_primitives::rpc::{ + RpcGenesisRecordsRequest, RpcQueryRequest, RpcStateChangesRequest, RpcStateChangesResponse, +}; +use near_primitives::types::{BlockId, BlockIdOrFinality, MaybeBlockId, ShardId}; use near_primitives::views::{ BlockView, ChunkView, EpochValidatorInfo, FinalExecutionOutcomeView, GasPriceView, - GenesisRecordsView, QueryResponse, StateChangesView, StatusResponse, + GenesisRecordsView, QueryResponse, StatusResponse, }; use crate::message::{from_slice, Message, RpcError}; @@ -185,7 +187,6 @@ jsonrpc_client!(pub struct JsonRpcClient { pub fn health(&mut self) -> RpcRequest<()>; pub fn tx(&mut self, hash: String, account_id: String) -> RpcRequest; pub fn chunk(&mut self, id: ChunkId) -> RpcRequest; - pub fn changes(&mut self, block_hash: CryptoHash, key_prefix: Vec) -> RpcRequest; pub fn validators(&mut self, block_id: MaybeBlockId) -> RpcRequest; pub fn gas_price(&mut self, block_id: MaybeBlockId) -> RpcRequest; }); @@ -213,9 +214,17 @@ impl JsonRpcClient { call_method(&self.client, &self.server_addr, "block", [block_id]) } - pub fn block(&mut self, request: BlockQueryInfo) -> RpcRequest { + pub fn block(&mut self, request: BlockIdOrFinality) -> RpcRequest { call_method(&self.client, &self.server_addr, "block", request) } + + #[allow(non_snake_case)] + pub fn EXPERIMENTAL_changes( + &mut self, + request: RpcStateChangesRequest, + ) -> RpcRequest { + call_method(&self.client, &self.server_addr, "EXPERIMENTAL_changes", request) + } } fn create_client() -> Client { diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index ebea3905b66..e382f60fad9 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -14,7 +14,7 @@ use futures::{FutureExt, TryFutureExt}; use prometheus; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; +use serde_json::Value; use tokio::time::{delay_for, timeout}; use validator::Validate; @@ -33,12 +33,14 @@ use near_network::types::{NetworkAdversarialMessage, NetworkViewClientMessages}; use near_network::{NetworkClientMessages, NetworkClientResponses}; use near_primitives::errors::{InvalidTxError, TxExecutionError}; use near_primitives::hash::CryptoHash; -use near_primitives::rpc::{BlockQueryInfo, RpcGenesisRecordsRequest, RpcQueryRequest}; +use near_primitives::rpc::{ + RpcGenesisRecordsRequest, RpcQueryRequest, RpcStateChangesRequest, RpcStateChangesResponse, +}; use near_primitives::serialize::{from_base, from_base64, BaseEncode}; use near_primitives::transaction::SignedTransaction; -use near_primitives::types::{AccountId, BlockId, MaybeBlockId, StateChangesRequest}; +use near_primitives::types::{AccountId, BlockId, BlockIdOrFinality, MaybeBlockId}; use near_primitives::utils::is_valid_account_id; -use near_primitives::views::{FinalExecutionStatus, Finality, GenesisRecordsView, QueryRequest}; +use near_primitives::views::{FinalExecutionStatus, GenesisRecordsView, QueryRequest}; mod metrics; @@ -212,7 +214,7 @@ impl JsonRpcHandler { "tx" => self.tx_status(request.params).await, "block" => self.block(request.params).await, "chunk" => self.chunk(request.params).await, - "changes" => self.changes(request.params).await, + "EXPERIMENTAL_changes" => self.changes(request.params).await, "next_light_client_block" => self.next_light_client_block(request.params).await, "network_info" => self.network_info().await, "gas_price" => self.gas_price(request.params).await, @@ -459,12 +461,11 @@ impl JsonRpcHandler { } }; // Use Finality::None here to make backward compatibility tests work - RpcQueryRequest { block_id: None, request, finality: Finality::None } + RpcQueryRequest { request, block_id_or_finality: BlockIdOrFinality::latest() } } else { parse_params::(params)? }; - let query = - Query::new(query_request.block_id, query_request.request, query_request.finality); + let query = Query::new(query_request.block_id_or_finality, query_request.request); timeout(self.polling_config.polling_timeout, async { loop { let result = self.view_client_addr.send(query.clone()).await; @@ -496,19 +497,13 @@ impl JsonRpcHandler { } async fn block(&self, params: Option) -> Result { - let block_query = if let Ok((block_id,)) = parse_params::<(BlockId,)>(params.clone()) { - BlockQueryInfo::BlockId(block_id) - } else { - parse_params::(params)? - }; - jsonify( - self.view_client_addr - .send(match block_query { - BlockQueryInfo::BlockId(block_id) => GetBlock::BlockId(block_id), - BlockQueryInfo::Finality(finality) => GetBlock::Finality(finality), - }) - .await, - ) + let block_id_or_finality = + if let Ok((block_id,)) = parse_params::<(BlockId,)>(params.clone()) { + BlockIdOrFinality::BlockId(block_id) + } else { + parse_params::(params)? + }; + jsonify(self.view_client_addr.send(GetBlock(block_id_or_finality)).await) } async fn chunk(&self, params: Option) -> Result { @@ -529,32 +524,23 @@ impl JsonRpcHandler { } async fn changes(&self, params: Option) -> Result { - let (block_hash, state_changes_request) = - parse_params::<(CryptoHash, StateChangesRequest)>(params)?; - let block_hash_copy = block_hash.clone(); + let RpcStateChangesRequest { block_id_or_finality, state_changes_request } = + parse_params(params)?; + let block = self + .view_client_addr + .send(GetBlock(block_id_or_finality)) + .await + .map_err(|err| RpcError::server_error(Some(err.to_string())))? + .map_err(|err| RpcError::server_error(Some(err)))?; + let block_hash = block.header.hash.clone(); jsonify( self.view_client_addr .send(GetKeyValueChanges { block_hash, state_changes_request }) .await .map(|v| { - v.map(|changes| { - json!({ - "block_hash": block_hash_copy, - "changes_by_key": changes - .into_iter() - .map(|(key, changes)| { - json!({ - "key": key, - "changes": changes.into_iter().map(|(cause, value)| { - json!({ - "cause": cause, - "value": value - }) - }).collect::>() - }) - }) - .collect::>() - }) + v.map(|changes| RpcStateChangesResponse { + block_hash: block.header.hash, + changes, }) }), ) diff --git a/chain/jsonrpc/tests/rpc_query.rs b/chain/jsonrpc/tests/rpc_query.rs index c6fb81857f2..3707eed503b 100644 --- a/chain/jsonrpc/tests/rpc_query.rs +++ b/chain/jsonrpc/tests/rpc_query.rs @@ -9,12 +9,10 @@ use near_jsonrpc_client::ChunkId; use near_network::test_utils::WaitOrTimeout; use near_primitives::account::{AccessKey, AccessKeyPermission}; use near_primitives::hash::CryptoHash; -use near_primitives::rpc::{ - BlockQueryInfo, RpcGenesisRecordsRequest, RpcPagination, RpcQueryRequest, -}; +use near_primitives::rpc::{RpcGenesisRecordsRequest, RpcPagination, RpcQueryRequest}; use near_primitives::test_utils::init_test_logger; -use near_primitives::types::{BlockId, ShardId}; -use near_primitives::views::{Finality, QueryRequest, QueryResponseKind}; +use near_primitives::types::{BlockId, BlockIdOrFinality, Finality, ShardId}; +use near_primitives::views::{QueryRequest, QueryResponseKind}; mod test_utils; @@ -71,12 +69,12 @@ fn test_block_by_id_hash() { fn test_block_query() { test_with_client!(client, async move { let block_response1 = - client.block(BlockQueryInfo::BlockId(BlockId::Height(0))).await.unwrap(); + client.block(BlockIdOrFinality::BlockId(BlockId::Height(0))).await.unwrap(); let block_response2 = client - .block(BlockQueryInfo::BlockId(BlockId::Hash(block_response1.header.hash))) + .block(BlockIdOrFinality::BlockId(BlockId::Hash(block_response1.header.hash))) .await .unwrap(); - let block_response3 = client.block(BlockQueryInfo::Finality(Finality::None)).await.unwrap(); + let block_response3 = client.block(BlockIdOrFinality::latest()).await.unwrap(); for block in [block_response1, block_response2, block_response3].into_iter() { assert_eq!(block.author, "test1"); assert_eq!(block.header.height, 0); @@ -91,8 +89,8 @@ fn test_block_query() { assert_eq!(block.header.validator_proposals.len(), 0); } // no doomslug final or nfg final block - assert!(client.block(BlockQueryInfo::Finality(Finality::DoomSlug)).await.is_err()); - assert!(client.block(BlockQueryInfo::Finality(Finality::NFG)).await.is_err()); + assert!(client.block(BlockIdOrFinality::Finality(Finality::DoomSlug)).await.is_err()); + assert!(client.block(BlockIdOrFinality::Finality(Finality::NFG)).await.is_err()); }); } @@ -173,53 +171,26 @@ fn test_query_account() { let block_hash = status.sync_info.latest_block_hash; let query_response_1 = client .query(RpcQueryRequest { - block_id: None, + block_id_or_finality: BlockIdOrFinality::latest(), request: QueryRequest::ViewAccount { account_id: "test".to_string() }, - finality: Finality::None, }) .await .unwrap(); let query_response_2 = client .query(RpcQueryRequest { - block_id: Some(BlockId::Height(0)), + block_id_or_finality: BlockIdOrFinality::BlockId(BlockId::Height(0)), request: QueryRequest::ViewAccount { account_id: "test".to_string() }, - finality: Finality::None, }) .await .unwrap(); let query_response_3 = client .query(RpcQueryRequest { - block_id: Some(BlockId::Hash(block_hash)), + block_id_or_finality: BlockIdOrFinality::BlockId(BlockId::Hash(block_hash)), request: QueryRequest::ViewAccount { account_id: "test".to_string() }, - finality: Finality::None, }) .await .unwrap(); - let query_response_4 = client - .query(RpcQueryRequest { - block_id: Some(BlockId::Hash(block_hash)), - request: QueryRequest::ViewAccount { account_id: "test".to_string() }, - finality: Finality::DoomSlug, - }) - .await - .unwrap(); - let query_response_5 = client - .query(RpcQueryRequest { - block_id: Some(BlockId::Hash(block_hash)), - request: QueryRequest::ViewAccount { account_id: "test".to_string() }, - finality: Finality::NFG, - }) - .await - .unwrap(); - for query_response in [ - query_response_1, - query_response_2, - query_response_3, - query_response_4, - query_response_5, - ] - .into_iter() - { + for query_response in [query_response_1, query_response_2, query_response_3].into_iter() { assert_eq!(query_response.block_height, 0); assert_eq!(query_response.block_hash, block_hash); let account_info = if let QueryResponseKind::ViewAccount(ref account) = @@ -235,6 +206,30 @@ fn test_query_account() { assert_eq!(account_info.storage_paid_at, 0); assert_eq!(account_info.storage_usage, 0); } + + let non_finalized_query_response_1 = client + .query(RpcQueryRequest { + block_id_or_finality: BlockIdOrFinality::Finality(Finality::DoomSlug), + request: QueryRequest::ViewAccount { account_id: "test".to_string() }, + }) + .await; + let non_finalized_query_response_2 = client + .query(RpcQueryRequest { + block_id_or_finality: BlockIdOrFinality::Finality(Finality::NFG), + request: QueryRequest::ViewAccount { account_id: "test".to_string() }, + }) + .await; + for query_response in + [non_finalized_query_response_1, non_finalized_query_response_2].into_iter() + { + let rpc_error = if let Err(err) = query_response { + err + } else { + panic!("the error is expected since the latest block is not finalized yet"); + }; + assert_eq!(rpc_error.code, -32000); + assert!(rpc_error.data.as_ref().unwrap().as_str().unwrap().contains("DB Not Found")); + } }); } @@ -263,9 +258,8 @@ fn test_query_access_keys() { test_with_client!(client, async move { let query_response = client .query(RpcQueryRequest { - block_id: None, + block_id_or_finality: BlockIdOrFinality::latest(), request: QueryRequest::ViewAccessKeyList { account_id: "test".to_string() }, - finality: Finality::None, }) .await .unwrap(); @@ -310,7 +304,7 @@ fn test_query_access_key() { test_with_client!(client, async move { let query_response = client .query(RpcQueryRequest { - block_id: None, + block_id_or_finality: BlockIdOrFinality::latest(), request: QueryRequest::ViewAccessKey { account_id: "test".to_string(), public_key: PublicKey::try_from( @@ -318,7 +312,6 @@ fn test_query_access_key() { ) .unwrap(), }, - finality: Finality::None, }) .await .unwrap(); @@ -339,12 +332,11 @@ fn test_query_state() { test_with_client!(client, async move { let query_response = client .query(RpcQueryRequest { - block_id: None, + block_id_or_finality: BlockIdOrFinality::latest(), request: QueryRequest::ViewState { account_id: "test".to_string(), prefix: vec![].into(), }, - finality: Finality::None, }) .await .unwrap(); @@ -364,13 +356,12 @@ fn test_query_call_function() { test_with_client!(client, async move { let query_response = client .query(RpcQueryRequest { - block_id: None, + block_id_or_finality: BlockIdOrFinality::latest(), request: QueryRequest::CallFunction { account_id: "test".to_string(), method_name: "method".to_string(), args: vec![].into(), }, - finality: Finality::None, }) .await .unwrap(); @@ -563,7 +554,7 @@ fn test_gas_price_by_height() { #[test] fn test_gas_price_by_hash() { test_with_client!(client, async move { - let block = client.block(BlockQueryInfo::BlockId(BlockId::Height(0))).await.unwrap(); + let block = client.block(BlockIdOrFinality::BlockId(BlockId::Height(0))).await.unwrap(); let gas_price = client.gas_price(Some(BlockId::Hash(block.header.hash))).await.unwrap(); assert!(gas_price.gas_price > 0); }); diff --git a/chain/jsonrpc/tests/rpc_transactions.rs b/chain/jsonrpc/tests/rpc_transactions.rs index ca99358012a..275b4c5f4e9 100644 --- a/chain/jsonrpc/tests/rpc_transactions.rs +++ b/chain/jsonrpc/tests/rpc_transactions.rs @@ -13,7 +13,7 @@ use near_primitives::hash::{hash, CryptoHash}; use near_primitives::serialize::to_base64; use near_primitives::test_utils::{init_integration_logger, init_test_logger}; use near_primitives::transaction::SignedTransaction; -use near_primitives::views::{FinalExecutionStatus, Finality}; +use near_primitives::views::FinalExecutionStatus; mod test_utils; @@ -32,7 +32,7 @@ fn test_send_tx_async() { let tx_hash2_2 = tx_hash2.clone(); let signer_account_id = "test1".to_string(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let header: BlockHeader = res.unwrap().unwrap().header.into(); let block_hash = header.hash; let signer = InMemorySigner::from_seed("test1", KeyType::ED25519, "test1"); @@ -88,7 +88,7 @@ fn test_send_tx_commit() { let mut client = new_client(&format!("http://{}", addr)); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { let header: BlockHeader = res.unwrap().unwrap().header.into(); let block_hash = header.hash; let signer = InMemorySigner::from_seed("test1", KeyType::ED25519, "test1"); @@ -133,45 +133,40 @@ fn test_expired_tx() { let block_hash = block_hash.clone(); let block_height = block_height.clone(); let mut client = new_client(&format!("http://{}", addr)); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - let header: BlockHeader = res.unwrap().unwrap().header.into(); - let hash = block_hash.lock().unwrap().clone(); - let height = block_height.lock().unwrap().clone(); - if let Some(block_hash) = hash { - if let Some(height) = height { - if header.inner_lite.height - height >= 2 { - let signer = InMemorySigner::from_seed( - "test1", - KeyType::ED25519, - "test1", - ); - let tx = SignedTransaction::send_money( - 1, - "test1".to_string(), - "test2".to_string(), - &signer, - 100, - block_hash, - ); - let bytes = tx.try_to_vec().unwrap(); - actix::spawn( - client - .broadcast_tx_commit(to_base64(&bytes)) - .map_err(|_| { - System::current().stop(); - }) - .map(|_| ()), - ); - } + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + let header: BlockHeader = res.unwrap().unwrap().header.into(); + let hash = block_hash.lock().unwrap().clone(); + let height = block_height.lock().unwrap().clone(); + if let Some(block_hash) = hash { + if let Some(height) = height { + if header.inner_lite.height - height >= 2 { + let signer = + InMemorySigner::from_seed("test1", KeyType::ED25519, "test1"); + let tx = SignedTransaction::send_money( + 1, + "test1".to_string(), + "test2".to_string(), + &signer, + 100, + block_hash, + ); + let bytes = tx.try_to_vec().unwrap(); + actix::spawn( + client + .broadcast_tx_commit(to_base64(&bytes)) + .map_err(|_| { + System::current().stop(); + }) + .map(|_| ()), + ); } - } else { - *block_hash.lock().unwrap() = Some(header.hash); - *block_height.lock().unwrap() = Some(header.inner_lite.height); - }; - future::ready(()) - }, - )); + } + } else { + *block_hash.lock().unwrap() = Some(header.hash); + *block_height.lock().unwrap() = Some(header.inner_lite.height); + }; + future::ready(()) + })); }), 100, 1000, diff --git a/chain/network/src/peer.rs b/chain/network/src/peer.rs index 35b2213ccec..dd747ac0586 100644 --- a/chain/network/src/peer.rs +++ b/chain/network/src/peer.rs @@ -283,8 +283,8 @@ impl Peer { PeerMessage::Routed(message) => { msg_hash = Some(message.hash()); match message.body { - RoutedMessageBody::QueryRequest { query_id, block_id, request, finality } => { - NetworkViewClientMessages::Query { query_id, block_id, request, finality } + RoutedMessageBody::QueryRequest { query_id, block_id_or_finality, request } => { + NetworkViewClientMessages::Query { query_id, block_id_or_finality, request } } RoutedMessageBody::QueryResponse { query_id, response } => { NetworkViewClientMessages::QueryResponse { query_id, response } diff --git a/chain/network/src/peer_manager.rs b/chain/network/src/peer_manager.rs index b8b9f851adf..1a0cd650cc8 100644 --- a/chain/network/src/peer_manager.rs +++ b/chain/network/src/peer_manager.rs @@ -961,11 +961,11 @@ impl Handler for PeerManagerActor { NetworkResponses::RouteNotFound } } - NetworkRequests::Query { query_id, account_id, block_id, request, finality } => { + NetworkRequests::Query { query_id, account_id, block_id_or_finality, request } => { if self.send_message_to_account( ctx, &account_id, - RoutedMessageBody::QueryRequest { query_id, block_id, request, finality }, + RoutedMessageBody::QueryRequest { query_id, block_id_or_finality, request }, ) { NetworkResponses::NoResponse } else { diff --git a/chain/network/src/types.rs b/chain/network/src/types.rs index 9ab8ff524fe..5b29470ca22 100644 --- a/chain/network/src/types.rs +++ b/chain/network/src/types.rs @@ -26,9 +26,9 @@ use near_primitives::hash::{hash, CryptoHash}; use near_primitives::network::{AnnounceAccount, PeerId}; use near_primitives::sharding::{ChunkHash, PartialEncodedChunk}; use near_primitives::transaction::{ExecutionOutcomeWithIdAndProof, SignedTransaction}; -use near_primitives::types::{AccountId, BlockHeight, EpochId, MaybeBlockId, ShardId}; +use near_primitives::types::{AccountId, BlockHeight, BlockIdOrFinality, EpochId, ShardId}; use near_primitives::utils::{from_timestamp, to_timestamp}; -use near_primitives::views::{FinalExecutionOutcomeView, Finality, QueryRequest, QueryResponse}; +use near_primitives::views::{FinalExecutionOutcomeView, QueryRequest, QueryResponse}; use crate::metrics; use crate::peer::Peer; @@ -213,9 +213,8 @@ pub enum RoutedMessageBody { TxStatusResponse(FinalExecutionOutcomeView), QueryRequest { query_id: String, - block_id: MaybeBlockId, + block_id_or_finality: BlockIdOrFinality, request: QueryRequest, - finality: Finality, }, QueryResponse { query_id: String, @@ -1006,9 +1005,8 @@ pub enum NetworkRequests { Query { query_id: String, account_id: AccountId, - block_id: MaybeBlockId, + block_id_or_finality: BlockIdOrFinality, request: QueryRequest, - finality: Finality, }, /// Request for receipt execution outcome ReceiptOutComeRequest(AccountId, CryptoHash), @@ -1198,7 +1196,7 @@ pub enum NetworkViewClientMessages { /// Transaction status response TxStatusResponse(FinalExecutionOutcomeView), /// General query - Query { query_id: String, block_id: MaybeBlockId, request: QueryRequest, finality: Finality }, + Query { query_id: String, block_id_or_finality: BlockIdOrFinality, request: QueryRequest }, /// Query response QueryResponse { query_id: String, response: Result }, /// Request for receipt outcome diff --git a/core/primitives/Cargo.toml b/core/primitives/Cargo.toml index 92694a0614e..2bf9e9dd0f3 100644 --- a/core/primitives/Cargo.toml +++ b/core/primitives/Cargo.toml @@ -6,11 +6,12 @@ edition = "2018" [dependencies] regex = "1" -bincode = { version = "1.0", features = ["i128"] } bs58 = "0.3" base64 = "0.11" byteorder = "1.2" chrono = { version = "0.4.4", features = ["serde"] } +derive_more = "0.99.3" +easy-ext = "0.1" sha2 = "0.8" lazy_static = "1.4" serde = { version = "1.0", features = ["derive"] } diff --git a/core/primitives/src/account.rs b/core/primitives/src/account.rs index 2aa15e8fa4d..ca0e0056ac6 100644 --- a/core/primitives/src/account.rs +++ b/core/primitives/src/account.rs @@ -5,7 +5,7 @@ use crate::hash::CryptoHash; use crate::types::{AccountId, Balance, BlockHeight, Nonce, StorageUsage}; /// Per account information stored in the state. -#[derive(BorshSerialize, BorshDeserialize, Serialize, PartialEq, Eq, Debug, Clone)] +#[derive(BorshSerialize, BorshDeserialize, PartialEq, Eq, Debug, Clone)] pub struct Account { /// The total not locked tokens. pub amount: Balance, diff --git a/core/primitives/src/hash.rs b/core/primitives/src/hash.rs index 8ef6d1d7dd0..1b6a49c01a2 100644 --- a/core/primitives/src/hash.rs +++ b/core/primitives/src/hash.rs @@ -8,16 +8,12 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::logging::pretty_hash; use crate::serialize::{from_base, to_base, BaseDecode}; -#[derive(Copy, Clone, PartialOrd, PartialEq, Eq, Ord)] +#[derive(Copy, Clone, PartialOrd, PartialEq, Eq, Ord, derive_more::AsRef)] +#[as_ref(forward)] pub struct Digest(pub [u8; 32]); -impl AsRef<[u8]> for Digest { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -#[derive(Copy, Clone, PartialOrd, Ord)] +#[derive(Copy, Clone, PartialOrd, Ord, derive_more::AsRef)] +#[as_ref(forward)] pub struct CryptoHash(pub Digest); impl<'a> From<&'a CryptoHash> for String { @@ -32,12 +28,6 @@ impl Default for CryptoHash { } } -impl AsRef<[u8]> for CryptoHash { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } -} - impl AsMut<[u8]> for CryptoHash { fn as_mut(&mut self) -> &mut [u8] { (self.0).0.as_mut() diff --git a/core/primitives/src/logging.rs b/core/primitives/src/logging.rs index 78e91094b7b..b6af8b3556b 100644 --- a/core/primitives/src/logging.rs +++ b/core/primitives/src/logging.rs @@ -1,7 +1,5 @@ use std::fmt::Debug; -use serde::Serialize; - use crate::serialize::to_base; const VECTOR_MAX_LENGTH: usize = 5; @@ -59,13 +57,6 @@ pub fn pretty_results(results: &[Option>]) -> String { format!("{:?}", pretty_vec(&v)) } -pub fn pretty_serializable(s: &T) -> String { - match bincode::serialize(&s) { - Ok(buf) => pretty_hash(&to_base(&buf)), - Err(e) => format!("[failed to serialize: {}]", e), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/core/primitives/src/rpc.rs b/core/primitives/src/rpc.rs index 0325677250b..489e6ebd2c1 100644 --- a/core/primitives/src/rpc.rs +++ b/core/primitives/src/rpc.rs @@ -1,10 +1,15 @@ +//! This module defines RPC types to nearcore public APIs. +//! +//! NOTE: This module should be only used in RPC server and RPC client implementations, and +//! should not leak these types anywhere else. use serde::{Deserialize, Serialize}; use smart_default::SmartDefault; use validator::Validate; use validator_derive::Validate; -use crate::types::{BlockId, MaybeBlockId}; -use crate::views::{Finality, QueryRequest}; +use crate::hash::CryptoHash; +use crate::types::{BlockIdOrFinality, StateChangesRequest}; +use crate::views::{QueryRequest, StateChangeWithCauseView}; #[derive(Debug, SmartDefault, Serialize, Deserialize, Validate)] #[serde(default)] @@ -24,15 +29,22 @@ pub struct RpcGenesisRecordsRequest { #[derive(Serialize, Deserialize)] pub struct RpcQueryRequest { - pub block_id: MaybeBlockId, + #[serde(flatten)] + pub block_id_or_finality: BlockIdOrFinality, #[serde(flatten)] pub request: QueryRequest, - pub finality: Finality, } #[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BlockQueryInfo { - BlockId(BlockId), - Finality(Finality), +pub struct RpcStateChangesRequest { + #[serde(flatten)] + pub block_id_or_finality: BlockIdOrFinality, + #[serde(flatten)] + pub state_changes_request: StateChangesRequest, +} + +#[derive(Serialize, Deserialize)] +pub struct RpcStateChangesResponse { + pub block_hash: CryptoHash, + pub changes: Vec, } diff --git a/core/primitives/src/serialize.rs b/core/primitives/src/serialize.rs index a1d8b972adb..b1a916e79b0 100644 --- a/core/primitives/src/serialize.rs +++ b/core/primitives/src/serialize.rs @@ -1,41 +1,6 @@ use std::convert::TryFrom; -use std::io; -use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; - -use crate::types::{FunctionArgs, StoreKey}; - -pub type EncodeResult = Result, io::Error>; -pub type DecodeResult = Result; - -// encode a type to byte array -pub trait Encode { - fn encode(&self) -> EncodeResult; -} - -// decode from byte array -pub trait Decode: Sized { - fn decode(data: &[u8]) -> DecodeResult; -} - -impl Encode for T { - fn encode(&self) -> EncodeResult { - bincode::serialize(&self) - .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to serialize")) - } -} - -impl Decode for T -where - T: DeserializeOwned, -{ - fn decode(data: &[u8]) -> DecodeResult { - bincode::deserialize(data) - .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to deserialize")) - } -} - -pub fn to_base>(input: &T) -> String { +pub fn to_base>(input: T) -> String { bs58::encode(input).into_string() } @@ -43,8 +8,8 @@ pub fn from_base(s: &str) -> Result, Box> { bs58::decode(s).into_vec().map_err(|err| err.into()) } -pub fn to_base64>(input: &T) -> String { - base64::encode(input) +pub fn to_base64>(input: T) -> String { + base64::encode(&input) } pub fn from_base64(s: &str) -> Result, Box> { @@ -78,34 +43,6 @@ pub trait BaseDecode: for<'a> TryFrom<&'a [u8], Error = Box { - impl Serialize for $type { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&to_base64(self.as_ref())) - } - } - - impl<'de> Deserialize<'de> for $type { - fn deserialize(deserializer: D) -> Result<$type, D::Error> - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - Ok($type::from( - from_base64(&s).map_err(|err| serde::de::Error::custom(err.to_string()))?, - )) - } - } - }; -} - -pretty_bytes_serialization!(StoreKey); -pretty_bytes_serialization!(FunctionArgs); - pub mod base_format { use serde::de; use serde::{Deserialize, Deserializer, Serializer}; @@ -218,6 +155,30 @@ pub mod vec_base_format { } } +pub mod base64_format { + use serde::de; + use serde::{Deserialize, Deserializer, Serializer}; + + use super::{from_base64, to_base64}; + + pub fn serialize(data: T, serializer: S) -> Result + where + S: Serializer, + T: AsRef<[u8]>, + { + serializer.serialize_str(&to_base64(data)) + } + + pub fn deserialize<'de, D, T>(deserializer: D) -> Result + where + D: Deserializer<'de>, + T: From>, + { + let s = String::deserialize(deserializer)?; + from_base64(&s).map_err(|err| de::Error::custom(err.to_string())).map(Into::into) + } +} + pub mod option_base64_format { use serde::de; use serde::{Deserialize, Deserializer, Serializer}; @@ -348,6 +309,10 @@ pub mod option_u128_dec_format { #[cfg(test)] mod tests { + use serde::{Deserialize, Serialize}; + + use crate::types::StoreKey; + use super::*; #[derive(Deserialize, Serialize)] @@ -358,6 +323,7 @@ mod tests { #[derive(Deserialize, Serialize)] struct StoreKeyStruct { + #[serde(with = "base64_format")] store_key: StoreKey, } diff --git a/core/primitives/src/types.rs b/core/primitives/src/types.rs index 076f3310f97..d28e985439a 100644 --- a/core/primitives/src/types.rs +++ b/core/primitives/src/types.rs @@ -1,11 +1,14 @@ use borsh::{BorshDeserialize, BorshSerialize}; +use derive_more::{AsRef as DeriveAsRef, From as DeriveFrom}; use serde::{Deserialize, Serialize}; use near_crypto::PublicKey; +use crate::account::{AccessKey, Account}; use crate::challenge::ChallengesResult; use crate::hash::CryptoHash; -use crate::serialize::u128_dec_format; +use crate::serialize::{base64_format, u128_dec_format}; +use crate::utils::{KeyForAccessKey, KeyForData}; /// Account identifier. Provides access to user's state. pub type AccountId = String; @@ -45,6 +48,23 @@ pub type PromiseId = Vec; /// Hash used by to store state root. pub type StateRoot = CryptoHash; +/// Different types of finality. +#[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, Clone, Debug, PartialEq, Eq)] +pub enum Finality { + #[serde(rename = "optimistic")] + None, + #[serde(rename = "near-final")] + DoomSlug, + #[serde(rename = "final")] + NFG, +} + +impl Default for Finality { + fn default() -> Self { + Finality::NFG + } +} + /// Account info for validators #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] pub struct AccountInfo { @@ -54,38 +74,33 @@ pub struct AccountInfo { pub amount: Balance, } -#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +/// This type is used to mark keys (arrays of bytes) that are queried from store. +/// +/// NOTE: Currently, this type is only used in the view_client and RPC to be able to transparently +/// pretty-serialize the bytes arrays as base64-encoded strings (see `serialize.rs`). +#[derive(Debug, Clone, PartialEq, Eq, DeriveAsRef, DeriveFrom, BorshSerialize, BorshDeserialize)] +#[as_ref(forward)] pub struct StoreKey(Vec); -impl AsRef<[u8]> for StoreKey { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} +/// This type is used to mark values returned from store (arrays of bytes). +/// +/// NOTE: Currently, this type is only used in the view_client and RPC to be able to transparently +/// pretty-serialize the bytes arrays as base64-encoded strings (see `serialize.rs`). +#[derive(Debug, Clone, PartialEq, Eq, DeriveAsRef, DeriveFrom, BorshSerialize, BorshDeserialize)] +#[as_ref(forward)] +pub struct StoreValue(Vec); -impl From> for StoreKey { - fn from(value: Vec) -> Self { - Self(value) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +/// This type is used to mark function arguments. +/// +/// NOTE: The main reason for this to exist (except the type-safety) is that the value is +/// transparently serialized and deserialized as a base64-encoded string when serde is used +/// (serde_json). +#[derive(Debug, Clone, PartialEq, Eq, DeriveAsRef, DeriveFrom, BorshSerialize, BorshDeserialize)] +#[as_ref(forward)] pub struct FunctionArgs(Vec); -impl AsRef<[u8]> for FunctionArgs { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl From> for FunctionArgs { - fn from(value: Vec) -> Self { - Self(value) - } -} - /// A structure used to index state changes due to transaction/receipt processing and other things. -#[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, Debug, Clone)] +#[derive(BorshSerialize, BorshDeserialize, Debug, Clone)] pub enum StateChangeCause { /// A type of update that does not get finalized. Used for verification and execution of /// immutable smart contract methods. Attempt fo finalize a `TrieUpdate` containing such @@ -116,20 +131,181 @@ pub enum StateChangeCause { ValidatorAccountsUpdate, } +pub type RawStateChangesList = Vec<(StateChangeCause, Option>)>; + /// key that was updated -> list of updates with the corresponding indexing event. -pub type StateChanges = - std::collections::BTreeMap, Vec<(StateChangeCause, Option>)>>; +pub type RawStateChanges = std::collections::BTreeMap, RawStateChangesList>; -#[derive(Deserialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(tag = "changes_type", rename_all = "snake_case")] pub enum StateChangesRequest { - AccountChanges { account_id: AccountId }, - DataChanges { account_id: AccountId, key_prefix: Vec }, - SingleAccessKeyChanges { account_id: AccountId, access_key_pk: PublicKey }, - AllAccessKeyChanges { account_id: AccountId }, - CodeChanges { account_id: AccountId }, - SinglePostponedReceiptChanges { account_id: AccountId, data_id: CryptoHash }, - AllPostponedReceiptChanges { account_id: AccountId }, + AccountChanges { + account_id: AccountId, + }, + SingleAccessKeyChanges { + account_id: AccountId, + access_key_pk: PublicKey, + }, + AllAccessKeyChanges { + account_id: AccountId, + }, + CodeChanges { + account_id: AccountId, + }, + DataChanges { + account_id: AccountId, + #[serde(rename = "key_prefix_base64", with = "base64_format")] + key_prefix: StoreKey, + }, +} + +#[derive(Debug)] +pub enum StateChangeValue { + AccountUpdate { account_id: AccountId, account: Account }, + AccountDeletion { account_id: AccountId }, + AccessKeyUpdate { account_id: AccountId, public_key: PublicKey, access_key: AccessKey }, + AccessKeyDeletion { account_id: AccountId, public_key: PublicKey }, + DataUpdate { account_id: AccountId, key: StoreKey, value: StoreValue }, + DataDeletion { account_id: AccountId, key: StoreKey }, + CodeUpdate { account_id: AccountId, code: Vec }, + CodeDeletion { account_id: AccountId }, +} + +#[derive(Debug)] +pub struct StateChangeWithCause { + pub cause: StateChangeCause, + pub value: StateChangeValue, +} + +pub type StateChanges = Vec; + +#[easy_ext::ext(StateChangesExt)] +impl StateChanges { + pub fn from_account_changes>( + changes_per_key: &mut dyn Iterator>, + account_id: &AccountId, + ) -> Result { + let mut changes = Self::new(); + + for change in changes_per_key { + let (_, state_changes) = change?; + changes.extend(state_changes.into_iter().map(|(cause, state_change)| { + StateChangeWithCause { + cause, + value: if let Some(state_change) = state_change { + StateChangeValue::AccountUpdate { + account_id: account_id.clone(), + account: <_>::try_from_slice(&state_change) + .expect("Failed to parse internally stored account information"), + } + } else { + StateChangeValue::AccountDeletion { account_id: account_id.clone() } + }, + } + })); + } + + Ok(changes) + } + + pub fn from_access_key_changes>( + changes_per_key: &mut dyn Iterator>, + account_id: &AccountId, + access_key_pk: Option<&PublicKey>, + ) -> Result { + let mut changes = Self::new(); + + for change in changes_per_key { + let (key, state_changes) = change?; + + let access_key_pk = match access_key_pk { + Some(access_key_pk) => access_key_pk.clone(), + None => KeyForAccessKey::parse_public_key(key.as_ref(), &account_id) + .expect("Failed to parse internally stored public key"), + }; + + changes.extend(state_changes.into_iter().map(|(cause, state_change)| { + StateChangeWithCause { + cause, + value: if let Some(state_change) = state_change { + StateChangeValue::AccessKeyUpdate { + account_id: account_id.clone(), + public_key: access_key_pk.clone(), + access_key: <_>::try_from_slice(&state_change) + .expect("Failed to parse internally stored access key"), + } + } else { + StateChangeValue::AccessKeyDeletion { + account_id: account_id.clone(), + public_key: access_key_pk.clone(), + } + }, + } + })); + } + + Ok(changes) + } + + pub fn from_code_changes>( + changes_per_key: &mut dyn Iterator>, + account_id: &AccountId, + ) -> Result { + let mut changes = Self::new(); + + for change in changes_per_key { + let (_, state_changes) = change?; + changes.extend(state_changes.into_iter().map(|(cause, state_change)| { + StateChangeWithCause { + cause, + value: if let Some(state_change) = state_change { + StateChangeValue::CodeUpdate { + account_id: account_id.clone(), + code: state_change.into(), + } + } else { + StateChangeValue::CodeDeletion { account_id: account_id.clone() } + }, + } + })); + } + + Ok(changes) + } + + pub fn from_data_changes>( + changes_per_key: &mut dyn Iterator>, + account_id: &AccountId, + ) -> Result { + let mut changes = Self::new(); + + for change in changes_per_key { + let (key, state_changes) = change?; + + let key = KeyForData::parse_data_key(key.as_ref(), &account_id) + .expect("Failed to parse internally stored data key"); + + changes.extend(state_changes.into_iter().map(|(cause, state_change)| { + StateChangeWithCause { + cause, + value: if let Some(state_change) = state_change { + StateChangeValue::DataUpdate { + account_id: account_id.clone(), + key: key.to_vec().into(), + value: state_change.into(), + } + } else { + StateChangeValue::DataDeletion { + account_id: account_id.clone(), + key: key.to_vec().into(), + } + }, + } + })); + } + + Ok(changes) + } } #[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, Serialize)] @@ -150,25 +326,21 @@ impl StateRootNode { /// EpochId of epoch T is the hash of last block in T-2 /// EpochId of first two epochs is 0 #[derive( + Debug, + Clone, + Default, Hash, Eq, PartialEq, - Clone, - Debug, + PartialOrd, + DeriveAsRef, BorshSerialize, BorshDeserialize, Serialize, - Default, - PartialOrd, )] +#[as_ref(forward)] pub struct EpochId(pub CryptoHash); -impl AsRef<[u8]> for EpochId { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } -} - /// Stores validator and its stake. #[derive(BorshSerialize, BorshDeserialize, Serialize, Debug, Clone, PartialEq, Eq)] pub struct ValidatorStake { @@ -253,6 +425,19 @@ pub enum BlockId { pub type MaybeBlockId = Option; +#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BlockIdOrFinality { + BlockId(BlockId), + Finality(Finality), +} + +impl BlockIdOrFinality { + pub fn latest() -> Self { + Self::Finality(Finality::None) + } +} + #[derive(Default, BorshSerialize, BorshDeserialize, Serialize, Clone, Debug, PartialEq)] pub struct ValidatorStats { pub produced: NumBlocks, diff --git a/core/primitives/src/utils.rs b/core/primitives/src/utils.rs index 7c3e8f03d4c..d6db6c5cb37 100644 --- a/core/primitives/src/utils.rs +++ b/core/primitives/src/utils.rs @@ -2,7 +2,7 @@ use std::cmp::max; use std::convert::AsRef; use std::fmt; -use borsh::BorshSerialize; +use borsh::{BorshDeserialize, BorshSerialize}; use byteorder::{LittleEndian, WriteBytesExt}; use chrono::{DateTime, NaiveDateTime, Utc}; use rand::distributions::Alphanumeric; @@ -35,89 +35,237 @@ pub mod col { pub const DELAYED_RECEIPT: &[u8] = &[8]; } -fn key_for_column_account_id(column: &[u8], account_key: &AccountId) -> Vec { - let mut key = Vec::with_capacity(column.len() + account_key.len()); - key.extend(column); - key.extend(account_key.as_bytes()); - key -} +#[derive(derive_more::AsRef, derive_more::Into)] +struct KeyForColumnAccountId(Vec); -pub fn key_for_account(account_key: &AccountId) -> Vec { - key_for_column_account_id(col::ACCOUNT, account_key) -} +impl KeyForColumnAccountId { + pub fn estimate_len(column: &[u8], account_id: &AccountId) -> usize { + column.len() + account_id.len() + } -pub fn key_for_data(account_id: &AccountId, data: &[u8]) -> Vec { - let mut bytes = key_for_account(account_id); - bytes.extend(ACCOUNT_DATA_SEPARATOR); - bytes.extend(data); - bytes + pub fn with_capacity(column: &[u8], account_id: &AccountId, reserve_capacity: usize) -> Self { + let mut key = + Vec::with_capacity(Self::estimate_len(&column, &account_id) + reserve_capacity); + key.extend(column); + key.extend(account_id.as_bytes()); + debug_assert_eq!(key.len(), Self::estimate_len(&column, &account_id)); + Self(key) + } } -pub fn prefix_for_access_key(account_id: &AccountId) -> Vec { - let mut key = key_for_column_account_id(col::ACCESS_KEY, account_id); - key.extend(col::ACCESS_KEY); - key -} +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForAccount(Vec); -pub fn prefix_for_data(account_id: &AccountId) -> Vec { - let mut prefix = key_for_account(account_id); - prefix.extend(ACCOUNT_DATA_SEPARATOR); - prefix -} +impl KeyForAccount { + pub fn estimate_len(account_id: &AccountId) -> usize { + KeyForColumnAccountId::estimate_len(col::ACCOUNT, account_id) + } + + pub fn with_capacity(account_id: &AccountId, reserve_capacity: usize) -> Self { + let key = KeyForColumnAccountId::with_capacity(col::ACCOUNT, account_id, reserve_capacity); + debug_assert_eq!(key.0.len(), Self::estimate_len(&account_id)); + Self(key.into()) + } -pub fn key_for_access_key(account_id: &AccountId, public_key: &PublicKey) -> Vec { - let mut key = key_for_column_account_id(col::ACCESS_KEY, account_id); - key.extend(col::ACCESS_KEY); - key.extend(&public_key.try_to_vec().expect("Failed to serialize public key")); - key + pub fn new(account_id: &AccountId) -> Self { + Self::with_capacity(&account_id, 0) + } } -pub fn key_for_all_access_keys(account_id: &AccountId) -> Vec { - key_for_column_account_id(col::ACCESS_KEY, account_id) +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForAccessKey(Vec); + +impl KeyForAccessKey { + fn estimate_prefix_len(account_id: &AccountId) -> usize { + KeyForColumnAccountId::estimate_len(col::ACCESS_KEY, account_id) + col::ACCESS_KEY.len() + } + + pub fn estimate_len(account_id: &AccountId, public_key: &PublicKey) -> usize { + let serialized_public_key = + public_key.try_to_vec().expect("Failed to serialize public key"); + Self::estimate_prefix_len(account_id) + serialized_public_key.len() + } + + pub fn get_prefix_with_capacity(account_id: &AccountId, reserved_capacity: usize) -> Self { + let mut key: Vec = KeyForColumnAccountId::with_capacity( + col::ACCESS_KEY, + account_id, + col::ACCESS_KEY.len() + reserved_capacity, + ) + .into(); + key.extend(col::ACCESS_KEY); + Self(key) + } + + pub fn get_prefix(account_id: &AccountId) -> Self { + Self::get_prefix_with_capacity(account_id, 0) + } + + pub fn new(account_id: &AccountId, public_key: &PublicKey) -> Self { + let serialized_public_key = + public_key.try_to_vec().expect("Failed to serialize public key"); + let mut key = Self::get_prefix_with_capacity(&account_id, serialized_public_key.len()); + key.0.extend(&serialized_public_key); + debug_assert_eq!(key.0.len(), Self::estimate_len(&account_id, &public_key)); + key + } + + pub fn parse_public_key( + raw_key: &[u8], + account_id: &AccountId, + ) -> Result { + let prefix_len = Self::estimate_prefix_len(account_id); + if raw_key.len() < prefix_len { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "raw key is too short for KeyForAccessKey", + )); + } + PublicKey::try_from_slice(&raw_key[prefix_len..]) + } } -pub fn key_for_code(account_key: &AccountId) -> Vec { - key_for_column_account_id(col::CODE, account_key) +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForData(Vec); + +impl KeyForData { + pub fn estimate_len(account_id: &AccountId, data: &[u8]) -> usize { + KeyForAccount::estimate_len(&account_id) + ACCOUNT_DATA_SEPARATOR.len() + data.len() + } + + pub fn get_prefix_with_capacity(account_id: &AccountId, reserved_capacity: usize) -> Self { + let mut prefix: Vec = KeyForAccount::with_capacity( + account_id, + ACCOUNT_DATA_SEPARATOR.len() + reserved_capacity, + ) + .into(); + prefix.extend(ACCOUNT_DATA_SEPARATOR); + Self(prefix) + } + + pub fn get_prefix(account_id: &AccountId) -> Self { + Self::get_prefix_with_capacity(account_id, 0) + } + + pub fn new(account_id: &AccountId, data: &[u8]) -> Self { + let mut key = Self::get_prefix_with_capacity(&account_id, data.len()); + key.0.extend(data); + debug_assert_eq!(key.0.len(), Self::estimate_len(&account_id, &data)); + key + } + + pub fn parse_data_key<'a>( + raw_key: &'a [u8], + account_id: &AccountId, + ) -> Result<&'a [u8], std::io::Error> { + let prefix_len = Self::estimate_len(account_id, &[]); + if raw_key.len() < prefix_len { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "raw key is too short for KeyForData", + )); + } + Ok(&raw_key[prefix_len..]) + } } -pub fn key_for_received_data(account_id: &AccountId, data_id: &CryptoHash) -> Vec { - let mut key = key_for_column_account_id(col::RECEIVED_DATA, account_id); - key.extend(ACCOUNT_DATA_SEPARATOR); - key.extend(data_id.as_ref()); - key +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForCode(Vec); + +impl KeyForCode { + pub fn new(account_id: &AccountId) -> Self { + Self(KeyForColumnAccountId::with_capacity(col::CODE, account_id, 0).into()) + } } -pub fn key_for_postponed_receipt_id(account_id: &AccountId, data_id: &CryptoHash) -> Vec { - let mut key = key_for_column_account_id(col::POSTPONED_RECEIPT_ID, account_id); - key.extend(ACCOUNT_DATA_SEPARATOR); - key.extend(data_id.as_ref()); - key +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForReceivedData(Vec); + +impl KeyForReceivedData { + pub fn new(account_id: &AccountId, data_id: &CryptoHash) -> Self { + let mut key: Vec = KeyForColumnAccountId::with_capacity( + col::RECEIVED_DATA, + account_id, + ACCOUNT_DATA_SEPARATOR.len() + data_id.as_ref().len(), + ) + .into(); + key.extend(ACCOUNT_DATA_SEPARATOR); + key.extend(data_id.as_ref()); + Self(key) + } } -pub fn key_for_pending_data_count(account_id: &AccountId, receipt_id: &CryptoHash) -> Vec { - let mut key = key_for_column_account_id(col::PENDING_DATA_COUNT, account_id); - key.extend(ACCOUNT_DATA_SEPARATOR); - key.extend(receipt_id.as_ref()); - key +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForPostponedReceiptId(Vec); + +impl KeyForPostponedReceiptId { + pub fn new(account_id: &AccountId, data_id: &CryptoHash) -> Self { + let mut key: Vec = KeyForColumnAccountId::with_capacity( + col::POSTPONED_RECEIPT_ID, + account_id, + ACCOUNT_DATA_SEPARATOR.len() + data_id.as_ref().len(), + ) + .into(); + key.extend(ACCOUNT_DATA_SEPARATOR); + key.extend(data_id.as_ref()); + Self(key) + } } -pub fn key_for_postponed_receipt(account_id: &AccountId, receipt_id: &CryptoHash) -> Vec { - let mut key = key_for_column_account_id(col::POSTPONED_RECEIPT, account_id); - key.extend(ACCOUNT_DATA_SEPARATOR); - key.extend(receipt_id.as_ref()); - key +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForPendingDataCount(Vec); + +impl KeyForPendingDataCount { + pub fn new(account_id: &AccountId, receipt_id: &CryptoHash) -> Self { + let mut key: Vec = KeyForColumnAccountId::with_capacity( + col::PENDING_DATA_COUNT, + account_id, + ACCOUNT_DATA_SEPARATOR.len() + receipt_id.as_ref().len(), + ) + .into(); + key.extend(ACCOUNT_DATA_SEPARATOR); + key.extend(receipt_id.as_ref()); + Self(key) + } } -pub fn key_for_all_postponed_receipts(account_id: &AccountId) -> Vec { - let mut key = key_for_column_account_id(col::POSTPONED_RECEIPT, account_id); - key.extend(ACCOUNT_DATA_SEPARATOR); - key +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForPostponedReceipt(Vec); + +impl KeyForPostponedReceipt { + pub fn new(account_id: &AccountId, receipt_id: &CryptoHash) -> Self { + let mut key: Vec = KeyForColumnAccountId::with_capacity( + col::POSTPONED_RECEIPT, + account_id, + ACCOUNT_DATA_SEPARATOR.len() + receipt_id.as_ref().len(), + ) + .into(); + key.extend(ACCOUNT_DATA_SEPARATOR); + key.extend(receipt_id.as_ref()); + Self(key) + } } -pub fn key_for_delayed_receipt(index: u64) -> Vec { - let mut key = col::DELAYED_RECEIPT.to_vec(); - key.extend(&index.to_le_bytes()); - key +#[derive(derive_more::AsRef, derive_more::Into)] +#[as_ref(forward)] +pub struct KeyForDelayedReceipt(Vec); + +impl KeyForDelayedReceipt { + pub fn new(index: u64) -> Self { + let index_bytes = index.to_le_bytes(); + let mut key = Vec::with_capacity(col::DELAYED_RECEIPT.len() + index_bytes.len()); + key.extend(col::DELAYED_RECEIPT); + key.extend(&index_bytes); + Self(key) + } } pub fn create_nonce_with_nonce(base: &CryptoHash, salt: u64) -> CryptoHash { @@ -317,8 +465,53 @@ where #[cfg(test)] mod tests { + use near_crypto::KeyType; + use super::*; + #[test] + fn test_key_for_account_consistency() { + let account_id = AccountId::from("test.account"); + assert_eq!( + (KeyForAccount::new(&account_id).as_ref() as &[u8]).len(), + KeyForAccount::estimate_len(&account_id) + ); + } + + #[test] + fn test_key_for_access_key_consistency() { + let account_id = AccountId::from("test.account"); + let public_key = PublicKey::empty(KeyType::ED25519); + let key_prefix = KeyForAccessKey::get_prefix(&account_id); + assert_eq!( + (key_prefix.as_ref() as &[u8]).len(), + KeyForAccessKey::estimate_prefix_len(&account_id) + ); + let key = KeyForAccessKey::new(&account_id, &public_key); + assert_eq!( + (key.as_ref() as &[u8]).len(), + KeyForAccessKey::estimate_len(&account_id, &public_key) + ); + assert_eq!( + KeyForAccessKey::parse_public_key(key.as_ref(), &account_id).unwrap(), + public_key + ); + } + + #[test] + fn test_key_for_data_consistency() { + let account_id = AccountId::from("test.account"); + let data_key = b"0123456789" as &[u8]; + let key_prefix = KeyForData::get_prefix(&account_id); + assert_eq!( + (key_prefix.as_ref() as &[u8]).len(), + KeyForData::estimate_len(&account_id, &[]) + ); + let key = KeyForData::new(&account_id, &data_key); + assert_eq!((key.as_ref() as &[u8]).len(), KeyForData::estimate_len(&account_id, &data_key)); + assert_eq!(KeyForData::parse_data_key(key.as_ref(), &account_id).unwrap(), data_key); + } + #[test] fn test_is_valid_account_id() { let ok_account_ids = vec![ diff --git a/core/primitives/src/views.rs b/core/primitives/src/views.rs index e14e32907b4..9d19374efab 100644 --- a/core/primitives/src/views.rs +++ b/core/primitives/src/views.rs @@ -1,3 +1,8 @@ +//! This module defines "stable" internal API to view internal data using view_client. +//! +//! These types should only change when we cannot avoid this. Thus, when the counterpart internal +//! type gets changed, the view should preserve the old shape and only re-map the necessary bits +//! from the source structure in the relevant `From` impl. use std::borrow::Cow; use std::convert::{TryFrom, TryInto}; use std::fmt; @@ -18,7 +23,8 @@ use crate::merkle::MerklePath; use crate::receipt::{ActionReceipt, DataReceipt, DataReceiver, Receipt, ReceiptEnum}; use crate::rpc::RpcPagination; use crate::serialize::{ - from_base64, option_base64_format, option_u128_dec_format, to_base64, u128_dec_format, + base64_format, from_base64, option_base64_format, option_u128_dec_format, to_base64, + u128_dec_format, }; use crate::sharding::{ChunkHash, ShardChunk, ShardChunkHeader, ShardChunkHeaderInner}; use crate::state_record::StateRecord; @@ -29,7 +35,8 @@ use crate::transaction::{ }; use crate::types::{ AccountId, Balance, BlockHeight, EpochId, FunctionArgs, Gas, Nonce, NumBlocks, ShardId, - StateChanges, StateRoot, StorageUsage, StoreKey, ValidatorStake, Version, + StateChangeCause, StateChangeValue, StateChangeWithCause, StateRoot, StorageUsage, StoreKey, + StoreValue, ValidatorStake, Version, }; /// A view of the account @@ -191,11 +198,27 @@ pub enum QueryResponseKind { #[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] #[serde(tag = "request_type", rename_all = "snake_case")] pub enum QueryRequest { - ViewAccount { account_id: AccountId }, - ViewState { account_id: AccountId, prefix: StoreKey }, - ViewAccessKey { account_id: AccountId, public_key: PublicKey }, - ViewAccessKeyList { account_id: AccountId }, - CallFunction { account_id: AccountId, method_name: String, args: FunctionArgs }, + ViewAccount { + account_id: AccountId, + }, + ViewState { + account_id: AccountId, + #[serde(rename = "prefix_base64", with = "base64_format")] + prefix: StoreKey, + }, + ViewAccessKey { + account_id: AccountId, + public_key: PublicKey, + }, + ViewAccessKeyList { + account_id: AccountId, + }, + CallFunction { + account_id: AccountId, + method_name: String, + #[serde(rename = "args_base64", with = "base64_format")] + args: FunctionArgs, + }, } #[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] @@ -847,11 +870,6 @@ impl fmt::Debug for FinalExecutionOutcomeView { } } -#[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Debug, Clone)] -pub struct StateChangesView { - pub changes: StateChanges, -} - #[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] pub struct ValidatorStakeView { pub account_id: AccountId, @@ -1041,19 +1059,130 @@ pub struct GasPriceView { pub gas_price: Balance, } -/// Different types of finality. -#[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize, Clone, Debug, PartialEq, Eq)] -pub enum Finality { - #[serde(rename = "optimistic")] - None, - #[serde(rename = "near-final")] - DoomSlug, - #[serde(rename = "final")] - NFG, +/// See crate::types::StateChangeCause for details. +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "type")] +pub enum StateChangeCauseView { + NotWritableToDisk, + InitialState, + TransactionProcessing { tx_hash: CryptoHash }, + ActionReceiptProcessingStarted { receipt_hash: CryptoHash }, + ActionReceiptGasReward { receipt_hash: CryptoHash }, + ReceiptProcessing { receipt_hash: CryptoHash }, + PostponedReceipt { receipt_hash: CryptoHash }, + UpdatedDelayedReceipts, + ValidatorAccountsUpdate, +} + +impl From for StateChangeCauseView { + fn from(state_change_cause: StateChangeCause) -> Self { + match state_change_cause { + StateChangeCause::NotWritableToDisk => Self::NotWritableToDisk, + StateChangeCause::InitialState => Self::InitialState, + StateChangeCause::TransactionProcessing { tx_hash } => { + Self::TransactionProcessing { tx_hash } + } + StateChangeCause::ActionReceiptProcessingStarted { receipt_hash } => { + Self::ActionReceiptProcessingStarted { receipt_hash } + } + StateChangeCause::ActionReceiptGasReward { receipt_hash } => { + Self::ActionReceiptGasReward { receipt_hash } + } + StateChangeCause::ReceiptProcessing { receipt_hash } => { + Self::ReceiptProcessing { receipt_hash } + } + StateChangeCause::PostponedReceipt { receipt_hash } => { + Self::PostponedReceipt { receipt_hash } + } + StateChangeCause::UpdatedDelayedReceipts => Self::UpdatedDelayedReceipts, + StateChangeCause::ValidatorAccountsUpdate => Self::ValidatorAccountsUpdate, + } + } } -impl Default for Finality { - fn default() -> Self { - Finality::NFG +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "type", content = "change")] +pub enum StateChangeValueView { + AccountUpdate { + account_id: AccountId, + #[serde(flatten)] + account: AccountView, + }, + AccountDeletion { + account_id: AccountId, + }, + AccessKeyUpdate { + account_id: AccountId, + public_key: PublicKey, + access_key: AccessKeyView, + }, + AccessKeyDeletion { + account_id: AccountId, + public_key: PublicKey, + }, + DataUpdate { + account_id: AccountId, + #[serde(rename = "key_base64", with = "base64_format")] + key: StoreKey, + #[serde(rename = "value_base64", with = "base64_format")] + value: StoreValue, + }, + DataDeletion { + account_id: AccountId, + #[serde(rename = "key_base64", with = "base64_format")] + key: StoreKey, + }, + CodeUpdate { + account_id: AccountId, + #[serde(rename = "code_base64", with = "base64_format")] + code: Vec, + }, + CodeDeletion { + account_id: AccountId, + }, +} + +impl From for StateChangeValueView { + fn from(state_change: StateChangeValue) -> Self { + match state_change { + StateChangeValue::AccountUpdate { account_id, account } => { + Self::AccountUpdate { account_id, account: account.into() } + } + StateChangeValue::AccountDeletion { account_id } => { + Self::AccountDeletion { account_id } + } + StateChangeValue::AccessKeyUpdate { account_id, public_key, access_key } => { + Self::AccessKeyUpdate { account_id, public_key, access_key: access_key.into() } + } + StateChangeValue::AccessKeyDeletion { account_id, public_key } => { + Self::AccessKeyDeletion { account_id, public_key } + } + StateChangeValue::DataUpdate { account_id, key, value } => { + Self::DataUpdate { account_id, key, value } + } + StateChangeValue::DataDeletion { account_id, key } => { + Self::DataDeletion { account_id, key } + } + StateChangeValue::CodeUpdate { account_id, code } => { + Self::CodeUpdate { account_id, code } + } + StateChangeValue::CodeDeletion { account_id } => Self::CodeDeletion { account_id }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct StateChangeWithCauseView { + pub cause: StateChangeCauseView, + #[serde(flatten)] + pub value: StateChangeValueView, +} + +impl From for StateChangeWithCauseView { + fn from(state_change_with_cause: StateChangeWithCause) -> Self { + let StateChangeWithCause { cause, value } = state_change_with_cause; + Self { cause: cause.into(), value: value.into() } } } + +pub type StateChangesView = Vec; diff --git a/core/store/src/lib.rs b/core/store/src/lib.rs index 1dd6074e4ae..e05fd9bee0c 100644 --- a/core/store/src/lib.rs +++ b/core/store/src/lib.rs @@ -18,8 +18,8 @@ use near_primitives::receipt::{Receipt, ReceivedData}; use near_primitives::serialize::to_base; use near_primitives::types::{AccountId, StorageUsage}; use near_primitives::utils::{ - key_for_access_key, key_for_account, key_for_code, key_for_postponed_receipt, - key_for_received_data, prefix_for_access_key, prefix_for_data, + KeyForAccessKey, KeyForAccount, KeyForCode, KeyForData, KeyForPostponedReceipt, + KeyForReceivedData, }; use crate::db::{DBOp, DBTransaction, Database, RocksDB}; @@ -223,11 +223,11 @@ pub fn create_store(path: &str) -> Arc { /// Reads an object from Trie. /// # Errors /// see StorageError -pub fn get( +pub fn get>( state_update: &TrieUpdate, - key: &[u8], + key: K, ) -> Result, StorageError> { - state_update.get(key).and_then(|opt| { + state_update.get(key.as_ref()).and_then(|opt| { opt.map_or_else( || Ok(None), |data| { @@ -242,8 +242,9 @@ pub fn get( } /// Writes an object into Trie. -pub fn set(state_update: &mut TrieUpdate, key: Vec, value: &T) { - value.try_to_vec().ok().map(|data| state_update.set(key, data)).or_else(|| None); +pub fn set>>(state_update: &mut TrieUpdate, key: K, value: &T) { + let data = value.try_to_vec().expect("Borsh serializer is not expected to ever fail"); + state_update.set(key.into(), data); } /// Number of bytes account and all of it's other data occupies in the storage. @@ -252,14 +253,14 @@ pub fn total_account_storage(_account_id: &AccountId, account: &Account) -> Stor } pub fn set_account(state_update: &mut TrieUpdate, key: &AccountId, account: &Account) { - set(state_update, key_for_account(key), account) + set(state_update, KeyForAccount::new(key), account) } pub fn get_account( state_update: &TrieUpdate, key: &AccountId, ) -> Result, StorageError> { - get(state_update, &key_for_account(key)) + get(state_update, &KeyForAccount::new(key)) } pub fn set_received_data( @@ -268,7 +269,7 @@ pub fn set_received_data( data_id: &CryptoHash, data: &ReceivedData, ) { - set(state_update, key_for_received_data(account_id, data_id), data); + set(state_update, KeyForReceivedData::new(account_id, data_id), data); } pub fn get_received_data( @@ -276,11 +277,11 @@ pub fn get_received_data( account_id: &AccountId, data_id: &CryptoHash, ) -> Result, StorageError> { - get(state_update, &key_for_received_data(account_id, data_id)) + get(state_update, &KeyForReceivedData::new(account_id, data_id)) } pub fn set_receipt(state_update: &mut TrieUpdate, receipt: &Receipt) { - let key = key_for_postponed_receipt(&receipt.receiver_id, &receipt.receipt_id); + let key = KeyForPostponedReceipt::new(&receipt.receiver_id, &receipt.receipt_id); set(state_update, key, receipt); } @@ -289,7 +290,7 @@ pub fn get_receipt( account_id: &AccountId, receipt_id: &CryptoHash, ) -> Result, StorageError> { - get(state_update, &key_for_postponed_receipt(account_id, receipt_id)) + get(state_update, &KeyForPostponedReceipt::new(account_id, receipt_id)) } pub fn set_access_key( @@ -298,7 +299,7 @@ pub fn set_access_key( public_key: &PublicKey, access_key: &AccessKey, ) { - set(state_update, key_for_access_key(account_id, public_key), access_key); + set(state_update, KeyForAccessKey::new(account_id, public_key), access_key); } pub fn get_access_key( @@ -306,7 +307,7 @@ pub fn get_access_key( account_id: &AccountId, public_key: &PublicKey, ) -> Result, StorageError> { - get(state_update, &key_for_access_key(account_id, public_key)) + get(state_update, &KeyForAccessKey::new(account_id, public_key)) } pub fn get_access_key_raw( @@ -317,7 +318,7 @@ pub fn get_access_key_raw( } pub fn set_code(state_update: &mut TrieUpdate, account_id: &AccountId, code: &ContractCode) { - state_update.set(key_for_code(account_id), code.code.clone()); + state_update.set(KeyForCode::new(account_id).into(), code.code.clone()); } pub fn get_code( @@ -325,7 +326,7 @@ pub fn get_code( account_id: &AccountId, ) -> Result, StorageError> { state_update - .get(&key_for_code(account_id)) + .get(KeyForCode::new(account_id)) .map(|opt| opt.map(|code| ContractCode::new(code.to_vec()))) } @@ -334,9 +335,9 @@ pub fn remove_account( state_update: &mut TrieUpdate, account_id: &AccountId, ) -> Result<(), StorageError> { - state_update.remove(&key_for_account(account_id)); - state_update.remove(&key_for_code(account_id)); - state_update.remove_starts_with(&prefix_for_access_key(account_id))?; - state_update.remove_starts_with(&prefix_for_data(account_id))?; + state_update.remove(KeyForAccount::new(account_id)); + state_update.remove(KeyForCode::new(account_id)); + state_update.remove_starts_with(KeyForAccessKey::get_prefix(account_id))?; + state_update.remove_starts_with(KeyForData::get_prefix(account_id))?; Ok(()) } diff --git a/core/store/src/trie/iterator.rs b/core/store/src/trie/iterator.rs index 24aff5aa2ad..4ca58869ce4 100644 --- a/core/store/src/trie/iterator.rs +++ b/core/store/src/trie/iterator.rs @@ -57,8 +57,8 @@ impl<'a> TrieIterator<'a> { } /// Position the iterator on the first element with key => `key`. - pub fn seek(&mut self, key: &[u8]) -> Result<(), StorageError> { - self.seek_nibble_slice(NibbleSlice::new(key)) + pub fn seek>(&mut self, key: K) -> Result<(), StorageError> { + self.seek_nibble_slice(NibbleSlice::new(key.as_ref())) } pub(crate) fn seek_nibble_slice(&mut self, mut key: NibbleSlice) -> Result<(), StorageError> { diff --git a/core/store/src/trie/mod.rs b/core/store/src/trie/mod.rs index cd1ba99e2ef..12156098635 100644 --- a/core/store/src/trie/mod.rs +++ b/core/store/src/trie/mod.rs @@ -11,7 +11,7 @@ use cached::Cached; use near_primitives::challenge::PartialState; use near_primitives::hash::{hash, CryptoHash}; -use near_primitives::types::{StateChangeCause, StateChanges, StateRoot, StateRootNode}; +use near_primitives::types::{RawStateChanges, StateChangeCause, StateRoot, StateRootNode}; use crate::db::DBCol::ColKeyValueChanges; use crate::trie::insert_delete::NodesStorage; @@ -522,7 +522,7 @@ impl TrieChanges { pub struct WrappedTrieChanges { trie: Arc, trie_changes: TrieChanges, - kv_changes: StateChanges, + kv_changes: RawStateChanges, block_hash: CryptoHash, } @@ -530,7 +530,7 @@ impl WrappedTrieChanges { pub fn new( trie: Arc, trie_changes: TrieChanges, - kv_changes: StateChanges, + kv_changes: RawStateChanges, block_hash: CryptoHash, ) -> Self { WrappedTrieChanges { trie, trie_changes, kv_changes, block_hash } diff --git a/core/store/src/trie/update.rs b/core/store/src/trie/update.rs index a714893e7b9..f8e55662eac 100644 --- a/core/store/src/trie/update.rs +++ b/core/store/src/trie/update.rs @@ -3,7 +3,7 @@ use std::iter::Peekable; use std::sync::Arc; use near_primitives::hash::CryptoHash; -use near_primitives::types::{StateChangeCause, StateChanges}; +use near_primitives::types::{RawStateChanges, StateChangeCause}; use crate::trie::TrieChanges; use crate::StorageError; @@ -17,7 +17,7 @@ pub type TrieUpdates = BTreeMap, Option>>; pub struct TrieUpdate { pub trie: Arc, root: CryptoHash, - committed: StateChanges, + committed: RawStateChanges, prospective: TrieUpdates, } @@ -50,7 +50,8 @@ impl TrieUpdate { pub fn new(trie: Arc, root: CryptoHash) -> Self { TrieUpdate { trie, root, committed: Default::default(), prospective: Default::default() } } - pub fn get(&self, key: &[u8]) -> Result>, StorageError> { + pub fn get>(&self, key: K) -> Result>, StorageError> { + let key = key.as_ref(); if let Some(value) = self.prospective.get(key) { return Ok(value.as_ref().map(>::clone)); } else if let Some(changes) = self.committed.get(key) { @@ -62,7 +63,11 @@ impl TrieUpdate { self.trie.get(&self.root, key) } - pub fn get_ref(&self, key: &[u8]) -> Result, StorageError> { + pub fn get_ref>( + &self, + key: K, + ) -> Result, StorageError> { + let key = key.as_ref(); if let Some(value) = self.prospective.get(key) { return Ok(value.as_ref().map(TrieUpdateValuePtr::MemoryRef)); } else if let Some(changes) = self.committed.get(key) { @@ -101,25 +106,28 @@ impl TrieUpdate { Ok(res) } - pub fn committed_updates_per_cause(&self) -> &StateChanges { + pub fn committed_updates_per_cause(&self) -> &RawStateChanges { &self.committed } pub fn set(&mut self, key: Vec, value: Vec) { self.prospective.insert(key, Some(value)); } - pub fn remove(&mut self, key: &[u8]) { - self.prospective.insert(key.to_vec(), None); + pub fn remove>>(&mut self, key: K) { + self.prospective.insert(key.into(), None); } - pub fn remove_starts_with(&mut self, prefix: &[u8]) -> Result<(), StorageError> { + pub fn remove_starts_with>( + &mut self, + key_prefix: K, + ) -> Result<(), StorageError> { let mut keys = vec![]; - for key in self.iter(prefix)? { + for key in self.iter(key_prefix.as_ref())? { keys.push(key); } for key in keys { let key = key?; - self.remove(&key); + self.remove(key); } Ok(()) } @@ -163,8 +171,8 @@ impl TrieUpdate { } /// Returns Error if the underlying storage fails - pub fn iter(&self, prefix: &[u8]) -> Result { - TrieUpdateIterator::new(self, prefix, b"", None) + pub fn iter>(&self, prefix: K) -> Result { + TrieUpdateIterator::new(self, prefix.as_ref(), b"", None) } pub fn range( @@ -377,7 +385,7 @@ mod tests { // Delete non-existing element. let mut trie_update = TrieUpdate::new(trie.clone(), CryptoHash::default()); - trie_update.remove(b"dog"); + trie_update.remove(b"dog".to_vec()); trie_update .commit(StateChangeCause::TransactionProcessing { tx_hash: CryptoHash::default() }); let (store_update, new_root) = trie_update.finalize().unwrap().into(trie.clone()).unwrap(); @@ -387,7 +395,7 @@ mod tests { // Add and right away delete element. let mut trie_update = TrieUpdate::new(trie.clone(), CryptoHash::default()); trie_update.set(b"dog".to_vec(), b"puppy".to_vec()); - trie_update.remove(b"dog"); + trie_update.remove(b"dog".to_vec()); trie_update .commit(StateChangeCause::TransactionProcessing { tx_hash: CryptoHash::default() }); let (store_update, new_root) = trie_update.finalize().unwrap().into(trie.clone()).unwrap(); @@ -403,7 +411,7 @@ mod tests { store_update.commit().ok(); assert_ne!(new_root, CryptoHash::default()); let mut trie_update = TrieUpdate::new(trie.clone(), new_root); - trie_update.remove(b"dog"); + trie_update.remove(b"dog".to_vec()); trie_update .commit(StateChangeCause::TransactionProcessing { tx_hash: CryptoHash::default() }); let (store_update, new_root) = trie_update.finalize().unwrap().into(trie.clone()).unwrap(); @@ -435,7 +443,7 @@ mod tests { assert_eq!(values.unwrap(), vec![b"dog".to_vec()]); let mut trie_update = TrieUpdate::new(trie.clone(), new_root); - trie_update.remove(b"dog"); + trie_update.remove(b"dog".to_vec()); let values: Result>, _> = trie_update.iter(b"dog").unwrap().collect(); assert_eq!(values.unwrap().len(), 0); @@ -444,7 +452,7 @@ mod tests { trie_update.set(b"dog2".to_vec(), b"puppy".to_vec()); trie_update .commit(StateChangeCause::TransactionProcessing { tx_hash: CryptoHash::default() }); - trie_update.remove(b"dog2"); + trie_update.remove(b"dog2".to_vec()); let values: Result>, _> = trie_update.iter(b"dog").unwrap().collect(); assert_eq!(values.unwrap(), vec![b"dog".to_vec()]); diff --git a/near/src/runtime.rs b/near/src/runtime.rs index 7a828b08676..04d9435d90a 100644 --- a/near/src/runtime.rs +++ b/near/src/runtime.rs @@ -31,7 +31,7 @@ use near_primitives::types::{ StateChangeCause, StateChanges, StateChangesRequest, StateRoot, StateRootNode, ValidatorStake, ValidatorStats, }; -use near_primitives::utils::{prefix_for_access_key, ACCOUNT_DATA_SEPARATOR}; +use near_primitives::utils::{KeyForAccessKey, ACCOUNT_DATA_SEPARATOR}; use near_primitives::views::{ AccessKeyInfoView, CallResult, EpochValidatorInfo, QueryError, QueryRequest, QueryResponse, QueryResponseKind, ViewStateResult, @@ -1180,7 +1180,8 @@ impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime { account_id: &AccountId, ) -> Result, Box> { let state_update = TrieUpdate::new(self.trie.clone(), state_root); - let prefix = prefix_for_access_key(account_id); + let prefix = KeyForAccessKey::get_prefix(account_id); + let prefix: &[u8] = prefix.as_ref(); let access_keys = match state_update.iter(&prefix) { Ok(iter) => iter .map(|key| { diff --git a/near/tests/rpc_nodes.rs b/near/tests/rpc_nodes.rs index ae03726ac19..428236fd0f9 100644 --- a/near/tests/rpc_nodes.rs +++ b/near/tests/rpc_nodes.rs @@ -12,7 +12,7 @@ use near_primitives::serialize::to_base64; use near_primitives::test_utils::{heavy_test, init_integration_logger}; use near_primitives::transaction::SignedTransaction; use near_primitives::types::BlockId; -use near_primitives::views::{FinalExecutionStatus, Finality, QueryResponseKind}; +use near_primitives::views::{FinalExecutionStatus, QueryResponseKind}; use testlib::{genesis_block, start_nodes}; /// Starts 2 validators and 2 light clients (not tracking anything). @@ -49,24 +49,22 @@ fn test_tx_propagation() { let tx_hash_clone = tx_hash.clone(); // We are sending this tx unstop, just to get over the warm up period. // Probably make sense to stop after 1 time though. - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - if res.unwrap().unwrap().header.height > 1 { - let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); - let bytes = transaction_copy.try_to_vec().unwrap(); - actix::spawn( - client - .broadcast_tx_async(to_base64(&bytes)) - .map_err(|err| panic!(err.to_string())) - .map_ok(move |result| { - assert_eq!(String::from(&tx_hash_clone), result) - }) - .map(drop), - ); - } - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + if res.unwrap().unwrap().header.height > 1 { + let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); + let bytes = transaction_copy.try_to_vec().unwrap(); + actix::spawn( + client + .broadcast_tx_async(to_base64(&bytes)) + .map_err(|err| panic!(err.to_string())) + .map_ok(move |result| { + assert_eq!(String::from(&tx_hash_clone), result) + }) + .map(drop), + ); + } + future::ready(()) + })); actix::spawn( view_client .send(TxStatus { tx_hash, signer_account_id: "near.1".to_string() }) @@ -125,30 +123,28 @@ fn test_tx_propagation_through_rpc() { let transaction_copy = transaction.clone(); // We are sending this tx unstop, just to get over the warm up period. // Probably make sense to stop after 1 time though. - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - if res.unwrap().unwrap().header.height > 1 { - let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); - let bytes = transaction_copy.try_to_vec().unwrap(); - actix::spawn( - client - .broadcast_tx_commit(to_base64(&bytes)) - .map_err(|err| panic!(err.to_string())) - .map_ok(move |result| { - if result.status - == FinalExecutionStatus::SuccessValue("".to_string()) - { - System::current().stop(); - } else { - panic!("wrong transaction status"); - } - }) - .map(drop), - ); - } - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + if res.unwrap().unwrap().header.height > 1 { + let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); + let bytes = transaction_copy.try_to_vec().unwrap(); + actix::spawn( + client + .broadcast_tx_commit(to_base64(&bytes)) + .map_err(|err| panic!(err.to_string())) + .map_ok(move |result| { + if result.status + == FinalExecutionStatus::SuccessValue("".to_string()) + { + System::current().stop(); + } else { + panic!("wrong transaction status"); + } + }) + .map(drop), + ); + } + future::ready(()) + })); }), 100, 20000, @@ -192,24 +188,22 @@ fn test_tx_status_with_light_client() { let transaction_copy = transaction.clone(); let signer_account_id = transaction_copy.transaction.signer_id.clone(); let tx_hash_clone = tx_hash.clone(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - if res.unwrap().unwrap().header.height > 1 { - let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); - let bytes = transaction_copy.try_to_vec().unwrap(); - actix::spawn( - client - .broadcast_tx_async(to_base64(&bytes)) - .map_err(|err| panic!("{:?}", err)) - .map_ok(move |result| { - assert_eq!(String::from(&tx_hash_clone), result) - }) - .map(drop), - ); - } - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + if res.unwrap().unwrap().header.height > 1 { + let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); + let bytes = transaction_copy.try_to_vec().unwrap(); + actix::spawn( + client + .broadcast_tx_async(to_base64(&bytes)) + .map_err(|err| panic!("{:?}", err)) + .map_ok(move |result| { + assert_eq!(String::from(&tx_hash_clone), result) + }) + .map(drop), + ); + } + future::ready(()) + })); let mut client = new_client(&format!("http://{}", rpc_addrs_copy1[2].clone())); actix::spawn( client @@ -265,24 +259,22 @@ fn test_tx_status_with_light_client1() { let transaction_copy = transaction.clone(); let signer_account_id = transaction_copy.transaction.signer_id.clone(); let tx_hash_clone = tx_hash.clone(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - if res.unwrap().unwrap().header.height > 1 { - let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); - let bytes = transaction_copy.try_to_vec().unwrap(); - actix::spawn( - client - .broadcast_tx_async(to_base64(&bytes)) - .map_err(|err| panic!("{}", err.to_string())) - .map_ok(move |result| { - assert_eq!(String::from(&tx_hash_clone), result) - }) - .map(drop), - ); - } - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + if res.unwrap().unwrap().header.height > 1 { + let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); + let bytes = transaction_copy.try_to_vec().unwrap(); + actix::spawn( + client + .broadcast_tx_async(to_base64(&bytes)) + .map_err(|err| panic!("{}", err.to_string())) + .map_ok(move |result| { + assert_eq!(String::from(&tx_hash_clone), result) + }) + .map(drop), + ); + } + future::ready(()) + })); let mut client = new_client(&format!("http://{}", rpc_addrs_copy1[2].clone())); actix::spawn( client @@ -320,29 +312,27 @@ fn test_rpc_routing() { WaitOrTimeout::new( Box::new(move |_ctx| { let rpc_addrs_copy = rpc_addrs.clone(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - if res.unwrap().unwrap().header.height > 1 { - let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); - actix::spawn( - client - .query_by_path("account/near.2".to_string(), "".to_string()) - .map_err(|err| { - println!("Error retrieving account: {:?}", err); - }) - .map_ok(move |result| match result.kind { - QueryResponseKind::ViewAccount(account_view) => { - assert_eq!(account_view.amount, TESTING_INIT_BALANCE); - System::current().stop(); - } - _ => panic!("wrong query response"), - }) - .map(drop), - ); - } - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + if res.unwrap().unwrap().header.height > 1 { + let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); + actix::spawn( + client + .query_by_path("account/near.2".to_string(), "".to_string()) + .map_err(|err| { + println!("Error retrieving account: {:?}", err); + }) + .map_ok(move |result| match result.kind { + QueryResponseKind::ViewAccount(account_view) => { + assert_eq!(account_view.amount, TESTING_INIT_BALANCE); + System::current().stop(); + } + _ => panic!("wrong query response"), + }) + .map(drop), + ); + } + future::ready(()) + })); }), 100, 20000, @@ -369,27 +359,22 @@ fn test_rpc_routing_error() { WaitOrTimeout::new( Box::new(move |_ctx| { let rpc_addrs_copy = rpc_addrs.clone(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - if res.unwrap().unwrap().header.height > 1 { - let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); - actix::spawn( - client - .query_by_path( - "account/nonexistent".to_string(), - "".to_string(), - ) - .map_err(|err| { - println!("error: {}", err.to_string()); - System::current().stop(); - }) - .map_ok(|_| panic!("wrong query response")) - .map(drop), - ); - } - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + if res.unwrap().unwrap().header.height > 1 { + let mut client = new_client(&format!("http://{}", rpc_addrs_copy[2])); + actix::spawn( + client + .query_by_path("account/nonexistent".to_string(), "".to_string()) + .map_err(|err| { + println!("error: {}", err.to_string()); + System::current().stop(); + }) + .map_ok(|_| panic!("wrong query response")) + .map(drop), + ); + } + future::ready(()) + })); }), 100, 20000, @@ -415,32 +400,30 @@ fn test_get_validator_info_rpc() { WaitOrTimeout::new( Box::new(move |_ctx| { let rpc_addrs_copy = rpc_addrs.clone(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - let res = res.unwrap().unwrap(); - if res.header.height > 1 { - let mut client = new_client(&format!("http://{}", rpc_addrs_copy[0])); - let block_hash = res.header.hash; - actix::spawn( - client - .validators(Some(BlockId::Hash(block_hash))) - .map_err(|err| { - panic!(format!("error: {:?}", err)); - }) - .map_ok(move |result| { - assert_eq!(result.current_validators.len(), 1); - assert!(result - .current_validators - .iter() - .any(|r| r.account_id == "near.0".to_string())); - System::current().stop(); - }) - .map(drop), - ); - } - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + let res = res.unwrap().unwrap(); + if res.header.height > 1 { + let mut client = new_client(&format!("http://{}", rpc_addrs_copy[0])); + let block_hash = res.header.hash; + actix::spawn( + client + .validators(Some(BlockId::Hash(block_hash))) + .map_err(|err| { + panic!(format!("error: {:?}", err)); + }) + .map_ok(move |result| { + assert_eq!(result.current_validators.len(), 1); + assert!(result + .current_validators + .iter() + .any(|r| r.account_id == "near.0".to_string())); + System::current().stop(); + }) + .map(drop), + ); + } + future::ready(()) + })); }), 100, 20000, diff --git a/near/tests/run_nodes.rs b/near/tests/run_nodes.rs index b04dcf2086a..0d448b2cbd0 100644 --- a/near/tests/run_nodes.rs +++ b/near/tests/run_nodes.rs @@ -6,7 +6,6 @@ use near_client::GetBlock; use near_network::test_utils::WaitOrTimeout; use near_primitives::test_utils::heavy_test; use near_primitives::types::{BlockHeightDelta, NumSeats, NumShards}; -use near_primitives::views::Finality; use testlib::start_nodes; fn run_nodes( @@ -26,7 +25,7 @@ fn run_nodes( let view_client = clients[clients.len() - 1].1.clone(); WaitOrTimeout::new( Box::new(move |_ctx| { - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then(move |res| { + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { match &res { Ok(Ok(b)) if b.header.height > num_blocks => System::current().stop(), Err(_) => return future::ready(()), diff --git a/near/tests/stake_nodes.rs b/near/tests/stake_nodes.rs index f2e69b5ce62..e1cd47efeea 100644 --- a/near/tests/stake_nodes.rs +++ b/near/tests/stake_nodes.rs @@ -17,8 +17,8 @@ use near_network::NetworkClientMessages; use near_primitives::hash::CryptoHash; use near_primitives::test_utils::{heavy_test, init_integration_logger}; use near_primitives::transaction::SignedTransaction; -use near_primitives::types::{AccountId, BlockHeightDelta, NumSeats}; -use near_primitives::views::{Finality, QueryRequest, QueryResponseKind, ValidatorInfo}; +use near_primitives::types::{AccountId, BlockHeightDelta, BlockIdOrFinality, NumSeats}; +use near_primitives::views::{QueryRequest, QueryResponseKind, ValidatorInfo}; use testlib::genesis_hash; #[derive(Clone)] @@ -215,13 +215,12 @@ fn test_validator_kickout() { test_node1 .view_client .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: test_nodes[i as usize] .account_id .clone(), }, - Finality::None, )) .then(move |res| { match res.unwrap().unwrap().unwrap().kind { @@ -245,13 +244,12 @@ fn test_validator_kickout() { test_node1 .view_client .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: test_nodes[i as usize] .account_id .clone(), }, - Finality::None, )) .then(move |res| { match res.unwrap().unwrap().unwrap().kind { @@ -365,11 +363,10 @@ fn test_validator_join() { test_node1 .view_client .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: test_nodes[1].account_id.clone(), }, - Finality::None, )) .then(move |res| match res.unwrap().unwrap().unwrap().kind { QueryResponseKind::ViewAccount(result) => { @@ -385,11 +382,10 @@ fn test_validator_join() { test_node1 .view_client .send(Query::new( - None, + BlockIdOrFinality::latest(), QueryRequest::ViewAccount { account_id: test_nodes[2].account_id.clone(), }, - Finality::None, )) .then(move |res| match res.unwrap().unwrap().unwrap().kind { QueryResponseKind::ViewAccount(result) => { @@ -445,38 +441,28 @@ fn test_inflation() { WaitOrTimeout::new( Box::new(move |_ctx| { let (done1_copy2, done2_copy2) = (done1_copy1.clone(), done2_copy1.clone()); - actix::spawn( - test_nodes[0].view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - let header_view = res.unwrap().unwrap().header; - if header_view.height >= 2 && header_view.height <= epoch_length { - if header_view.total_supply == initial_total_supply { - done1_copy2.store(true, Ordering::SeqCst); - } - } - future::ready(()) - }, - ), - ); - actix::spawn( - test_nodes[0].view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - let header_view = res.unwrap().unwrap().header; - if header_view.height > epoch_length - && header_view.height < epoch_length * 2 - { - let inflation = initial_total_supply - * max_inflation_rate as u128 - * epoch_length as u128 - / (100 * num_blocks_per_year as u128); - if header_view.total_supply == initial_total_supply + inflation { - done2_copy2.store(true, Ordering::SeqCst); - } - } - future::ready(()) - }, - ), - ); + actix::spawn(test_nodes[0].view_client.send(GetBlock::latest()).then(move |res| { + let header_view = res.unwrap().unwrap().header; + if header_view.height >= 2 && header_view.height <= epoch_length { + if header_view.total_supply == initial_total_supply { + done1_copy2.store(true, Ordering::SeqCst); + } + } + future::ready(()) + })); + actix::spawn(test_nodes[0].view_client.send(GetBlock::latest()).then(move |res| { + let header_view = res.unwrap().unwrap().header; + if header_view.height > epoch_length && header_view.height < epoch_length * 2 { + let inflation = initial_total_supply + * max_inflation_rate as u128 + * epoch_length as u128 + / (100 * num_blocks_per_year as u128); + if header_view.total_supply == initial_total_supply + inflation { + done2_copy2.store(true, Ordering::SeqCst); + } + } + future::ready(()) + })); if done1_copy1.load(Ordering::SeqCst) && done2_copy1.load(Ordering::SeqCst) { System::current().stop(); } diff --git a/near/tests/sync_nodes.rs b/near/tests/sync_nodes.rs index 08923a3a706..1c60d92c318 100644 --- a/near/tests/sync_nodes.rs +++ b/near/tests/sync_nodes.rs @@ -20,7 +20,6 @@ use near_primitives::test_utils::{heavy_test, init_integration_logger}; use near_primitives::transaction::SignedTransaction; use near_primitives::types::{BlockHeightDelta, EpochId, ValidatorStake}; use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner}; -use near_primitives::views::Finality; use testlib::genesis_block; // This assumes that there is no height skipped. Otherwise epoch hash calculation will be wrong. @@ -116,7 +115,7 @@ fn sync_nodes() { WaitOrTimeout::new( Box::new(move |_ctx| { - actix::spawn(view_client2.send(GetBlock::Finality(Finality::None)).then(|res| { + actix::spawn(view_client2.send(GetBlock::latest()).then(|res| { match &res { Ok(Ok(b)) if b.header.height == 13 => System::current().stop(), Err(_) => return future::ready(()), @@ -178,23 +177,20 @@ fn sync_after_sync_nodes() { let client11 = client1.clone(); let signer1 = signer.clone(); let next_step1 = next_step.clone(); - actix::spawn(view_client2.send(GetBlock::Finality(Finality::None)).then( - move |res| { - match &res { - Ok(Ok(b)) if b.header.height == 13 => { - if !next_step1.load(Ordering::Relaxed) { - let _ = - add_blocks(blocks1, client11, 10, epoch_length, &signer1); - next_step1.store(true, Ordering::Relaxed); - } + actix::spawn(view_client2.send(GetBlock::latest()).then(move |res| { + match &res { + Ok(Ok(b)) if b.header.height == 13 => { + if !next_step1.load(Ordering::Relaxed) { + let _ = add_blocks(blocks1, client11, 10, epoch_length, &signer1); + next_step1.store(true, Ordering::Relaxed); } - Ok(Ok(b)) if b.header.height > 20 => System::current().stop(), - Err(_) => return future::ready(()), - _ => {} - }; - future::ready(()) - }, - )); + } + Ok(Ok(b)) if b.header.height > 20 => System::current().stop(), + Err(_) => return future::ready(()), + _ => {} + }; + future::ready(()) + })); }), 100, 60000, @@ -255,36 +251,32 @@ fn sync_state_stake_change() { let started_copy = started.clone(); let near2_copy = near2.clone(); let dir2_path_copy = dir2_path.clone(); - actix::spawn(view_client1.send(GetBlock::Finality(Finality::None)).then( - move |res| { - let latest_height = res.unwrap().unwrap().header.height; - if !started_copy.load(Ordering::SeqCst) && latest_height > 10 { - started_copy.store(true, Ordering::SeqCst); - let (_, view_client2) = start_with_config(&dir2_path_copy, near2_copy); + actix::spawn(view_client1.send(GetBlock::latest()).then(move |res| { + let latest_height = res.unwrap().unwrap().header.height; + if !started_copy.load(Ordering::SeqCst) && latest_height > 10 { + started_copy.store(true, Ordering::SeqCst); + let (_, view_client2) = start_with_config(&dir2_path_copy, near2_copy); - WaitOrTimeout::new( - Box::new(move |_ctx| { - actix::spawn( - view_client2.send(GetBlock::Finality(Finality::None)).then( - move |res| { - if let Ok(block) = res.unwrap() { - if block.header.height > latest_height + 1 { - System::current().stop() - } - } - future::ready(()) - }, - ), - ); - }), - 100, - 30000, - ) - .start(); - } - future::ready(()) - }, - )); + WaitOrTimeout::new( + Box::new(move |_ctx| { + actix::spawn(view_client2.send(GetBlock::latest()).then( + move |res| { + if let Ok(block) = res.unwrap() { + if block.header.height > latest_height + 1 { + System::current().stop() + } + } + future::ready(()) + }, + )); + }), + 100, + 30000, + ) + .start(); + } + future::ready(()) + })); }), 100, 35000, diff --git a/near/tests/sync_state_nodes.rs b/near/tests/sync_state_nodes.rs index 9bc0e44d994..2116a9f7c4c 100644 --- a/near/tests/sync_state_nodes.rs +++ b/near/tests/sync_state_nodes.rs @@ -10,7 +10,6 @@ use near_chain_configs::Genesis; use near_client::GetBlock; use near_network::test_utils::{convert_boot_nodes, open_port, WaitOrTimeout}; use near_primitives::test_utils::{heavy_test, init_integration_logger}; -use near_primitives::views::Finality; /// One client is in front, another must sync to it using state (fast) sync. #[test] @@ -37,52 +36,47 @@ fn sync_state_nodes() { let view_client2_holder2 = view_client2_holder.clone(); let genesis2 = Arc::clone(&genesis); - actix::spawn(view_client1.send(GetBlock::Finality(Finality::None)).then( - move |res| { - match &res { - Ok(Ok(b)) if b.header.height >= 101 => { - let mut view_client2_holder2 = - view_client2_holder2.write().unwrap(); - - if view_client2_holder2.is_none() { - let mut near2 = - load_test_config("test2", port2, Arc::clone(&genesis2)); - near2.client_config.skip_sync_wait = false; - near2.client_config.min_num_peers = 1; - near2.network_config.boot_nodes = - convert_boot_nodes(vec![("test1", port1)]); - - let dir2 = TempDir::new("sync_nodes_2").unwrap(); - let (_, view_client2) = - start_with_config(dir2.path(), near2); - *view_client2_holder2 = Some(view_client2); - } + actix::spawn(view_client1.send(GetBlock::latest()).then(move |res| { + match &res { + Ok(Ok(b)) if b.header.height >= 101 => { + let mut view_client2_holder2 = + view_client2_holder2.write().unwrap(); + + if view_client2_holder2.is_none() { + let mut near2 = + load_test_config("test2", port2, Arc::clone(&genesis2)); + near2.client_config.skip_sync_wait = false; + near2.client_config.min_num_peers = 1; + near2.network_config.boot_nodes = + convert_boot_nodes(vec![("test1", port1)]); + + let dir2 = TempDir::new("sync_nodes_2").unwrap(); + let (_, view_client2) = start_with_config(dir2.path(), near2); + *view_client2_holder2 = Some(view_client2); } - Ok(Ok(b)) if b.header.height < 101 => { - println!("FIRST STAGE {}", b.header.height) - } - Err(_) => return future::ready(()), - _ => {} - }; - future::ready(()) - }, - )); + } + Ok(Ok(b)) if b.header.height < 101 => { + println!("FIRST STAGE {}", b.header.height) + } + Err(_) => return future::ready(()), + _ => {} + }; + future::ready(()) + })); } if let Some(view_client2) = &*view_client2_holder.write().unwrap() { - actix::spawn(view_client2.send(GetBlock::Finality(Finality::None)).then( - |res| { - match &res { - Ok(Ok(b)) if b.header.height >= 101 => System::current().stop(), - Ok(Ok(b)) if b.header.height < 101 => { - println!("SECOND STAGE {}", b.header.height) - } - Err(_) => return future::ready(()), - _ => {} - }; - future::ready(()) - }, - )); + actix::spawn(view_client2.send(GetBlock::latest()).then(|res| { + match &res { + Ok(Ok(b)) if b.header.height >= 101 => System::current().stop(), + Ok(Ok(b)) if b.header.height < 101 => { + println!("SECOND STAGE {}", b.header.height) + } + Err(_) => return future::ready(()), + _ => {} + }; + future::ready(()) + })); } else { } }), @@ -155,68 +149,63 @@ fn sync_state_nodes_multishard() { let view_client2_holder2 = view_client2_holder.clone(); let genesis2 = Arc::clone(&genesis); - actix::spawn(view_client1.send(GetBlock::Finality(Finality::None)).then( - move |res| { - match &res { - Ok(Ok(b)) if b.header.height >= 101 => { - let mut view_client2_holder2 = - view_client2_holder2.write().unwrap(); - - if view_client2_holder2.is_none() { - let mut near2 = - load_test_config("test2", port2, Arc::clone(&genesis2)); - near2.client_config.skip_sync_wait = false; - near2.client_config.min_num_peers = 3; - near2.client_config.min_block_production_delay = - Duration::from_millis(200); - near2.client_config.max_block_production_delay = - Duration::from_millis(400); - near2.network_config.boot_nodes = convert_boot_nodes(vec![ - ("test1", port1), - ("test3", port3), - ("test4", port4), - ]); - - let dir2 = TempDir::new("sync_nodes_2").unwrap(); - let (_, view_client2) = - start_with_config(dir2.path(), near2); - *view_client2_holder2 = Some(view_client2); - } - } - Ok(Ok(b)) if b.header.height < 101 => { - println!("FIRST STAGE {}", b.header.height) + actix::spawn(view_client1.send(GetBlock::latest()).then(move |res| { + match &res { + Ok(Ok(b)) if b.header.height >= 101 => { + let mut view_client2_holder2 = + view_client2_holder2.write().unwrap(); + + if view_client2_holder2.is_none() { + let mut near2 = + load_test_config("test2", port2, Arc::clone(&genesis2)); + near2.client_config.skip_sync_wait = false; + near2.client_config.min_num_peers = 3; + near2.client_config.min_block_production_delay = + Duration::from_millis(200); + near2.client_config.max_block_production_delay = + Duration::from_millis(400); + near2.network_config.boot_nodes = convert_boot_nodes(vec![ + ("test1", port1), + ("test3", port3), + ("test4", port4), + ]); + + let dir2 = TempDir::new("sync_nodes_2").unwrap(); + let (_, view_client2) = start_with_config(dir2.path(), near2); + *view_client2_holder2 = Some(view_client2); } - Err(_) => return future::ready(()), - _ => {} - }; - future::ready(()) - }, - )); + } + Ok(Ok(b)) if b.header.height < 101 => { + println!("FIRST STAGE {}", b.header.height) + } + Err(_) => return future::ready(()), + _ => {} + }; + future::ready(()) + })); } if let Some(view_client2) = &*view_client2_holder.write().unwrap() { - actix::spawn(view_client2.send(GetBlock::Finality(Finality::None)).then( - |res| { - match &res { - Ok(Ok(b)) if b.header.height >= 101 => System::current().stop(), - Ok(Ok(b)) if b.header.height < 101 => { - println!("SECOND STAGE {}", b.header.height) - } - Ok(Err(e)) => { - println!("SECOND STAGE ERROR1: {:?}", e); - return future::ready(()); - } - Err(e) => { - println!("SECOND STAGE ERROR2: {:?}", e); - return future::ready(()); - } - _ => { - assert!(false); - } - }; - future::ready(()) - }, - )); + actix::spawn(view_client2.send(GetBlock::latest()).then(|res| { + match &res { + Ok(Ok(b)) if b.header.height >= 101 => System::current().stop(), + Ok(Ok(b)) if b.header.height < 101 => { + println!("SECOND STAGE {}", b.header.height) + } + Ok(Err(e)) => { + println!("SECOND STAGE ERROR1: {:?}", e); + return future::ready(()); + } + Err(e) => { + println!("SECOND STAGE ERROR2: {:?}", e); + return future::ready(()); + } + _ => { + assert!(false); + } + }; + future::ready(()) + })); } }), 100, @@ -264,70 +253,63 @@ fn sync_empty_state() { let genesis2 = Arc::clone(&genesis); let dir2 = dir2.clone(); - actix::spawn(view_client1.send(GetBlock::Finality(Finality::None)).then( - move |res| { - match &res { - Ok(Ok(b)) if b.header.height >= state_sync_horizon + 1 => { - let mut view_client2_holder2 = - view_client2_holder2.write().unwrap(); - - if view_client2_holder2.is_none() { - let mut near2 = - load_test_config("test2", port2, Arc::clone(&genesis2)); - near2.network_config.boot_nodes = - convert_boot_nodes(vec![("test1", port1)]); - near2.client_config.min_num_peers = 1; - near2.client_config.min_block_production_delay = - Duration::from_millis(200); - near2.client_config.max_block_production_delay = - Duration::from_millis(400); - near2.client_config.state_fetch_horizon = - state_sync_horizon; - near2.client_config.block_header_fetch_horizon = - block_header_fetch_horizon; - near2.client_config.block_fetch_horizon = - block_fetch_horizon; - near2.client_config.tracked_shards = vec![0, 1, 2, 3]; - - let (_, view_client2) = - start_with_config(dir2.path(), near2); - *view_client2_holder2 = Some(view_client2); - } - } - Ok(Ok(b)) if b.header.height <= state_sync_horizon => { - println!("FIRST STAGE {}", b.header.height) + actix::spawn(view_client1.send(GetBlock::latest()).then(move |res| { + match &res { + Ok(Ok(b)) if b.header.height >= state_sync_horizon + 1 => { + let mut view_client2_holder2 = + view_client2_holder2.write().unwrap(); + + if view_client2_holder2.is_none() { + let mut near2 = + load_test_config("test2", port2, Arc::clone(&genesis2)); + near2.network_config.boot_nodes = + convert_boot_nodes(vec![("test1", port1)]); + near2.client_config.min_num_peers = 1; + near2.client_config.min_block_production_delay = + Duration::from_millis(200); + near2.client_config.max_block_production_delay = + Duration::from_millis(400); + near2.client_config.state_fetch_horizon = state_sync_horizon; + near2.client_config.block_header_fetch_horizon = + block_header_fetch_horizon; + near2.client_config.block_fetch_horizon = block_fetch_horizon; + near2.client_config.tracked_shards = vec![0, 1, 2, 3]; + + let (_, view_client2) = start_with_config(dir2.path(), near2); + *view_client2_holder2 = Some(view_client2); } - Err(_) => return future::ready(()), - _ => {} - }; - future::ready(()) - }, - )); + } + Ok(Ok(b)) if b.header.height <= state_sync_horizon => { + println!("FIRST STAGE {}", b.header.height) + } + Err(_) => return future::ready(()), + _ => {} + }; + future::ready(()) + })); } if let Some(view_client2) = &*view_client2_holder.write().unwrap() { - actix::spawn(view_client2.send(GetBlock::Finality(Finality::None)).then( - |res| { - match &res { - Ok(Ok(b)) if b.header.height >= 40 => System::current().stop(), - Ok(Ok(b)) if b.header.height < 40 => { - println!("SECOND STAGE {}", b.header.height) - } - Ok(Err(e)) => { - println!("SECOND STAGE ERROR1: {:?}", e); - return future::ready(()); - } - Err(e) => { - println!("SECOND STAGE ERROR2: {:?}", e); - return future::ready(()); - } - _ => { - assert!(false); - } - }; - future::ready(()) - }, - )); + actix::spawn(view_client2.send(GetBlock::latest()).then(|res| { + match &res { + Ok(Ok(b)) if b.header.height >= 40 => System::current().stop(), + Ok(Ok(b)) if b.header.height < 40 => { + println!("SECOND STAGE {}", b.header.height) + } + Ok(Err(e)) => { + println!("SECOND STAGE ERROR1: {:?}", e); + return future::ready(()); + } + Err(e) => { + println!("SECOND STAGE ERROR2: {:?}", e); + return future::ready(()); + } + _ => { + assert!(false); + } + }; + future::ready(()) + })); } }), 100, diff --git a/near/tests/track_shards.rs b/near/tests/track_shards.rs index 9a4b72d59e8..9eb0607cb5a 100644 --- a/near/tests/track_shards.rs +++ b/near/tests/track_shards.rs @@ -8,7 +8,6 @@ use near_client::{GetBlock, GetChunk}; use near_network::test_utils::WaitOrTimeout; use near_primitives::hash::CryptoHash; use near_primitives::test_utils::{heavy_test, init_integration_logger}; -use near_primitives::views::Finality; use testlib::start_nodes; #[test] @@ -40,19 +39,17 @@ fn track_shards() { )); } else { let last_block_hash1 = last_block_hash.clone(); - actix::spawn(view_client.send(GetBlock::Finality(Finality::None)).then( - move |res| { - match &res { - Ok(Ok(b)) if b.header.height > 10 => { - *last_block_hash1.write().unwrap() = - Some(b.header.hash.clone().into()); - } - Err(_) => return future::ready(()), - _ => {} - }; - future::ready(()) - }, - )); + actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { + match &res { + Ok(Ok(b)) if b.header.height > 10 => { + *last_block_hash1.write().unwrap() = + Some(b.header.hash.clone().into()); + } + Err(_) => return future::ready(()), + _ => {} + }; + future::ready(()) + })); } }), 100, diff --git a/pytest/lib/cluster.py b/pytest/lib/cluster.py index 151e1815d70..97468d6f110 100644 --- a/pytest/lib/cluster.py +++ b/pytest/lib/cluster.py @@ -125,28 +125,34 @@ def get_validators(self): def get_account(self, acc, finality='optimistic'): return self.json_rpc('query', {"request_type": "view_account", "account_id": acc, "block_id": None, "finality": finality}) - def get_block(self, block_hash): - return self.json_rpc('block', [block_hash]) + def get_block(self, block_id): + return self.json_rpc('block', [block_id]) + + def get_chunk(self, chunk_id): + return self.json_rpc('chunk', [chunk_id]) + + def get_tx(self, tx_hash, tx_recipient_id): + return self.json_rpc('tx', [tx_hash, tx_recipient_id]) + + def get_changes(self, changes_request): + return self.json_rpc('EXPERIMENTAL_changes', changes_request) - def get_changes(self, block_hash, state_changes_request): - return self.json_rpc('changes', [block_hash, state_changes_request]) - def validators(self): return set(map(lambda v: v['account_id'], self.get_status()['validators'])) + class RpcNode(BaseNode): """ A running node only interact by rpc queries """ def __init__(self, host, rpc_port): super(RpcNode, self).__init__() self.host = host self.rpc_port = rpc_port - + def rpc_addr(self): return (self.host, self.rpc_port) - class LocalNode(BaseNode): def __init__(self, port, rpc_port, near_root, node_dir, blacklist): super(LocalNode, self).__init__() diff --git a/pytest/lib/transaction.py b/pytest/lib/transaction.py index 59cfbdc020d..3ca7c4e5b76 100644 --- a/pytest/lib/transaction.py +++ b/pytest/lib/transaction.py @@ -151,6 +151,42 @@ def sign_and_serialize_transaction(receiverId, nonce, actions, blockHash, accoun return BinarySerializer(tx_schema).serialize(signedTx) +def create_create_account_action(): + createAccount = CreateAccount() + action = Action() + action.enum = 'createAccount' + action.createAccount = createAccount + return action + +def create_full_access_key_action(pk): + permission = AccessKeyPermission() + permission.enum = 'fullAccess' + permission.fullAccess = FullAccessPermission() + accessKey = AccessKey() + accessKey.nonce = 0 + accessKey.permission = permission + publicKey = PublicKey() + publicKey.keyType = 0 + publicKey.data = pk + addKey = AddKey() + addKey.accessKey = accessKey + addKey.publicKey = publicKey + action = Action() + action.enum = 'addKey' + action.addKey = addKey + return action + +def create_delete_access_key_action(pk): + publicKey = PublicKey() + publicKey.keyType = 0 + publicKey.data = pk + deleteKey = DeleteKey() + deleteKey.publicKey = publicKey + action = Action() + action.enum = 'deleteKey' + action.deleteKey = deleteKey + return action + def create_payment_action(amount): transfer = Transfer() transfer.deposit = amount @@ -189,6 +225,21 @@ def create_function_call_action(methodName, args, gas, deposit): action.functionCall = functionCall return action +def sign_create_account_tx(creator_key, new_account_id, nonce, block_hash): + action = create_create_account_action() + return sign_and_serialize_transaction(new_account_id, nonce, [action], block_hash, creator_key.account_id, creator_key.decoded_pk(), creator_key.decoded_sk()) + +def sign_create_account_with_full_access_key_and_balance_tx(creator_key, new_account_id, new_key, balance, nonce, block_hash): + create_account_action = create_create_account_action() + full_access_key_action = create_full_access_key_action(new_key.decoded_pk()) + payment_action = create_payment_action(balance) + actions = [create_account_action, full_access_key_action, payment_action] + return sign_and_serialize_transaction(new_account_id, nonce, actions, block_hash, creator_key.account_id, creator_key.decoded_pk(), creator_key.decoded_sk()) + +def sign_delete_access_key_tx(signer_key, target_account_id, key_for_deletion, nonce, block_hash): + action = create_delete_access_key_action(key_for_deletion.decoded_pk()) + return sign_and_serialize_transaction(target_account_id, nonce, [action], block_hash, signer_key.account_id, signer_key.decoded_pk(), signer_key.decoded_sk()) + def sign_payment_tx(key, to, amount, nonce, blockHash): action = create_payment_action(amount) return sign_and_serialize_transaction(to, nonce, [action], blockHash, key.account_id, key.decoded_pk(), key.decoded_sk()) @@ -203,4 +254,4 @@ def sign_deploy_contract_tx(signer_key, code, nonce, blockHash): def sign_function_call_tx(signer_key, methodName, args, gas, deposit, nonce, blockHash): action = create_function_call_action(methodName, args, gas, deposit) - return sign_and_serialize_transaction(signer_key.account_id, nonce, [action], blockHash, signer_key.account_id, signer_key.decoded_pk(), signer_key.decoded_sk()) \ No newline at end of file + return sign_and_serialize_transaction(signer_key.account_id, nonce, [action], blockHash, signer_key.account_id, signer_key.decoded_pk(), signer_key.decoded_sk()) diff --git a/pytest/tests/sanity/rpc_key_value_changes.py b/pytest/tests/sanity/rpc_key_value_changes.py index b4cc45b8334..4a5fd60decf 100644 --- a/pytest/tests/sanity/rpc_key_value_changes.py +++ b/pytest/tests/sanity/rpc_key_value_changes.py @@ -1,100 +1,298 @@ -# Spins up four nodes, deploy an smart contract to one node, -# and call a set key method on it. +# Spins up four nodes, deploy a smart contract to one node, +# and call various scenarios to trigger store changes. # Check that the key changes are observable via `changes` RPC call. -import sys, time -import base58 +import sys +import base58, base64 import json +import threading import deepdiff sys.path.append('lib') -from cluster import start_cluster -from transaction import sign_deploy_contract_tx, sign_function_call_tx +from cluster import start_cluster, Key from utils import load_binary_file +import transaction nodes = start_cluster(4, 0, 1, None, [["epoch_length", 10], ["block_producer_kickout_threshold", 80]], {}) +# Plan: +# 1. Create a new account. +# 2. Observe the changes in the block where the receipt lands. + +status = nodes[0].get_status() +latest_block_hash = status['sync_info']['latest_block_hash'] +new_account_id = 'rpc_key_value_changes' +create_account_tx = transaction.sign_create_account_tx( + nodes[0].signer_key, + new_account_id, + 5, + base58.b58decode(latest_block_hash.encode('utf8')) +) +new_account_response = nodes[0].send_tx_and_wait(create_account_tx, 10) + +state_changes_request = { + "block_id": new_account_response['result']['receipts_outcome'][0]['block_hash'], + "changes_type": "account_changes", + "account_id": new_account_id, +} +expected_changes_response = { + "block_hash": state_changes_request["block_id"], + "changes": [ + { + "cause": { + "type": "receipt_processing", + "receipt_hash": new_account_response['result']['receipts_outcome'][0]['id'] + }, + "type": "account_update", + "change": { + "account_id": new_account_id, + "amount": "0", + "locked": "0", + "code_hash": "11111111111111111111111111111111", + "storage_usage": 100, + } + } + ] +} + +for node in nodes: + changes_response = nodes[0].get_changes(state_changes_request)['result'] + del changes_response['changes'][0]['change']['storage_paid_at'] + assert not deepdiff.DeepDiff(changes_response, expected_changes_response), \ + "query same changes gives different results (expected VS actual):\n%r\n%r" \ + % (expected_changes_response, changes_response) + +# Plan: +# 1. Create a new account with an access key. +# 2. Observe the changes in the block where the receipt lands. +# 3. Remove the access key. +# 4. Observe the changes in the block where the receipt lands. + +# re-use the key as a new account access key +new_key = Key( + account_id='rpc_key_value_changes_full_access', + pk=nodes[1].signer_key.pk, + sk=nodes[1].signer_key.sk, +) + +status = nodes[0].get_status() +latest_block_hash = status['sync_info']['latest_block_hash'] +create_account_tx = transaction.sign_create_account_with_full_access_key_and_balance_tx( + creator_key=nodes[0].signer_key, + new_account_id=new_key.account_id, + new_key=new_key, + balance=10**17, + nonce=7, + block_hash=base58.b58decode(latest_block_hash.encode('utf8')) +) +new_account_response = nodes[0].send_tx_and_wait(create_account_tx, 10) + +state_changes_request = { + "block_id": new_account_response['result']['receipts_outcome'][0]['block_hash'], + "changes_type": "all_access_key_changes", + "account_id": new_key.account_id, +} +expected_changes_response = { + "block_hash": state_changes_request["block_id"], + "changes": [ + { + "cause": { + "type": "receipt_processing", + "receipt_hash": new_account_response["result"]["receipts_outcome"][0]["id"], + }, + "type": "access_key_update", + "change": { + "account_id": new_key.account_id, + "public_key": new_key.pk, + "access_key": {"nonce": 0, "permission": "FullAccess"}, + } + } + ] +} + +for node in nodes: + changes_response = nodes[0].get_changes(state_changes_request)['result'] + assert not deepdiff.DeepDiff(changes_response, expected_changes_response), \ + "query same changes gives different results (expected VS actual):\n%r\n%r" \ + % (expected_changes_response, changes_response) + +status = nodes[0].get_status() +latest_block_hash = status['sync_info']['latest_block_hash'] +delete_access_key_tx = transaction.sign_delete_access_key_tx( + signer_key=new_key, + target_account_id=new_key.account_id, + key_for_deletion=new_key, + nonce=8, + block_hash=base58.b58decode(latest_block_hash.encode('utf8')) +) +delete_access_key_response = nodes[1].send_tx_and_wait(delete_access_key_tx, 10) + +state_changes_request = { + "block_id": delete_access_key_response['result']['receipts_outcome'][0]['block_hash'], + "changes_type": "all_access_key_changes", + "account_id": new_key.account_id, +} +expected_changes_response = { + "block_hash": state_changes_request["block_id"], + "changes": [ + { + "cause": { + 'type': 'transaction_processing', + 'tx_hash': delete_access_key_response['result']['transaction']['hash'], + }, + "type": "access_key_update", + "change": { + "account_id": new_key.account_id, + "public_key": new_key.pk, + "access_key": {"nonce": 8, "permission": "FullAccess"}, + } + }, + { + "cause": { + "type": "receipt_processing", + "receipt_hash": delete_access_key_response["result"]["receipts_outcome"][0]["id"] + }, + "type": "access_key_deletion", + "change": { + "account_id": new_key.account_id, + "public_key": new_key.pk, + } + } + ] +} + +for node in nodes: + changes_response = nodes[0].get_changes(state_changes_request)['result'] + assert not deepdiff.DeepDiff(changes_response, expected_changes_response), \ + "query same changes gives different results (expected VS actual):\n%r\n%r" \ + % (expected_changes_response, changes_response) + + +# Plan: +# 1. Deploy a contract. +# 2. Observe the code changes in the block where the transaction outcome "lands". +# 3. Send two transactions to be included into the same block setting and overriding the value of +# the same key (`my_key`). +# 4. Observe the changes in the block where the transaction outcome "lands". + +hello_smart_contract = load_binary_file('../tests/hello.wasm') status = nodes[0].get_status() -hash_ = status['sync_info']['latest_block_hash'] -hash_ = base58.b58decode(hash_.encode('utf8')) -tx = sign_deploy_contract_tx(nodes[0].signer_key, load_binary_file('../tests/hello.wasm'), 10, hash_) -nodes[0].send_tx(tx) +latest_block_hash = status['sync_info']['latest_block_hash'] +deploy_contract_tx = transaction.sign_deploy_contract_tx( + nodes[0].signer_key, + hello_smart_contract, + 10, + base58.b58decode(latest_block_hash.encode('utf8')) +) +deploy_contract_response = nodes[0].send_tx_and_wait(deploy_contract_tx, 10) + +state_changes_request = { + "block_id": deploy_contract_response['result']['transaction_outcome']['block_hash'], + "changes_type": "code_changes", + "account_id": nodes[0].signer_key.account_id, +} + +expected_changes_response = { + "block_hash": state_changes_request["block_id"], + "changes": [ + { + "cause": { + "type": "receipt_processing", + "receipt_hash": deploy_contract_response["result"]["receipts_outcome"][0]["id"], + }, + "type": "code_update", + "change": { + "account_id": nodes[0].signer_key.account_id, + "code_base64": base64.b64encode(hello_smart_contract).decode('utf-8'), + } + }, + ] +} + +for node in nodes: + changes_response = node.get_changes(state_changes_request)['result'] + assert not deepdiff.DeepDiff(changes_response, expected_changes_response), \ + "query same changes gives different results (expected VS actual):\n%r\n%r" \ + % (expected_changes_response, changes_response) + +status = nodes[1].get_status() +latest_block_hash = status['sync_info']['latest_block_hash'] -time.sleep(3) +def set_value_1(): + function_call_1_tx = transaction.sign_function_call_tx( + nodes[0].signer_key, + 'setKeyValue', + json.dumps({"key": "my_key", "value": "my_value_1"}).encode('utf-8'), + 100000000000, + 100000000000, + 20, + base58.b58decode(latest_block_hash.encode('utf8')) + ) + res = nodes[1].send_tx_and_wait(function_call_1_tx, 10) +function_call_1_thread = threading.Thread(target=set_value_1) +function_call_1_thread.start() -status2 = nodes[1].get_status() -hash_2 = status2['sync_info']['latest_block_hash'] -hash_2 = base58.b58decode(hash_2.encode('utf8')) -tx2 = sign_function_call_tx( +function_call_2_tx = transaction.sign_function_call_tx( nodes[0].signer_key, 'setKeyValue', - json.dumps({"key": "mykey", "value": "myvalue"}).encode('utf-8'), + json.dumps({"key": "my_key", "value": "my_value_2"}).encode('utf-8'), 100000000000, 100000000000, - 20, - hash_2 + 30, + base58.b58decode(latest_block_hash.encode('utf8')) ) -res = nodes[1].send_tx_and_wait(tx2, 10) -assert res['result']['receipts_outcome'][0]['outcome']['status'] == {'SuccessValue': ''}, "Expected successful execution, but the output was: %s" % res - -# send method=changes params:=[block_hash, {"changes_type": "data_changes", "account_id": account_id, "key_prefix": key_prefix_as_array_of_bytes}] -# e.g. method=changes params:=["8jT2x22z757m378fsKQe2JHin1k56k1jM1sQoqeqyABx", {"changes_type": "data_changes", "account_id": "test0", "key_prefix": [109, 121, 107, 101, 121]}] -# expect: -# { -# "id": "dontcare", -# "jsonrpc": "2.0", -# "result": { -# "block_hash": "8jT2x22z757m378fsKQe2JHin1k56k1jM1sQoqeqyABx", -# "changes_by_key": [ -# { -# "key": [ 109, 121, 107, 101, 121 ], # it is b"mykey" -# "changes": [ -# { -# "cause": { -# "ReceiptProcessing": { -# "hash": "CWdLzVZGAHNNJi9RstMEidZhPhKJbiszSqSy5M3yph6J" -# } -# }, -# "value": [ 109, 121, 118, 97, 108, 117, 101 ] # it is b"myvalue" -# } -# ] -# } -# ] -# } -#} - -tx_block_hash = res['result']['transaction_outcome']['block_hash'] +function_call_2_response = nodes[1].send_tx_and_wait(function_call_2_tx, 10) +assert function_call_2_response['result']['receipts_outcome'][0]['outcome']['status'] == {'SuccessValue': ''}, \ + "Expected successful execution, but the output was: %s" % function_call_2_response +function_call_1_thread.join() + +tx_block_hash = function_call_2_response['result']['transaction_outcome']['block_hash'] tx_account_id = nodes[0].signer_key.account_id -changes = nodes[0].get_changes( - tx_block_hash, - { - "changes_type": "data_changes", - "account_id": tx_account_id, - "key_prefix": list(b"mykey") - } -) -changes_by_key = changes['result']['changes_by_key'] -assert len(changes_by_key) == 1 -changes_by_mykey = changes_by_key[0] -assert set(changes_by_mykey) >= {'key', 'changes'} -assert changes_by_mykey['key'] == [109, 121, 107, 101, 121], \ - "changed key is expected to be 'mykey' as an array of bytes, but %r found" \ - % changes_by_mykey['key'] -value_changes = changes_by_mykey['changes'] -assert len(value_changes) == 1 -assert set(value_changes[0]['cause']) == {'ReceiptProcessing'} -assert value_changes[0]['value'] == [109, 121, 118, 97, 108, 117, 101], \ - "changed value is expected to be 'myvalue' as an array of bytes, but %r found" % value_changes -for node in nodes[1:]: - changes_from_another_node = node.get_changes( - tx_block_hash, +state_changes_request = { + "block_id": tx_block_hash, + "changes_type": "data_changes", + "account_id": tx_account_id, + "key_prefix_base64": base64.b64encode(b"my_key").decode('utf-8'), +} + +expected_changes_response = { + "block_hash": state_changes_request["block_id"], + "changes": [ + { + "cause": { + "type": "receipt_processing", + }, + "type": "data_update", + "change": { + "account_id": tx_account_id, + "key_base64": base64.b64encode(b"my_key").decode('utf-8'), + "value_base64": base64.b64encode(b"my_value_1").decode('utf-8'), + } + }, { - "changes_type": "data_changes", - "account_id": tx_account_id, - "key_prefix": list(b"mykey") + "cause": { + "type": "receipt_processing", + "receipt_hash": function_call_2_response["result"]["receipts_outcome"][0]["id"] + }, + "type": "data_update", + "change": { + "account_id": tx_account_id, + "key_base64": base64.b64encode(b"my_key").decode('utf-8'), + "value_base64": base64.b64encode(b"my_value_2").decode('utf-8'), + } } - ) - assert not deepdiff.DeepDiff(changes_from_another_node, changes), "query same changes gives different result" + ] +} + + +for node in nodes: + changes_response = node.get_changes(state_changes_request)['result'] + # We fetch the transaction in a separate thread, so receiving the right receipt hash is a bit + # involved process, nevermind. + del changes_response['changes'][0]['cause']['receipt_hash'] + assert not deepdiff.DeepDiff(changes_response, expected_changes_response), \ + "query same changes gives different results (expected VS actual):\n%r\n%r" \ + % (expected_changes_response, changes_response) diff --git a/runtime/runtime/Cargo.toml b/runtime/runtime/Cargo.toml index d9875a6ed2a..c581c2ed5b8 100644 --- a/runtime/runtime/Cargo.toml +++ b/runtime/runtime/Cargo.toml @@ -5,7 +5,6 @@ authors = ["Near Inc "] edition = "2018" [dependencies] -bincode = "1.0.0" byteorder = "1.2" serde = "1.0" serde_derive = "1.0" diff --git a/runtime/runtime/src/actions.rs b/runtime/runtime/src/actions.rs index 69a97e11443..658f627c618 100644 --- a/runtime/runtime/src/actions.rs +++ b/runtime/runtime/src/actions.rs @@ -13,7 +13,7 @@ use near_primitives::transaction::{ }; use near_primitives::types::{AccountId, Balance, BlockHeight, BlockHeightDelta, ValidatorStake}; use near_primitives::utils::{ - is_valid_sub_account_id, is_valid_top_level_account_id, key_for_access_key, + is_valid_sub_account_id, is_valid_top_level_account_id, KeyForAccessKey, }; use near_runtime_fees::RuntimeFeesConfig; use near_store::{ @@ -388,7 +388,7 @@ pub(crate) fn action_delete_key( return Ok(()); } // Remove access key - state_update.remove(&key_for_access_key(account_id, &delete_key.public_key)); + state_update.remove(KeyForAccessKey::new(account_id, &delete_key.public_key)); let storage_usage_config = &fee_config.storage_usage_config; account.storage_usage = account .storage_usage diff --git a/runtime/runtime/src/balance_checker.rs b/runtime/runtime/src/balance_checker.rs index 66bf5ac7ea5..e6a65d7010f 100644 --- a/runtime/runtime/src/balance_checker.rs +++ b/runtime/runtime/src/balance_checker.rs @@ -12,9 +12,7 @@ use near_primitives::receipt::{Receipt, ReceiptEnum}; use near_primitives::transaction::SignedTransaction; use near_primitives::types::{AccountId, Balance}; use near_primitives::utils::col::DELAYED_RECEIPT_INDICES; -use near_primitives::utils::{ - key_for_delayed_receipt, key_for_postponed_receipt_id, system_account, -}; +use near_primitives::utils::{system_account, KeyForDelayedReceipt, KeyForPostponedReceiptId}; use near_runtime_fees::RuntimeFeesConfig; use near_store::{get, get_account, get_receipt, TrieUpdate}; use std::collections::HashSet; @@ -39,7 +37,7 @@ pub(crate) fn check_balance( let get_delayed_receipts = |from_index, to_index, state| { (from_index..to_index) .map(|index| { - get(state, &key_for_delayed_receipt(index))?.ok_or_else(|| { + get(state, &KeyForDelayedReceipt::new(index))?.ok_or_else(|| { StorageError::StorageInconsistentState(format!( "Delayed receipt #{} should be in the state", index @@ -147,7 +145,7 @@ pub(crate) fn check_balance( ReceiptEnum::Data(data_receipt) => { if let Some(receipt_id) = get( initial_state, - &key_for_postponed_receipt_id(account_id, &data_receipt.data_id), + &KeyForPostponedReceiptId::new(account_id, &data_receipt.data_id), )? { Ok(Some((account_id.clone(), receipt_id))) } else { diff --git a/runtime/runtime/src/ext.rs b/runtime/runtime/src/ext.rs index bd53926da98..b95fbf19d8a 100644 --- a/runtime/runtime/src/ext.rs +++ b/runtime/runtime/src/ext.rs @@ -12,7 +12,7 @@ use near_primitives::transaction::{ DeployContractAction, FunctionCallAction, StakeAction, TransferAction, }; use near_primitives::types::{AccountId, Balance}; -use near_primitives::utils::{create_nonce_with_nonce, prefix_for_data}; +use near_primitives::utils::{create_nonce_with_nonce, KeyForData}; use near_store::{TrieUpdate, TrieUpdateIterator, TrieUpdateValuePtr}; use near_vm_logic::{External, HostError, VMLogicError, ValuePtr}; use sha3::{Keccak256, Keccak512}; @@ -53,7 +53,7 @@ impl<'a> RuntimeExt<'a> { ) -> Self { RuntimeExt { trie_update, - storage_prefix: prefix_for_data(account_id), + storage_prefix: KeyForData::get_prefix(account_id).into(), action_receipts: vec![], iters: HashMap::new(), last_iter_id: 0, @@ -125,7 +125,7 @@ impl<'a> External for RuntimeExt<'a> { fn storage_remove(&mut self, key: &[u8]) -> ExtResult<()> { let storage_key = self.create_storage_key(key); - self.trie_update.remove(&storage_key); + self.trie_update.remove(storage_key); Ok(()) } diff --git a/runtime/runtime/src/lib.rs b/runtime/runtime/src/lib.rs index 6fae07a6693..6a08aa33fd8 100644 --- a/runtime/runtime/src/lib.rs +++ b/runtime/runtime/src/lib.rs @@ -18,14 +18,13 @@ use near_primitives::transaction::{ Action, ExecutionOutcome, ExecutionOutcomeWithId, ExecutionStatus, LogEntry, SignedTransaction, }; use near_primitives::types::{ - AccountId, Balance, BlockHeight, BlockHeightDelta, Gas, Nonce, StateChangeCause, StateChanges, - StateRoot, ValidatorStake, + AccountId, Balance, BlockHeight, BlockHeightDelta, Gas, Nonce, RawStateChanges, + StateChangeCause, StateRoot, ValidatorStake, }; use near_primitives::utils::col::DELAYED_RECEIPT_INDICES; use near_primitives::utils::{ - create_nonce_with_nonce, key_for_delayed_receipt, key_for_pending_data_count, - key_for_postponed_receipt, key_for_postponed_receipt_id, key_for_received_data, system_account, - ACCOUNT_DATA_SEPARATOR, + create_nonce_with_nonce, system_account, KeyForDelayedReceipt, KeyForPendingDataCount, + KeyForPostponedReceipt, KeyForPostponedReceiptId, KeyForReceivedData, ACCOUNT_DATA_SEPARATOR, }; use near_store::{ get, get_account, get_receipt, get_received_data, set, set_access_key, set_account, set_code, @@ -109,7 +108,7 @@ pub struct ApplyResult { pub validator_proposals: Vec, pub outgoing_receipts: Vec, pub outcomes: Vec, - pub key_value_changes: StateChanges, + pub key_value_changes: RawStateChanges, pub stats: ApplyStats, } @@ -419,7 +418,7 @@ impl Runtime { "received data should be in the state".to_string(), ) })?; - state_update.remove(&key_for_received_data(account_id, data_id)); + state_update.remove(KeyForReceivedData::new(account_id, data_id)); match data { Some(value) => Ok(PromiseResult::Successful(value)), None => Ok(PromiseResult::Failed), @@ -700,26 +699,26 @@ impl Runtime { // If we don't have a postponed receipt yet, we don't need to do anything for now. if let Some(receipt_id) = get( state_update, - &key_for_postponed_receipt_id(account_id, &data_receipt.data_id), + &KeyForPostponedReceiptId::new(account_id, &data_receipt.data_id), )? { // There is already a receipt that is awaiting for the just received data. // Removing this pending data_id for the receipt from the state. state_update - .remove(&key_for_postponed_receipt_id(account_id, &data_receipt.data_id)); + .remove(KeyForPostponedReceiptId::new(account_id, &data_receipt.data_id)); // Checking how many input data items is pending for the receipt. let pending_data_count: u32 = - get(state_update, &key_for_pending_data_count(account_id, &receipt_id))? + get(state_update, &KeyForPendingDataCount::new(account_id, &receipt_id))? .ok_or_else(|| { - StorageError::StorageInconsistentState( - "pending data count should be in the state".to_string(), - ) - })?; + StorageError::StorageInconsistentState( + "pending data count should be in the state".to_string(), + ) + })?; if pending_data_count == 1 { // It was the last input data pending for this receipt. We'll cleanup // some receipt related fields from the state and execute the receipt. // Removing pending data count from the state. - state_update.remove(&key_for_pending_data_count(account_id, &receipt_id)); + state_update.remove(KeyForPendingDataCount::new(account_id, &receipt_id)); // Fetching the receipt itself. let ready_receipt = get_receipt(state_update, account_id, &receipt_id)? .ok_or_else(|| { @@ -728,7 +727,7 @@ impl Runtime { ) })?; // Removing the receipt from the state. - state_update.remove(&key_for_postponed_receipt(account_id, &receipt_id)); + state_update.remove(KeyForPostponedReceipt::new(account_id, &receipt_id)); // Executing the receipt. It will read all the input data and clean it up // from the state. return self @@ -746,7 +745,7 @@ impl Runtime { // pending data count in the state. set( state_update, - key_for_pending_data_count(account_id, &receipt_id), + KeyForPendingDataCount::new(account_id, &receipt_id), &(pending_data_count.checked_sub(1).ok_or_else(|| { StorageError::StorageInconsistentState( "pending data count is 0, but there is a new DataReceipt" @@ -770,7 +769,7 @@ impl Runtime { // receipt_id for the pending data_id into the state. set( state_update, - key_for_postponed_receipt_id(account_id, data_id), + KeyForPostponedReceiptId::new(account_id, data_id), &receipt.receipt_id, ) } @@ -793,7 +792,7 @@ impl Runtime { // Save the counter for the number of pending input data items into the state. set( state_update, - key_for_pending_data_count(account_id, &receipt.receipt_id), + KeyForPendingDataCount::new(account_id, &receipt.receipt_id), &pending_data_count, ); // Save the receipt itself into the state. @@ -1021,7 +1020,7 @@ impl Runtime { if total_gas_burnt >= gas_limit { break; } - let key = key_for_delayed_receipt(delayed_receipts_indices.first_index); + let key = KeyForDelayedReceipt::new(delayed_receipts_indices.first_index); let receipt: Receipt = get(&state_update, &key)?.ok_or_else(|| { StorageError::StorageInconsistentState(format!( "Delayed receipt #{} should be in the state", @@ -1037,7 +1036,7 @@ impl Runtime { )) })?; - state_update.remove(&key); + state_update.remove(key); // Math checked above: first_index is less than next_available_index delayed_receipts_indices.first_index += 1; process_receipt(&receipt, &mut state_update, &mut total_gas_burnt)?; @@ -1096,7 +1095,7 @@ impl Runtime { ) -> Result<(), StorageError> { set( state_update, - key_for_delayed_receipt(delayed_receipts_indices.next_available_index), + KeyForDelayedReceipt::new(delayed_receipts_indices.next_available_index), receipt, ); delayed_receipts_indices.next_available_index = @@ -1221,7 +1220,7 @@ impl Runtime { pending_data_count += 1; set( &mut state_update, - key_for_postponed_receipt_id(account_id, data_id), + KeyForPostponedReceiptId::new(account_id, data_id), &receipt.receipt_id, ) } @@ -1231,7 +1230,7 @@ impl Runtime { } else { set( &mut state_update, - key_for_pending_data_count(account_id, &receipt.receipt_id), + KeyForPendingDataCount::new(account_id, &receipt.receipt_id), &pending_data_count, ); set_receipt(&mut state_update, &receipt); diff --git a/runtime/runtime/src/state_viewer.rs b/runtime/runtime/src/state_viewer.rs index 8bd37431121..b2e181c398d 100644 --- a/runtime/runtime/src/state_viewer.rs +++ b/runtime/runtime/src/state_viewer.rs @@ -9,7 +9,7 @@ use near_primitives::account::{AccessKey, Account}; use near_primitives::hash::CryptoHash; use near_primitives::serialize::to_base64; use near_primitives::types::{AccountId, BlockHeight}; -use near_primitives::utils::{is_valid_account_id, prefix_for_data}; +use near_primitives::utils::{is_valid_account_id, KeyForData}; use near_primitives::views::{StateItem, ViewStateResult}; use near_runtime_fees::RuntimeFeesConfig; use near_store::{get_access_key, get_account, TrieUpdate}; @@ -62,14 +62,13 @@ impl TrieViewer { return Err(format!("Account ID '{}' is not valid", account_id).into()); } let mut values = vec![]; - let mut query = prefix_for_data(account_id); - let acc_sep_len = query.len(); - query.extend_from_slice(prefix); + let query = KeyForData::new(account_id, prefix); + let acc_sep_len = KeyForData::estimate_len(account_id, &[]); let mut iter = state_update.trie.iter(&state_update.get_root())?; iter.seek(&query)?; for item in iter { let (key, value) = item?; - if !key.starts_with(&query) { + if !key.starts_with(&query.as_ref()) { break; } values.push(StateItem { @@ -173,7 +172,7 @@ impl TrieViewer { #[cfg(test)] mod tests { use near_primitives::types::StateChangeCause; - use near_primitives::utils::key_for_data; + use near_primitives::utils::KeyForData; use near_primitives::views::StateItem; use testlib::runtime_utils::{ alice_account, encode_int, get_runtime_and_trie, get_test_trie_viewer, @@ -263,10 +262,10 @@ mod tests { fn test_view_state() { let (_, trie, root) = get_runtime_and_trie(); let mut state_update = TrieUpdate::new(trie.clone(), root); - state_update.set(key_for_data(&alice_account(), b"test123"), b"123".to_vec()); - state_update.set(key_for_data(&alice_account(), b"test321"), b"321".to_vec()); - state_update.set(key_for_data(&"alina".to_string(), b"qqq"), b"321".to_vec()); - state_update.set(key_for_data(&"alex".to_string(), b"qqq"), b"321".to_vec()); + state_update.set(KeyForData::new(&alice_account(), b"test123").into(), b"123".to_vec()); + state_update.set(KeyForData::new(&alice_account(), b"test321").into(), b"321".to_vec()); + state_update.set(KeyForData::new(&"alina".to_string(), b"qqq").into(), b"321".to_vec()); + state_update.set(KeyForData::new(&"alex".to_string(), b"qqq").into(), b"321".to_vec()); state_update.commit(StateChangeCause::InitialState); let (db_changes, new_root) = state_update.finalize().unwrap().into(trie.clone()).unwrap(); db_changes.commit().unwrap(); diff --git a/test-utils/testlib/src/user/rpc_user.rs b/test-utils/testlib/src/user/rpc_user.rs index 02809d1a486..f8aa54710b0 100644 --- a/test-utils/testlib/src/user/rpc_user.rs +++ b/test-utils/testlib/src/user/rpc_user.rs @@ -13,10 +13,9 @@ use near_jsonrpc::client::{new_client, JsonRpcClient}; use near_jsonrpc::ServerError; use near_primitives::hash::CryptoHash; use near_primitives::receipt::Receipt; -use near_primitives::rpc::BlockQueryInfo; use near_primitives::serialize::{to_base, to_base64}; use near_primitives::transaction::SignedTransaction; -use near_primitives::types::{AccountId, BlockHeight, BlockId, MaybeBlockId}; +use near_primitives::types::{AccountId, BlockHeight, BlockId, BlockIdOrFinality, MaybeBlockId}; use near_primitives::views::{ AccessKeyView, AccountView, BlockView, EpochValidatorInfo, ExecutionOutcomeView, FinalExecutionOutcomeView, QueryResponse, ViewStateResult, @@ -114,8 +113,10 @@ impl User for RpcUser { } fn get_block(&self, height: BlockHeight) -> Option { - self.actix(move |mut client| client.block(BlockQueryInfo::BlockId(BlockId::Height(height)))) - .ok() + self.actix(move |mut client| { + client.block(BlockIdOrFinality::BlockId(BlockId::Height(height))) + }) + .ok() } fn get_transaction_result(&self, _hash: &CryptoHash) -> ExecutionOutcomeView { diff --git a/tests/test_cases_runtime.rs b/tests/test_cases_runtime.rs index 1ff15534b14..5eb69df41e9 100644 --- a/tests/test_cases_runtime.rs +++ b/tests/test_cases_runtime.rs @@ -4,7 +4,7 @@ mod test { use near_chain_configs::Genesis; use near_primitives::serialize::to_base64; use near_primitives::state_record::StateRecord; - use near_primitives::utils::key_for_data; + use near_primitives::utils::KeyForData; use testlib::node::RuntimeNode; use testlib::runtime_utils::{add_test_contract, alice_account, bob_account}; use testlib::standard_test_cases::*; @@ -30,7 +30,7 @@ mod test { } } genesis.records.as_mut().push(StateRecord::Data { - key: to_base64(&key_for_data(&bob_account(), b"test")), + key: to_base64(&KeyForData::new(&bob_account(), b"test")), value: to_base64(b"123"), }); RuntimeNode::new_from_genesis(&alice_account(), genesis) @@ -51,7 +51,7 @@ mod test { _ => {} } genesis.records.as_mut().push(StateRecord::Data { - key: to_base64(&key_for_data(&bob_account(), b"test")), + key: to_base64(&KeyForData::new(&bob_account(), b"test")), value: to_base64(b"123"), }); RuntimeNode::new_from_genesis(&alice_account(), genesis)