diff --git a/Cargo.lock b/Cargo.lock index c58f14d4260..e053c68acbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -549,10 +549,10 @@ dependencies = [ [[package]] name = "cached" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "once_cell 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1696,14 +1696,6 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "lock_api" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "lock_api" version = "0.2.0" @@ -1903,7 +1895,7 @@ name = "near-chain" version = "0.1.0" dependencies = [ "borsh 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1926,7 +1918,7 @@ version = "0.1.0" dependencies = [ "actix 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "borsh 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1950,7 +1942,7 @@ dependencies = [ "actix 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "borsh 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2002,7 +1994,7 @@ name = "near-epoch-manager" version = "0.0.1" dependencies = [ "borsh 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "near-chain 0.1.0", "near-crypto 0.1.0", @@ -2080,7 +2072,7 @@ dependencies = [ "borsh 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2149,7 +2141,7 @@ dependencies = [ "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "borsh 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2198,7 +2190,7 @@ version = "0.4.0" dependencies = [ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "near-runtime-fees 0.4.0", "near-vm-errors 0.4.0", "near-vm-logic 0.4.0", @@ -2273,7 +2265,7 @@ dependencies = [ "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "borsh 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "genesis-populate 0.1.0", "indicatif 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2334,11 +2326,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "once_cell" -version = "0.1.8" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "opaque-debug" @@ -2444,15 +2433,6 @@ name = "parity-wasm" version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "parking_lot" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "parking_lot" version = "0.8.0" @@ -2473,18 +2453,6 @@ dependencies = [ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "parking_lot_core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "parking_lot_core" version = "0.5.0" @@ -3056,11 +3024,6 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "scopeguard" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "scopeguard" version = "1.0.0" @@ -4091,7 +4054,7 @@ dependencies = [ "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -"checksum cached 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24f3f5ca5651b8360fb859d6fd1b80fb4c25bc3cbf3ffc3a74d3d6a79fba328e" +"checksum cached 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b052fd10f32987c3bd028d91ef86190b36fba5c8fccb5515d42083f061e6104" "checksum cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "926013f2860c46252efceabb19f4a6b308197505082c609025aa6706c011d427" "checksum cc 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)" = "0213d356d3c4ea2c18c40b037c3be23cd639825c18f25ee670ac7813beeef99c" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" @@ -4209,7 +4172,6 @@ dependencies = [ "checksum libsodium-sys 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1c344ff12b90ef8fa1f0fffacd348c1fd041db331841fec9eab23fdb991f5e73" "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" "checksum local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e1ceb20f39ff7ae42f3ff9795f3986b1daad821caaa1e1732a0944103a5a1a66" -"checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" "checksum lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed946d4529956a20f2d63ebe1b69996d5a2137c91913fe3ebbeff957f5bca7ff" "checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" @@ -4234,7 +4196,7 @@ dependencies = [ "checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" "checksum num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcef43580c035376c0705c42792c294b66974abbfd2789b511784023f71f3273" "checksum number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" -"checksum once_cell 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "532c29a261168a45ce28948f9537ddd7a5dd272cc513b3017b1e82a88f962c37" +"checksum once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "891f486f630e5c5a4916c7e16c4b24a53e78c860b646e9f8e005e4f16847bfed" "checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" "checksum openssl 0.10.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2f372b2b53ce10fb823a337aaa674e3a7d072b957c6264d0f4ff0bd86e657449" "checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" @@ -4247,10 +4209,8 @@ dependencies = [ "checksum parity-secp256k1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4fca4f82fccae37e8bbdaeb949a4a218a1bbc485d11598f193d2a908042e5fc1" "checksum parity-snappy-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1a413d51e5e1927320c9de992998e4a279dffb8c8a7363570198bd8383e66f1b" "checksum parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" -"checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" "checksum parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa7767817701cce701d5585b9c4db3cdd02086398322c1d7e8bf5094a96a2ce7" "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -"checksum parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" "checksum parking_lot_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cb88cb1cb3790baa6776844f968fea3be44956cf184fa1be5a03341f5491278c" "checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" @@ -4310,7 +4270,6 @@ dependencies = [ "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" "checksum same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "585e8ddcedc187886a30fa705c47985c3fa88d06624095856b36ca0b82ff4421" "checksum schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "87f550b06b6cba9c8b8be3ee73f391990116bf527450d2556e9b9ce263b9a021" -"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" "checksum sct 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f5adf8fbd58e1b1b52699dc8bed2630faecb6d8c7bee77d009d6bbe4af569b9" "checksum security-framework 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eee63d0f4a9ec776eeb30e220f0bc1e092c3ad744b2a379e3993070364d3adc2" diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index ec2d87ddc54..f2d8bc2063e 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -13,7 +13,7 @@ kvdb = "0.1" rand = "0.7" serde = "1.0" serde_derive = "1.0" -cached = "0.9.0" +cached = "0.11.0" lazy_static = "1.4" rust-lzma = "0.5" diff --git a/chain/chunks/Cargo.toml b/chain/chunks/Cargo.toml index 4670c97ef99..a5c9fb852dc 100644 --- a/chain/chunks/Cargo.toml +++ b/chain/chunks/Cargo.toml @@ -15,7 +15,7 @@ borsh = "0.2.9" serde = "1.0" serde_derive = "1.0" reed-solomon-erasure = "3.1.1" -cached = "0.9.0" +cached = "0.11.0" near-crypto = { path = "../../core/crypto" } near-primitives = { path = "../../core/primitives" } diff --git a/chain/chunks/src/chunk_cache.rs b/chain/chunks/src/chunk_cache.rs index 522aa9b6de9..bdf798dd346 100644 --- a/chain/chunks/src/chunk_cache.rs +++ b/chain/chunks/src/chunk_cache.rs @@ -1,5 +1,7 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use cached::{Cached, SizedCache}; +use near_primitives::hash::CryptoHash; use near_primitives::sharding::{ ChunkHash, PartialEncodedChunk, PartialEncodedChunkPart, ReceiptProof, ShardChunkHeader, }; @@ -7,6 +9,8 @@ use near_primitives::types::{BlockIndex, ShardId}; const HEIGHT_HORIZON: u64 = 1024; const MAX_HEIGHTS_AHEAD: u64 = 5; +const CHUNK_HEADER_HEIGHT_HORIZON: u64 = 10; +const NUM_BLOCK_HASH_TO_CHUNK_HEADER: usize = 30; pub struct EncodedChunksCacheEntry { pub header: ShardChunkHeader, @@ -18,7 +22,8 @@ pub struct EncodedChunksCache { largest_seen_height: BlockIndex, encoded_chunks: HashMap, - height_map: HashMap>, + height_map: HashMap>, + block_hash_to_chunk_headers: SizedCache>, } impl EncodedChunksCacheEntry { @@ -49,6 +54,7 @@ impl EncodedChunksCache { largest_seen_height: 0, encoded_chunks: HashMap::new(), height_map: HashMap::new(), + block_hash_to_chunk_headers: SizedCache::with_size(NUM_BLOCK_HASH_TO_CHUNK_HEADER), } } @@ -61,6 +67,10 @@ impl EncodedChunksCache { } pub fn insert(&mut self, chunk_hash: ChunkHash, entry: EncodedChunksCacheEntry) { + self.height_map + .entry(entry.header.inner.height_created) + .or_insert_with(|| HashSet::default()) + .insert(chunk_hash.clone()); self.encoded_chunks.insert(chunk_hash, entry); } @@ -91,8 +101,13 @@ impl EncodedChunksCache { ) -> bool { let chunk_hash = partial_encoded_chunk.chunk_hash.clone(); if self.encoded_chunks.contains_key(&chunk_hash) || partial_encoded_chunk.header.is_some() { - self.get_or_insert_from_header(chunk_hash, partial_encoded_chunk.header.as_ref()) - .merge_in_partial_encoded_chunk(&partial_encoded_chunk); + let entry = self.get_or_insert_from_header( + chunk_hash.clone(), + partial_encoded_chunk.header.as_ref(), + ); + let height = entry.header.inner.height_created; + entry.merge_in_partial_encoded_chunk(&partial_encoded_chunk); + self.height_map.entry(height).or_insert_with(|| HashSet::default()).insert(chunk_hash); return true; } else { return false; @@ -127,4 +142,78 @@ impl EncodedChunksCache { } } } + + pub fn insert_chunk_header(&mut self, shard_id: ShardId, header: ShardChunkHeader) { + let height = header.inner.height_created; + if height >= self.largest_seen_height.saturating_sub(CHUNK_HEADER_HEIGHT_HORIZON) + && height <= self.largest_seen_height + MAX_HEIGHTS_AHEAD + { + let mut block_hash_to_chunk_headers = self + .block_hash_to_chunk_headers + .cache_remove(&header.inner.prev_block_hash) + .unwrap_or_else(|| vec![]); + let prev_block_hash = header.inner.prev_block_hash; + block_hash_to_chunk_headers.push((shard_id, header)); + self.block_hash_to_chunk_headers + .cache_set(prev_block_hash, block_hash_to_chunk_headers); + } + } + + pub fn get_chunk_headers_for_block( + &mut self, + prev_block_hash: &CryptoHash, + ) -> Vec<(ShardId, ShardChunkHeader)> { + self.block_hash_to_chunk_headers.cache_remove(prev_block_hash).unwrap_or_else(|| vec![]) + } + + pub fn num_chunks_for_block(&mut self, prev_block_hash: &CryptoHash) -> ShardId { + self.block_hash_to_chunk_headers + .cache_get(prev_block_hash) + .map(|x| x.len() as ShardId) + .unwrap_or_else(|| 0) + } +} + +#[cfg(test)] +mod tests { + use crate::chunk_cache::EncodedChunksCache; + use crate::ChunkRequestInfo; + use near_crypto::{InMemorySigner, KeyType}; + use near_primitives::hash::CryptoHash; + use near_primitives::sharding::{PartialEncodedChunk, ShardChunkHeader}; + use std::collections::HashMap; + + #[test] + fn test_cache_removal() { + let mut cache = EncodedChunksCache::new(); + let signer = InMemorySigner::from_random("test".to_string(), KeyType::ED25519); + let partial_encoded_chunk = PartialEncodedChunk { + shard_id: 0, + chunk_hash: Default::default(), + header: Some(ShardChunkHeader::new( + CryptoHash::default(), + CryptoHash::default(), + CryptoHash::default(), + CryptoHash::default(), + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + CryptoHash::default(), + CryptoHash::default(), + vec![], + &signer, + )), + parts: vec![], + receipts: vec![], + }; + assert!(cache.merge_in_partial_encoded_chunk(&partial_encoded_chunk)); + cache.update_largest_seen_height::(2000, &HashMap::default()); + assert!(cache.encoded_chunks.is_empty()); + assert!(cache.height_map.is_empty()); + } } diff --git a/chain/chunks/src/lib.rs b/chain/chunks/src/lib.rs index ad1035479f5..5135e3edd71 100644 --- a/chain/chunks/src/lib.rs +++ b/chain/chunks/src/lib.rs @@ -123,7 +123,7 @@ impl RequestPool { } pub fn remove(&mut self, chunk_hash: &ChunkHash) { - let _ = self.requests.remove(chunk_hash); + self.requests.remove(chunk_hash); } pub fn fetch(&mut self) -> Vec<(ChunkHash, ChunkRequestInfo)> { @@ -156,8 +156,6 @@ pub struct ShardsManager { network_adapter: Arc, encoded_chunks: EncodedChunksCache, - block_hash_to_chunk_headers: HashMap>, - requested_partial_encoded_chunks: RequestPool, } @@ -173,7 +171,6 @@ impl ShardsManager { runtime_adapter, network_adapter, encoded_chunks: EncodedChunksCache::new(), - block_hash_to_chunk_headers: HashMap::new(), requested_partial_encoded_chunks: RequestPool::new( Duration::from_millis(CHUNK_REQUEST_RETRY_MS), Duration::from_millis(CHUNK_REQUEST_SWITCH_TO_OTHERS_MS), @@ -409,18 +406,15 @@ impl ShardsManager { Ok(()) } - pub fn num_chunks_for_block(&mut self, prev_block_hash: CryptoHash) -> ShardId { - self.block_hash_to_chunk_headers - .get(&prev_block_hash) - .map(|x| x.len() as ShardId) - .unwrap_or_else(|| 0) + pub fn num_chunks_for_block(&mut self, prev_block_hash: &CryptoHash) -> ShardId { + self.encoded_chunks.num_chunks_for_block(prev_block_hash) } pub fn prepare_chunks( &mut self, - prev_block_hash: CryptoHash, + prev_block_hash: &CryptoHash, ) -> Vec<(ShardId, ShardChunkHeader)> { - self.block_hash_to_chunk_headers.remove(&prev_block_hash).unwrap_or_else(|| vec![]) + self.encoded_chunks.get_chunk_headers_for_block(&prev_block_hash) } pub fn insert_transaction(&mut self, shard_id: ShardId, tx: SignedTransaction) { @@ -716,11 +710,9 @@ impl ShardsManager { entry.parts.len() >= self.runtime_adapter.num_data_parts(&prev_block_hash); if have_all_parts { - self.block_hash_to_chunk_headers - .entry(header.inner.prev_block_hash) - .or_insert_with(|| vec![]) - .push((partial_encoded_chunk.shard_id, header.clone())); + self.encoded_chunks.insert_chunk_header(partial_encoded_chunk.shard_id, header.clone()); } + let entry = self.encoded_chunks.get(&chunk_hash).unwrap(); if have_all_parts && have_all_receipts { let cares_about_shard = self.cares_about_shard_this_or_next_epoch( @@ -1062,10 +1054,7 @@ impl ShardsManager { } // Add it to the set of chunks to be included in the next block - self.block_hash_to_chunk_headers - .entry(prev_block_hash) - .or_insert_with(|| vec![]) - .push((shard_id, encoded_chunk.header.clone())); + self.encoded_chunks.insert_chunk_header(shard_id, encoded_chunk.header.clone()); // Store the chunk in the permanent storage self.decode_and_persist_encoded_chunk(encoded_chunk, chain_store, merkle_paths)?; diff --git a/chain/client/Cargo.toml b/chain/client/Cargo.toml index 1272e235a3f..7273076e099 100644 --- a/chain/client/Cargo.toml +++ b/chain/client/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "1.0", features = ["derive"] } serde_derive = "1.0" serde_json = "1.0" sysinfo = "0.9.0" -cached = "0.9.0" +cached = "0.11.0" lazy_static = "1.4" borsh = "0.2.7" diff --git a/chain/client/src/client.rs b/chain/client/src/client.rs index 591e5b3032b..ce6c31d525e 100644 --- a/chain/client/src/client.rs +++ b/chain/client/src/client.rs @@ -238,7 +238,7 @@ impl Client { let total_approvals = total_block_producers - min(if prev_same_bp { 1 } else { 2 }, total_block_producers); let num_approvals = self.approvals.cache_get(&prev_hash).map(|h| h.len()).unwrap_or(0); - let num_chunks = self.shards_mgr.num_chunks_for_block(prev_hash); + let num_chunks = self.shards_mgr.num_chunks_for_block(&prev_hash); if head.height > 0 && num_approvals < min(total_approvals, 2 * total_block_producers / 3) && num_chunks < self.runtime_adapter.num_shards() @@ -249,7 +249,7 @@ impl Client { return Ok(None); } - let new_chunks = self.shards_mgr.prepare_chunks(prev_hash); + let new_chunks = self.shards_mgr.prepare_chunks(&prev_hash); // If we are producing empty blocks and there are no transactions. if !self.config.produce_empty_blocks && new_chunks.is_empty() { debug!(target: "client", "Empty blocks, skipping block production"); @@ -920,10 +920,7 @@ impl Client { shard_id: ShardId, ) -> Result { let head = self.chain.head()?; - // TODO(MarX, #1366): Forward tx even if I am a validator. - // How many validators ahead of current time should we forward tx? let target_height = head.height + TX_ROUTING_HEIGHT_HORIZON - 1; - self.runtime_adapter.get_chunk_producer(&head.epoch_id, target_height, shard_id) } diff --git a/chain/epoch_manager/Cargo.toml b/chain/epoch_manager/Cargo.toml index 489c38eba32..1e703db8437 100644 --- a/chain/epoch_manager/Cargo.toml +++ b/chain/epoch_manager/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] protocol_defining_rand = { package = "rand", version = "0.6.5" } log = "0.4" -cached = "0.9.0" +cached = "0.11.0" borsh = "0.2.9" rand = "0.7" serde = "1.0" diff --git a/chain/network/Cargo.toml b/chain/network/Cargo.toml index a7def6ea29e..7eb2aed8c32 100644 --- a/chain/network/Cargo.toml +++ b/chain/network/Cargo.toml @@ -19,7 +19,7 @@ byteorder = "1.2" lazy_static = "1.4" borsh = "0.2.9" -cached = "0.9.0" +cached = "0.11.0" near-crypto = { path = "../../core/crypto" } near-primitives = { path = "../../core/primitives" } diff --git a/chain/network/src/codec.rs b/chain/network/src/codec.rs index f9fbc31a95c..0e38c7fa8b1 100644 --- a/chain/network/src/codec.rs +++ b/chain/network/src/codec.rs @@ -154,4 +154,12 @@ mod test { }); test_codec(msg); } + + #[test] + fn test_account_id_bytes() { + let account_id = "near0".to_string(); + let enc = account_id.as_bytes(); + let dec_account_id = String::from_utf8_lossy(enc).to_string(); + assert_eq!(account_id, dec_account_id); + } } diff --git a/chain/network/src/peer_manager.rs b/chain/network/src/peer_manager.rs index 7ac8da5c01e..9961f505b73 100644 --- a/chain/network/src/peer_manager.rs +++ b/chain/network/src/peer_manager.rs @@ -98,7 +98,7 @@ impl PeerManagerActor { config: NetworkConfig, client_addr: Recipient, ) -> Result> { - let peer_store = PeerStore::new(store, &config.boot_nodes)?; + let peer_store = PeerStore::new(store.clone(), &config.boot_nodes)?; debug!(target: "network", "Found known peers: {} (boot nodes={})", peer_store.len(), config.boot_nodes.len()); let me = config.public_key.clone().into(); @@ -109,7 +109,7 @@ impl PeerManagerActor { peer_store, active_peers: HashMap::default(), outgoing_peers: HashSet::default(), - routing_table: RoutingTable::new(me), + routing_table: RoutingTable::new(me, store), monitor_peers_attempts: 0, pending_update_nonce_request: HashMap::new(), }) @@ -141,16 +141,18 @@ impl PeerManagerActor { "Failed to save peer data" ); + let target_peer_id = full_peer_info.peer_info.id.clone(); + let new_edge = Edge::new( - self.peer_id.clone(), // source - full_peer_info.peer_info.id.clone(), // target + self.peer_id.clone(), // source + target_peer_id.clone(), // target edge_info.nonce, edge_info.signature, full_peer_info.edge_info.signature.clone(), ); self.active_peers.insert( - full_peer_info.peer_info.id.clone(), + target_peer_id.clone(), ActivePeer { addr: addr.clone(), full_peer_info, @@ -165,7 +167,7 @@ impl PeerManagerActor { // TODO(MarX, #1363): Implement sync service. Right now all edges and known validators // are sent during handshake. let known_edges = self.routing_table.get_edges(); - let known_accounts = self.routing_table.get_accounts(); + let known_accounts = self.routing_table.get_announce_accounts(); let wait_for_sync = 1; // Start syncing network point of view. Wait until both parties are connected before start @@ -178,6 +180,12 @@ impl PeerManagerActor { }), }); + // Ask for peers list on connection. + let _ = addr.do_send(SendMessage { message: PeerMessage::PeersRequest }); + if let Some(active_peer) = act.active_peers.get_mut(&target_peer_id) { + active_peer.last_time_peer_requested = Utc::now(); + } + if peer_type == PeerType::Outbound { // Only broadcast new message from the outbound endpoint. // Wait a time out before broadcasting this new edge to let the other party finish handshake. @@ -519,8 +527,6 @@ impl PeerManagerActor { .and_then(|_, _, _| actix::fut::ok(())) .spawn(ctx); } else { - // TODO(MarX): This should be unreachable! Probably it is reaching this point because - // the peer is added to the routing table before being added to the set of active peers. debug!(target: "network", "Sending message to: {} (which is not an active peer) Active Peers: {:?}\n{:?}", peer_id, @@ -574,11 +580,11 @@ impl PeerManagerActor { // TODO(MarX, #1369): Message is dropped here. Define policy for this case. near_metrics::inc_counter(&metrics::DROP_MESSAGE_UNKNOWN_ACCOUNT); debug!(target: "network", "{:?} Drop message to {} Reason {:?}. Known peers: {:?} Message {:?}", - self.config.account_id, - account_id, - find_route_error, - self.routing_table.account_peers.keys(), - msg, + self.config.account_id, + account_id, + find_route_error, + self.routing_table.get_accounts_keys(), + msg, ); return; } @@ -679,9 +685,6 @@ impl Handler for PeerManagerActor { NetworkRequests::FetchInfo => { let (sent_bytes_per_sec, received_bytes_per_sec) = self.get_total_bytes_per_sec(); - let known_producers = - self.routing_table.account_peers.keys().cloned().collect::>(); - NetworkResponses::Info(NetworkInfo { active_peers: self .active_peers @@ -693,7 +696,7 @@ impl Handler for PeerManagerActor { most_weight_peers: self.most_weight_peers(), sent_bytes_per_sec, received_bytes_per_sec, - known_producers, + known_producers: self.routing_table.get_accounts_keys(), }) } NetworkRequests::Block { block } => { @@ -831,7 +834,7 @@ impl Handler for PeerManagerActor { .into_iter() .filter_map(|announce_account| { if let Some(current_announce_account) = - self.routing_table.account_peers.get(&announce_account.account_id) + self.routing_table.get_announce(&announce_account.account_id) { if announce_account.epoch_id == current_announce_account.epoch_id { None diff --git a/chain/network/src/peer_store.rs b/chain/network/src/peer_store.rs index 710ec86f347..dfd96b42cb1 100644 --- a/chain/network/src/peer_store.rs +++ b/chain/network/src/peer_store.rs @@ -141,6 +141,7 @@ impl PeerStore { /// Return healthy known peers up to given amount. pub fn healthy_peers(&self, max_count: u32) -> Vec { // TODO: better healthy peer definition here. + // Discussion: wdyt about using reachable peers in the current routing table? self.find_peers( |p| match p.status { KnownPeerStatus::Banned(_, _) => false, diff --git a/chain/network/src/routing.rs b/chain/network/src/routing.rs index e2e50b1825b..bb80259dfeb 100644 --- a/chain/network/src/routing.rs +++ b/chain/network/src/routing.rs @@ -1,20 +1,24 @@ use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use std::time::{Duration, Instant}; use borsh::{BorshDeserialize, BorshSerialize}; use byteorder::WriteBytesExt; use bytes::LittleEndian; use cached::{Cached, SizedCache}; +use log::warn; use log::{debug, trace}; use near_crypto::{SecretKey, Signature}; use near_metrics; use near_primitives::hash::{hash, CryptoHash}; use near_primitives::types::AccountId; +use near_store::{Store, COL_ACCOUNT_ANNOUNCEMENTS}; use crate::metrics; use crate::types::{AnnounceAccount, PeerId, PeerIdOrHash, Ping, Pong}; +const ANNOUNCE_ACCOUNT_CACHE_SIZE: usize = 10000; const ROUTE_BACK_CACHE_SIZE: usize = 10000; const ROUND_ROBIN_MAX_NONCE_DIFFERENCE_ALLOWED: usize = 10; @@ -225,13 +229,15 @@ impl Edge { pub struct RoutingTable { // TODO(MarX, #1363): Use cache and file storing to keep this information. /// PeerId associated for every known account id. - pub account_peers: HashMap, + account_peers: SizedCache, /// Active PeerId that are part of the shortest path to each PeerId. pub peer_forwarding: HashMap>, /// Store last update for known edges. pub edges_info: HashMap<(PeerId, PeerId), Edge>, /// Hash of messages that requires routing back to respective previous hop. pub route_back: SizedCache, + /// Access to store on disk + store: Arc, /// Current view of the network. Nodes are Peers and edges are active connections. raw_graph: Graph, /// Number of times each active connection was used to route a message. @@ -257,12 +263,13 @@ pub enum FindRouteError { } impl RoutingTable { - pub fn new(peer_id: PeerId) -> Self { + pub fn new(peer_id: PeerId, store: Arc) -> Self { Self { - account_peers: HashMap::new(), + account_peers: SizedCache::with_size(ANNOUNCE_ACCOUNT_CACHE_SIZE), peer_forwarding: HashMap::new(), edges_info: HashMap::new(), route_back: SizedCache::with_size(ROUTE_BACK_CACHE_SIZE), + store, raw_graph: Graph::new(peer_id), route_nonce: HashMap::new(), recalculation_scheduled: None, @@ -277,7 +284,7 @@ impl RoutingTable { pub fn find_route_from_peer_id(&mut self, peer_id: &PeerId) -> Result { if let Some(routes) = self.peer_forwarding.get(&peer_id) { // Strategy similar to Round Robin. Select node with least nonce and send it. Increase its - // nonce by one. Additionally if the difference between the highest and nonce and the lowest + // nonce by one. Additionally if the difference between the highest nonce and the lowest // nonce is greater than some threshold increase the lowest nonce to be at least // max nonce - threshold. @@ -331,9 +338,8 @@ impl RoutingTable { } /// Find peer that owns this AccountId. - pub fn account_owner(&self, account_id: &AccountId) -> Result { - self.account_peers - .get(account_id) + pub fn account_owner(&mut self, account_id: &AccountId) -> Result { + self.get_announce(account_id) .map(|announce_account| announce_account.peer_id.clone()) .ok_or_else(|| FindRouteError::AccountNotFound) } @@ -342,18 +348,23 @@ impl RoutingTable { /// Note: There is at most on peer id per account id. pub fn add_account(&mut self, announce_account: AnnounceAccount) { let account_id = announce_account.account_id.clone(); - self.account_peers.insert(account_id, announce_account); - near_metrics::inc_counter(&metrics::ACCOUNT_KNOWN); + self.account_peers.cache_set(account_id.clone(), announce_account.clone()); + + // Add account to store + let mut update = self.store.store_update(); + if let Err(e) = update + .set_ser(COL_ACCOUNT_ANNOUNCEMENTS, account_id.as_bytes(), &announce_account) + .and_then(|_| update.commit()) + { + warn!(target: "network", "Error saving announce account to store: {:?}", e); + } } // TODO(MarX, #1694): Allow one account id to be routed to several peer id. - pub fn contains_account(&self, announce_account: &AnnounceAccount) -> bool { - self.account_peers.get(&announce_account.account_id).map_or( - false, - |current_announce_account| { - current_announce_account.epoch_id == announce_account.epoch_id - }, - ) + pub fn contains_account(&mut self, announce_account: &AnnounceAccount) -> bool { + self.get_announce(&announce_account.account_id).map_or(false, |current_announce_account| { + current_announce_account.epoch_id == announce_account.epoch_id + }) } /// Add this edge to the current view of the network. @@ -423,10 +434,6 @@ impl RoutingTable { self.edges_info.iter().map(|(_, edge)| edge.clone()).collect() } - pub fn get_accounts(&self) -> Vec { - self.account_peers.iter().map(|(_key, value)| value.clone()).collect() - } - pub fn add_route_back(&mut self, hash: CryptoHash, peer_id: PeerId) { self.route_back.cache_set(hash, peer_id); } @@ -466,13 +473,12 @@ impl RoutingTable { (pings, pongs) } - pub fn info(&self) -> RoutingTableInfo { + pub fn info(&mut self) -> RoutingTableInfo { let account_peers = self - .account_peers - .iter() - .map(|(key, value)| (key.clone(), value.peer_id.clone())) + .get_announce_accounts() + .into_iter() + .map(|announce_account| (announce_account.account_id, announce_account.peer_id)) .collect(); - RoutingTableInfo { account_peers, peer_forwarding: self.peer_forwarding.clone() } } @@ -494,6 +500,40 @@ impl RoutingTable { ); near_metrics::set_gauge(&metrics::PEER_REACHABLE, self.peer_forwarding.len() as i64); } + + /// Public interface for `account_peers` + /// + /// Get keys currently on cache. + pub fn get_accounts_keys(&mut self) -> Vec { + self.account_peers.key_order().cloned().collect() + } + + /// Get announce accounts on cache. + pub fn get_announce_accounts(&mut self) -> Vec { + self.account_peers.value_order().cloned().collect() + } + + /// Get account announce from + pub fn get_announce(&mut self, account_id: &AccountId) -> Option { + if let Some(announce_account) = self.account_peers.cache_get(&account_id) { + Some(announce_account.clone()) + } else { + self.store + .get_ser(COL_ACCOUNT_ANNOUNCEMENTS, account_id.as_bytes()) + .and_then(|res: Option| { + if let Some(announce_account) = res { + self.add_account(announce_account.clone()); + Ok(Some(announce_account)) + } else { + Ok(None) + } + }) + .unwrap_or_else(|e| { + warn!(target: "network", "Error loading announce account from store: {:?}", e); + None + }) + } + } } pub struct ProcessEdgeResult { diff --git a/chain/network/src/test_utils.rs b/chain/network/src/test_utils.rs index 939c29525be..6b451d39b83 100644 --- a/chain/network/src/test_utils.rs +++ b/chain/network/src/test_utils.rs @@ -1,13 +1,18 @@ use std::collections::{HashMap, HashSet}; +use std::mem::size_of; use std::net::TcpListener; use std::time::{Duration, Instant}; use actix::{Actor, AsyncContext, Context, System}; +use byteorder::{ByteOrder, LittleEndian}; use futures::future; use futures::future::Future; +use rand::{thread_rng, RngCore}; use tokio::timer::Delay; use near_crypto::{KeyType, SecretKey}; +use near_primitives::hash::hash; +use near_primitives::types::EpochId; use crate::types::{NetworkConfig, PeerId, PeerInfo}; @@ -147,6 +152,12 @@ pub fn random_peer_id() -> PeerId { sk.public_key().into() } +pub fn random_epoch_id() -> EpochId { + let mut buffer = [0u8; size_of::()]; + LittleEndian::write_u64(&mut buffer, thread_rng().next_u64()); + EpochId(hash(buffer.as_ref())) +} + pub fn expected_routing_tables( current: HashMap>, expected: Vec<(PeerId, Vec)>, diff --git a/chain/network/tests/cache.rs b/chain/network/tests/cache.rs new file mode 100644 index 00000000000..98c9ac9b076 --- /dev/null +++ b/chain/network/tests/cache.rs @@ -0,0 +1,106 @@ +use near_crypto::Signature; +use near_network::routing::RoutingTable; +use near_network::test_utils::{random_epoch_id, random_peer_id}; +use near_network::types::AnnounceAccount; +use near_store::test_utils::create_test_store; + +#[test] +fn announcement_same_epoch() { + let store = create_test_store(); + + let peer_id0 = random_peer_id(); + let peer_id1 = random_peer_id(); + let epoch_id0 = random_epoch_id(); + + let mut routing_table = RoutingTable::new(peer_id0.clone(), store); + + let announce0 = AnnounceAccount { + account_id: "near0".to_string(), + peer_id: peer_id0.clone(), + epoch_id: epoch_id0.clone(), + signature: Signature::default(), + }; + + // Same as announce1 but with different peer id + let announce1 = AnnounceAccount { + account_id: "near0".to_string(), + peer_id: peer_id1.clone(), + epoch_id: epoch_id0, + signature: Signature::default(), + }; + + routing_table.add_account(announce0.clone()); + assert!(routing_table.contains_account(&announce0)); + assert!(routing_table.contains_account(&announce1)); + assert_eq!(routing_table.get_announce_accounts().len(), 1); + assert_eq!(routing_table.account_owner(&announce0.account_id).unwrap(), peer_id0); + routing_table.add_account(announce1.clone()); + assert_eq!(routing_table.get_announce_accounts().len(), 1); + assert_eq!(routing_table.account_owner(&announce1.account_id).unwrap(), peer_id1); +} + +#[test] +fn dont_load_on_build() { + let store = create_test_store(); + + let peer_id0 = random_peer_id(); + let peer_id1 = random_peer_id(); + let epoch_id0 = random_epoch_id(); + let epoch_id1 = random_epoch_id(); + + let mut routing_table = RoutingTable::new(peer_id0.clone(), store.clone()); + + let announce0 = AnnounceAccount { + account_id: "near0".to_string(), + peer_id: peer_id0.clone(), + epoch_id: epoch_id0.clone(), + signature: Signature::default(), + }; + + // Same as announce1 but with different peer id + let announce1 = AnnounceAccount { + account_id: "near1".to_string(), + peer_id: peer_id1, + epoch_id: epoch_id1, + signature: Signature::default(), + }; + + routing_table.add_account(announce0.clone()); + routing_table.add_account(announce1.clone()); + let accounts = routing_table.get_announce_accounts(); + assert!(vec![announce0.clone(), announce1.clone()] + .iter() + .all(|announce| { accounts.contains(announce) })); + assert_eq!(routing_table.get_announce_accounts().len(), 2); + + let mut routing_table1 = RoutingTable::new(peer_id0, store); + assert!(routing_table1.get_announce_accounts().is_empty()); +} + +#[test] +fn load_from_disk() { + let store = create_test_store(); + + let peer_id0 = random_peer_id(); + let epoch_id0 = random_epoch_id(); + + let mut routing_table = RoutingTable::new(peer_id0.clone(), store.clone()); + let mut routing_table1 = RoutingTable::new(peer_id0.clone(), store.clone()); + + let announce0 = AnnounceAccount { + account_id: "near0".to_string(), + peer_id: peer_id0.clone(), + epoch_id: epoch_id0.clone(), + signature: Signature::default(), + }; + + // Announcement is added to cache of the first routing table and to disk + routing_table.add_account(announce0.clone()); + assert_eq!(routing_table.get_announce_accounts().len(), 1); + // Cache of second routing table is empty + assert_eq!(routing_table1.get_announce_accounts().len(), 0); + // Try to find this peer and load it from disk + assert_eq!(routing_table1.account_owner(&announce0.account_id).unwrap(), peer_id0); + // Cache of second routing table should contain account loaded from disk + assert_eq!(routing_table1.get_announce_accounts().len(), 1); +} diff --git a/chain/network/tests/routing.rs b/chain/network/tests/routing.rs index cf55acbe602..1a7ca4b2bd3 100644 --- a/chain/network/tests/routing.rs +++ b/chain/network/tests/routing.rs @@ -265,12 +265,13 @@ impl StateMachine { struct Runner { num_nodes: usize, num_validators: usize, + peer_max_count: u32, state_machine: Option, } impl Runner { - fn new(num_nodes: usize, num_validators: usize) -> Self { - Self { num_nodes, num_validators, state_machine: Some(StateMachine::new()) } + fn new(num_nodes: usize, num_validators: usize, peer_max_count: u32) -> Self { + Self { num_nodes, num_validators, peer_max_count, state_machine: Some(StateMachine::new()) } } fn push(&mut self, action: Action) { @@ -302,7 +303,7 @@ impl Runner { vec![], validators.clone(), genesis_time, - 0, + self.peer_max_count, ) }) .collect(); @@ -345,7 +346,7 @@ fn simple() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(2, 1); + let mut runner = Runner::new(2, 1, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::CheckRoutingTable(0, vec![(1, vec![1])])); @@ -361,7 +362,7 @@ fn three_nodes_path() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(3, 2); + let mut runner = Runner::new(3, 2, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::AddEdge(1, 2)); @@ -379,7 +380,7 @@ fn three_nodes_star() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(3, 2); + let mut runner = Runner::new(3, 2, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::AddEdge(1, 2)); @@ -401,7 +402,7 @@ fn join_components() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(4, 4); + let mut runner = Runner::new(4, 4, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::AddEdge(2, 3)); @@ -430,7 +431,7 @@ fn account_propagation() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(3, 2); + let mut runner = Runner::new(3, 2, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::CheckAccountId(1, vec![0, 1])); @@ -447,7 +448,7 @@ fn ping_simple() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(2, 2); + let mut runner = Runner::new(2, 2, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::CheckRoutingTable(0, vec![(1, vec![1])])); @@ -465,7 +466,7 @@ fn ping_jump() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(3, 2); + let mut runner = Runner::new(3, 2, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::AddEdge(1, 2)); @@ -484,7 +485,7 @@ fn simple_remove() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(3, 3); + let mut runner = Runner::new(3, 3, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::AddEdge(1, 2)); @@ -504,7 +505,7 @@ fn square() { init_test_logger(); System::run(|| { - let mut runner = Runner::new(4, 4); + let mut runner = Runner::new(4, 4, 0); runner.push(Action::AddEdge(0, 1)); runner.push(Action::AddEdge(1, 2)); @@ -521,3 +522,32 @@ fn square() { }) .unwrap(); } + +/// Spin up four nodes and connect them in a square. +/// Each node will have at most two connections. +/// Turn off two non adjacent nodes, and check other two nodes create +/// a connection among them. +#[test] +fn churn_attack() { + init_test_logger(); + + System::run(|| { + let mut runner = Runner::new(4, 4, 1); + + runner.push(Action::AddEdge(0, 1)); + runner.push(Action::AddEdge(1, 2)); + runner.push(Action::AddEdge(2, 3)); + runner.push(Action::AddEdge(3, 0)); + runner + .push(Action::CheckRoutingTable(0, vec![(1, vec![1]), (3, vec![3]), (2, vec![1, 3])])); + runner + .push(Action::CheckRoutingTable(2, vec![(1, vec![1]), (3, vec![3]), (0, vec![1, 3])])); + runner.push(Action::Stop(1)); + runner.push(Action::Stop(3)); + runner.push(Action::CheckRoutingTable(0, vec![(2, vec![2])])); + runner.push(Action::CheckRoutingTable(2, vec![(0, vec![0])])); + + runner.run(); + }) + .unwrap(); +} diff --git a/core/store/Cargo.toml b/core/store/Cargo.toml index 47b276256ab..a1afc8a88d3 100644 --- a/core/store/Cargo.toml +++ b/core/store/Cargo.toml @@ -12,7 +12,7 @@ kvdb-memorydb = "0.1" kvdb-rocksdb = "0.1.3" serde = "1.0" serde_derive = "1.0" -cached = "0.9.0" +cached = "0.11.0" log = "0.4" borsh = "0.2.9" diff --git a/core/store/src/lib.rs b/core/store/src/lib.rs index 3de88eedf5d..eb80178171b 100644 --- a/core/store/src/lib.rs +++ b/core/store/src/lib.rs @@ -60,7 +60,9 @@ pub const COL_LAST_APPROVALS_PER_ACCOUNT: Option = Some(21); pub const COL_MY_LAST_APPROVALS_PER_CHAIN: Option = Some(22); pub const COL_STATE_PARTS: Option = Some(23); pub const COL_EPOCH_START: Option = Some(24); -const NUM_COLS: u32 = 25; +// Map account_id to announce_account +pub const COL_ACCOUNT_ANNOUNCEMENTS: Option = Some(25); +const NUM_COLS: u32 = 26; pub struct Store { storage: Arc, diff --git a/runtime/near-vm-runner/Cargo.toml b/runtime/near-vm-runner/Cargo.toml index 58e301c0458..f97bb25e41f 100644 --- a/runtime/near-vm-runner/Cargo.toml +++ b/runtime/near-vm-runner/Cargo.toml @@ -13,7 +13,7 @@ This crate implements the specification of the interface that Near blockchain ex """ [dependencies] -cached = "0.9.0" +cached = "0.11.0" wasmer-runtime = { version = "0.9.0", features = ["default-backend-singlepass"], default-features = false } wasmer-runtime-core = { version = "0.9.0", features = ["backend-singlepass"]} near-runtime-fees = { path="../near-runtime-fees", version = "0.4.0" } diff --git a/runtime/runtime/Cargo.toml b/runtime/runtime/Cargo.toml index c3c3797ceca..b4e3a2e91c8 100644 --- a/runtime/runtime/Cargo.toml +++ b/runtime/runtime/Cargo.toml @@ -17,7 +17,7 @@ sodiumoxide = "0.2.5" lazy_static = "1.4" borsh = "0.2.9" -cached = "0.9.0" +cached = "0.11.0" near-crypto = { path = "../../core/crypto" } near-primitives = { path = "../../core/primitives" }