Merge pull request paritytech#291 from subspace/partial-replica-impl-…

…refactoring-and-fixes

Partial replica impl refactoring and fixes

.github/workflows/rust.yaml

@@ -29,7 +29,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         # TODO: Below can be removed when https://github.com/actions-rs/toolchain/issues/126 is resolved
         with:
-          toolchain: nightly
+          toolchain: nightly-2022-02-15
           target: wasm32-unknown-unknown
           override: true
           components: rustfmt, clippy
@@ -73,7 +73,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         # TODO: Below can be removed when https://github.com/actions-rs/toolchain/issues/126 is resolved
         with:
-          toolchain: nightly
+          toolchain: nightly-2022-02-15
           target: wasm32-unknown-unknown
           override: true
           components: rustfmt, clippy
@@ -134,7 +134,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         # TODO: Below can be removed when https://github.com/actions-rs/toolchain/issues/126 is resolved
         with:
-          toolchain: nightly
+          toolchain: nightly-2022-02-15
           target: wasm32-unknown-unknown
           override: true
           components: rustfmt, clippy

.github/workflows/rustdoc.yml

@@ -21,7 +21,7 @@ jobs:
     - name: Install Rust toolchain
       uses: actions-rs/toolchain@v1
       with:
-        toolchain: nightly
+        toolchain: nightly-2022-02-15
         target: wasm32-unknown-unknown
         profile: minimal
         override: true

.github/workflows/snapshot-build.yml

@@ -118,7 +118,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         # TODO: Below can be removed when https://github.com/actions-rs/toolchain/issues/126 is resolved
         with:
-          toolchain: nightly
+          toolchain: nightly-2022-02-15
           target: wasm32-unknown-unknown
           override: true
 

Dockerfile-farmer

@@ -1,6 +1,6 @@
 FROM ubuntu:20.04
 
-ARG RUSTC_VERSION=nightly
+ARG RUSTC_VERSION=nightly-2022-02-15
 
 WORKDIR /code
 

Dockerfile-node

@@ -1,6 +1,6 @@
 FROM ubuntu:20.04
 
-ARG RUSTC_VERSION=nightly
+ARG RUSTC_VERSION=nightly-2022-02-15
 
 WORKDIR /code
 

crates/subspace-core-primitives/src/lib.rs

@@ -256,7 +256,7 @@ impl AsMut<[u8]> for Piece {
 }
 
 /// Flat representation of multiple pieces concatenated for higher efficient for processing.
-#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)]
+#[derive(Debug, Default, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))]
 pub struct FlatPieces(Vec<u8>);

crates/subspace-farmer/Cargo.toml

@@ -17,11 +17,9 @@ arc-swap = "1.5.0"
 async-oneshot = "0.5.0"
 async-trait = "0.1.52"
 clap = { version = "3.1.6", features = ["color", "derive"] }
-crossbeam = "0.8"
 dirs = "4.0.0"
 env_logger = "0.9.0"
 event-listener-primitives = "2.0.1"
-fs2 = "0.4"
 futures = "0.3.21"
 hex = "0.4.3"
 hex-buffer-serde = "0.3.0"

crates/subspace-farmer/README.md

@@ -39,12 +39,7 @@ sudo apt-get install llvm clang
 
 Then install the framer using Cargo:
 ```
-cargo +nightly install subspace-farmer
-```
-
-NOTE: Above command requires nightly compiler for now, make sure to install it if you don't have one yet:
-```
-rustup toolchain install nightly
+cargo install subspace-farmer
 ```
 
 ## Usage

crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs

@@ -4,6 +4,7 @@ use log::info;
 use std::mem;
 use std::sync::Arc;
 use std::time::Duration;
+use subspace_core_primitives::PIECE_SIZE;
 use subspace_farmer::ws_rpc_server::{RpcServer, RpcServerImpl};
 use subspace_farmer::{
     Commitments, FarmerData, Farming, Identity, ObjectMappings, Plot, Plotting, RpcClient, WsRpc,
@@ -42,7 +43,14 @@ pub(crate) async fn farm(
     let plot_fut = tokio::task::spawn_blocking({
         let base_directory = base_directory.clone();
 
-        move || Plot::open_or_create(&base_directory, address, plot_size)
+        // TODO: Piece count should account for database overhead of various additional databases
+        move || {
+            Plot::open_or_create(
+                &base_directory,
+                address,
+                plot_size.map(|plot_size| plot_size / PIECE_SIZE as u64),
+            )
+        }
     });
     let plot = plot_fut.await.unwrap()?;
 

crates/subspace-farmer/src/commitments.rs

@@ -13,7 +13,7 @@ use rocksdb::DB;
 use std::io;
 use std::path::PathBuf;
 use std::sync::Arc;
-use subspace_core_primitives::{FlatPieces, Piece, Salt, Tag, PIECE_SIZE};
+use subspace_core_primitives::{Piece, Salt, Tag, PIECE_SIZE};
 use thiserror::Error;
 
 const BATCH_SIZE: u64 = (16 * 1024 * 1024 / PIECE_SIZE) as u64;
@@ -170,7 +170,7 @@ impl Commitments {
         Ok(())
     }
 
-    pub(crate) fn remove_pieces(&self, pieces: Vec<Piece>) -> Result<(), CommitmentError> {
+    pub(crate) fn remove_pieces(&self, pieces: &[Piece]) -> Result<(), CommitmentError> {
         let salts = self.inner.commitment_databases.lock().get_salts();
 
         for salt in salts {
@@ -189,28 +189,26 @@ impl Commitments {
 
             let db_guard = db_entry.lock();
 
-            let db = match db_guard.clone() {
-                Some(db) => db,
-                None => {
-                    continue;
+            if let Some(db) = db_guard.as_ref() {
+                for piece in pieces {
+                    let tag = subspace_solving::create_tag(piece, salt);
+                    db.delete(tag).map_err(CommitmentError::CommitmentDb)?;
                 }
-            };
-
-            for piece in &pieces {
-                let tag = subspace_solving::create_tag(piece, salt);
-                db.delete(tag).map_err(CommitmentError::CommitmentDb)?;
             }
         }
 
         Ok(())
     }
 
     /// Create commitments for all salts for specified pieces
-    pub(crate) fn create_for_pieces(
-        &self,
-        pieces: &Arc<FlatPieces>,
-        offsets: impl IntoIterator<Item = PieceOffset> + Clone,
-    ) -> Result<(), CommitmentError> {
+    pub(crate) fn create_for_pieces<'a, 'iter, F, Iter>(
+        &'a self,
+        pieces_with_offsets: F,
+    ) -> Result<(), CommitmentError>
+    where
+        F: Fn() -> Iter,
+        Iter: Iterator<Item = (PieceOffset, &'iter [u8])>,
+    {
         let salts = self.inner.commitment_databases.lock().get_salts();
 
         for salt in salts {
@@ -229,22 +227,18 @@ impl Commitments {
 
             let db_guard = db_entry.lock();
 
-            let db = match db_guard.clone() {
-                Some(db) => db,
-                None => {
-                    continue;
+            if let Some(db) = db_guard.as_ref() {
+                let tags_with_offset: Vec<(PieceOffset, Tag)> = pieces_with_offsets()
+                    .map(|(piece_offset, piece)| {
+                        (piece_offset, subspace_solving::create_tag(piece, salt))
+                    })
+                    .collect();
+
+                for (piece_offset, tag) in tags_with_offset {
+                    db.put(tag, piece_offset.to_le_bytes())
+                        .map_err(CommitmentError::CommitmentDb)?;
                 }
             };
-
-            let tags: Vec<Tag> = pieces
-                .par_chunks_exact(PIECE_SIZE)
-                .map(|piece| subspace_solving::create_tag(piece, salt))
-                .collect();
-
-            for (tag, offset) in tags.iter().zip(offsets.clone().into_iter()) {
-                db.put(tag, offset.to_le_bytes())
-                    .map_err(CommitmentError::CommitmentDb)?;
-            }
         }
 
         Ok(())

crates/subspace-farmer/src/commitments/tests.rs

@@ -19,11 +19,11 @@ async fn create() {
     let salt: Salt = [1u8; 8];
     let correct_tag: Tag = [23, 245, 162, 52, 107, 135, 192, 210];
     let solution_range = u64::from_be_bytes([0xff_u8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
-    let index = 0;
 
     let plot = Plot::open_or_create(&base_directory, [0; 32].into(), None).unwrap();
     let commitments = Commitments::new(base_directory.path().join("commitments")).unwrap();
-    plot.write_many(Arc::new(pieces), index).unwrap();
+    let piece_indexes = (0..).take(pieces.count()).collect();
+    plot.write_many(Arc::new(pieces), piece_indexes).unwrap();
     commitments.create(salt, plot).unwrap();
 
     let (tag, _) = commitments
@@ -47,7 +47,8 @@ async fn find_by_tag() {
     let mut rng = StdRng::seed_from_u64(0);
     let mut pieces: FlatPieces = vec![0u8; 1024 * PIECE_SIZE].try_into().unwrap();
     rng.fill(pieces.as_mut());
-    plot.write_many(Arc::new(pieces), 0).unwrap();
+    let piece_indexes = (0..).take(pieces.count()).collect();
+    plot.write_many(Arc::new(pieces), piece_indexes).unwrap();
 
     commitments.create(salt, plot).unwrap();
 
@@ -120,19 +121,19 @@ async fn remove_commitments() {
     let salt: Salt = [1u8; 8];
     let correct_tag: Tag = [23, 245, 162, 52, 107, 135, 192, 210];
     let solution_range = u64::from_be_bytes([0xff_u8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
-    let index = 0;
 
     let plot = Plot::open_or_create(&base_directory, [0; 32].into(), None).unwrap();
     let commitments = Commitments::new(base_directory.path().join("commitments")).unwrap();
-    plot.write_many(Arc::new(pieces), index).unwrap();
+    let piece_indexes = (0..).take(pieces.count()).collect();
+    plot.write_many(Arc::new(pieces), piece_indexes).unwrap();
     commitments.create(salt, plot.clone()).unwrap();
 
     let (_, offset) = commitments
         .find_by_range(correct_tag, solution_range, salt)
         .unwrap();
 
     commitments
-        .remove_pieces(vec![plot.read_piece_with_index(offset).unwrap().0])
+        .remove_pieces(&[plot.read_piece_with_index(offset).unwrap().0])
         .unwrap();
 
     assert!(commitments

crates/subspace-farmer/src/farming/tests.rs

@@ -25,14 +25,14 @@ async fn farming_simulator(slots: Vec<SlotInfo>, tags: Vec<Tag>) {
 
     let pieces: FlatPieces = vec![9u8; 4096].try_into().unwrap();
     let salt: Salt = slots[0].salt; // the first slots salt should be used for the initial commitments
-    let index = 0;
 
     let address = identity.public_key().to_bytes().into();
     let plot = Plot::open_or_create(&base_directory, address, None).unwrap();
 
     let commitments = Commitments::new(base_directory.path().join("commitments")).unwrap();
 
-    plot.write_many(Arc::new(pieces), index).unwrap();
+    let piece_indexes = (0..).take(pieces.count()).collect();
+    plot.write_many(Arc::new(pieces), piece_indexes).unwrap();
     commitments.create(salt, plot.clone()).unwrap();
 
     let client = MockRpc::new();