From b8baf16416009410bb0e3ec29c7d36f804386ca1 Mon Sep 17 00:00:00 2001 From: wcampbell Date: Wed, 19 Jul 2023 19:47:03 -0400 Subject: [PATCH 1/6] Update bench.bash --- bench.bash | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bench.bash b/bench.bash index c1ca9063..47305132 100755 --- a/bench.bash +++ b/bench.bash @@ -3,22 +3,25 @@ BACKHAND="./target/release/unsquashfs" BACKHAND_MSRV="./target-msrv/release/unsquashfs" BACKHAND_MUSL="./target/x86_64-unknown-linux-musl/release/unsquashfs" +BACKHAND_MUSL_NATIVE="./target-native/x86_64-unknown-linux-musl/release/unsquashfs" UNSQUASHFS="/usr/bin/unsquashfs" bench () { file $1 hyperfine --runs 20 --warmup 5 -i "$BACKHAND -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \ "$BACKHAND_MUSL -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \ + "$BACKHAND_MUSL_NATIVE -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \ "$BACKHAND_MSRV -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \ - "$UNSQUASHFS -d $(mktemp -d /tmp/BHXXX) -p 1 -f -o $(rz-ax $2) -ignore-errors $1" \ - "$UNSQUASHFS -d $(mktemp -d /tmp/BHXXX) -f -o $(rz-ax $2) -ignore-errors $1" + "$UNSQUASHFS -quiet -no-progress -d $(mktemp -d /tmp/BHXXX) -p 1 -f -o $(rz-ax $2) -ignore-errors $1" \ + "$UNSQUASHFS -quiet -no-progress -d $(mktemp -d /tmp/BHXXX) -f -o $(rz-ax $2) -ignore-errors $1" } # install msrv (make sure no perf regressions) -rustup toolchain install 1.64.0 -cargo +1.64.0 build --release --target-dir target-msrv +rustup toolchain install 1.65.0 +cargo +1.65.0 build --release --target-dir target-msrv cargo build --release cargo build --release --target x86_64-unknown-linux-musl +RUSTFLAGS="-C target-cpu=native" cargo build --release --target x86_64-unknown-linux-musl --target-dir target-native # xz bench "test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin" 0x225fd0 From 63d0d83cd03f494d45d9dfd53497d885424f55a7 Mon Sep 17 00:00:00 2001 From: wcampbell Date: Wed, 19 Jul 2023 19:48:50 -0400 Subject: [PATCH 2/6] backhand: Use thread-safe structs --- src/filesystem/node.rs | 4 ++-- src/filesystem/reader.rs | 15 ++++++------ src/filesystem/writer.rs | 17 +++++++------ src/kinds.rs | 52 +++++++++++++++++++++------------------- src/reader.rs | 4 ++-- src/squashfs.rs | 10 ++++---- 6 files changed, 54 insertions(+), 48 deletions(-) diff --git a/src/filesystem/node.rs b/src/filesystem/node.rs index c2c9b9f2..47b4b412 100644 --- a/src/filesystem/node.rs +++ b/src/filesystem/node.rs @@ -1,8 +1,8 @@ use core::fmt; -use std::cell::RefCell; use std::io::Read; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use super::normalize_squashfs_path; use crate::data::Added; @@ -106,7 +106,7 @@ pub struct SquashfsFileReader { /// Read file from other SquashfsFile or an user file pub enum SquashfsFileWriter<'a> { - UserDefined(RefCell>), + UserDefined(Arc>), SquashfsFile(FilesystemReaderFile<'a>), Consumed(usize, Added), } diff --git a/src/filesystem/reader.rs b/src/filesystem/reader.rs index 02d1e86e..250b9c68 100644 --- a/src/filesystem/reader.rs +++ b/src/filesystem/reader.rs @@ -1,5 +1,5 @@ -use std::cell::RefCell; use std::io::{Read, SeekFrom}; +use std::sync::Mutex; use super::node::Nodes; use crate::compressor::{CompressionOptions, Compressor}; @@ -86,9 +86,9 @@ pub struct FilesystemReader { /// All files and directories in filesystem pub root: Nodes, // File reader - pub(crate) reader: RefCell>, + pub(crate) reader: Mutex>, // Cache used in the decompression - pub(crate) cache: RefCell, + pub(crate) cache: Mutex, } impl FilesystemReader { @@ -291,7 +291,7 @@ impl<'a> SquashfsRawData<'a> { data.resize(block_size, 0); //NOTE: storing/restoring the file-pos is not required at the //moment of writing, but in the future, it may. - let mut reader = self.file.system.reader.borrow_mut(); + let mut reader = self.file.system.reader.lock().unwrap(); reader.seek(SeekFrom::Start(self.pos))?; reader.read_exact(data)?; self.pos = reader.stream_position()?; @@ -301,7 +301,7 @@ impl<'a> SquashfsRawData<'a> { }) } BlockFragment::Fragment(fragment) => { - let cache = self.file.system.cache.borrow(); + let cache = self.file.system.cache.lock().unwrap(); if let Some(cache_bytes) = cache.fragment_cache.get(&fragment.start) { //if in cache, just return the cache, don't read it let cache_size = cache_bytes.len(); @@ -316,7 +316,7 @@ impl<'a> SquashfsRawData<'a> { //otherwise read and return it let frag_size = fragment.size.size() as usize; data.resize(frag_size, 0); - let mut reader = self.file.system.reader.borrow_mut(); + let mut reader = self.file.system.reader.lock().unwrap(); reader.seek(SeekFrom::Start(fragment.start))?; reader.read_exact(data)?; Ok(RawDataBlock { @@ -368,7 +368,8 @@ impl<'a> SquashfsRawData<'a> { self.file .system .cache - .borrow_mut() + .lock() + .unwrap() .fragment_cache .insert(self.file.fragment().unwrap().start, output_buf.clone()); } diff --git a/src/filesystem/writer.rs b/src/filesystem/writer.rs index 815cc3b3..d48f3afc 100644 --- a/src/filesystem/writer.rs +++ b/src/filesystem/writer.rs @@ -1,9 +1,9 @@ -use std::cell::RefCell; use std::ffi::OsStr; use std::io::{Read, Seek, SeekFrom, Write}; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; -use std::rc::Rc; +use std::sync::Arc; +use std::sync::Mutex; use std::time::{SystemTime, UNIX_EPOCH}; use deku::bitvec::BitVec; @@ -94,7 +94,7 @@ impl<'a> Default for FilesystemWriter<'a> { id_table: Id::root(), fs_compressor: FilesystemCompressor::default(), kind: Kind { - inner: Rc::new(LE_V4_0), + inner: Arc::new(LE_V4_0), }, root: Nodes::new_root(NodeHeader::default()), block_log: (block_size as f32).log2() as u16, @@ -277,7 +277,7 @@ impl<'a> FilesystemWriter<'a> { path: P, header: NodeHeader, ) -> Result<(), BackhandError> { - let reader = RefCell::new(Box::new(reader)); + let reader = Arc::new(Mutex::new(reader)); let new_file = InnerNode::File(SquashfsFileWriter::UserDefined(reader)); self.insert_node(path, header, new_file)?; Ok(()) @@ -306,7 +306,8 @@ impl<'a> FilesystemWriter<'a> { let file = self .mut_file(find_path) .ok_or(BackhandError::FileNotFound)?; - *file = SquashfsFileWriter::UserDefined(RefCell::new(Box::new(reader))); + let reader = Arc::new(Mutex::new(reader)); + *file = SquashfsFileWriter::UserDefined(reader); Ok(()) } @@ -437,9 +438,11 @@ impl<'a> FilesystemWriter<'a> { _ => None, }); for file in files { - let (filesize, added) = match &file { + let (filesize, added) = match file { SquashfsFileWriter::UserDefined(file) => { - data_writer.add_bytes(file.borrow_mut().as_mut(), writer)? + let file_ptr = Arc::clone(file); + let mut file_lock = file_ptr.lock().unwrap(); + data_writer.add_bytes(&mut *file_lock, writer)? } SquashfsFileWriter::SquashfsFile(file) => { // if the source file and the destination files are both diff --git a/src/kinds.rs b/src/kinds.rs index 75538205..ea2675e3 100644 --- a/src/kinds.rs +++ b/src/kinds.rs @@ -1,7 +1,7 @@ //! Types of image formats use core::fmt; -use std::rc::Rc; +use std::sync::Arc; use crate::compressor::{CompressionAction, DefaultCompressor}; @@ -31,7 +31,7 @@ pub enum Endian { Big, } -pub struct InnerKind { +pub struct InnerKind { /// Magic at the beginning of the image pub(crate) magic: [u8; 4], /// Endian used for all data types @@ -51,7 +51,7 @@ pub struct InnerKind { /// See [Kind Constants](`crate::kind#constants`) for a list of custom Kinds pub struct Kind { /// "Easier for the eyes" type for the real Kind - pub(crate) inner: Rc>, + pub(crate) inner: Arc>, } impl fmt::Debug for Kind { @@ -115,21 +115,21 @@ impl Kind { /// /// let kind = Kind::new(&CustomCompressor); /// ``` - pub fn new(compressor: &'static C) -> Self { + pub fn new(compressor: &'static C) -> Self { Self { - inner: Rc::new(InnerKind { + inner: Arc::new(InnerKind { compressor, ..LE_V4_0 }), } } - pub fn new_with_const( + pub fn new_with_const( compressor: &'static C, - c: InnerKind, + c: InnerKind, ) -> Self { Self { - inner: Rc::new(InnerKind { compressor, ..c }), + inner: Arc::new(InnerKind { compressor, ..c }), } } @@ -154,7 +154,7 @@ impl Kind { }; Ok(Kind { - inner: Rc::new(kind), + inner: Arc::new(kind), }) } @@ -167,9 +167,11 @@ impl Kind { /// # use backhand::{kind, kind::Kind}; /// let kind = Kind::from_const(kind::LE_V4_0).unwrap(); /// ``` - pub fn from_const(inner: InnerKind) -> Result { + pub fn from_const( + inner: InnerKind, + ) -> Result { Ok(Kind { - inner: Rc::new(inner), + inner: Arc::new(inner), }) } @@ -183,7 +185,7 @@ impl Kind { /// Set magic type at the beginning of the image // TODO: example pub fn with_magic(mut self, magic: Magic) -> Self { - Rc::get_mut(&mut self.inner).unwrap().magic = magic.magic(); + Arc::get_mut(&mut self.inner).unwrap().magic = magic.magic(); self } @@ -196,10 +198,10 @@ impl Kind { pub fn with_type_endian(mut self, endian: Endian) -> Self { match endian { Endian::Little => { - Rc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Little; + Arc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Little; } Endian::Big => { - Rc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Big; + Arc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Big; } } self @@ -210,10 +212,10 @@ impl Kind { pub fn with_data_endian(mut self, endian: Endian) -> Self { match endian { Endian::Little => { - Rc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Little; + Arc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Little; } Endian::Big => { - Rc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Big; + Arc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Big; } } self @@ -224,12 +226,12 @@ impl Kind { pub fn with_all_endian(mut self, endian: Endian) -> Self { match endian { Endian::Little => { - Rc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Little; - Rc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Little; + Arc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Little; + Arc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Little; } Endian::Big => { - Rc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Big; - Rc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Big; + Arc::get_mut(&mut self.inner).unwrap().type_endian = deku::ctx::Endian::Big; + Arc::get_mut(&mut self.inner).unwrap().data_endian = deku::ctx::Endian::Big; } } self @@ -238,14 +240,14 @@ impl Kind { /// Set major and minor version // TODO: example pub fn with_version(mut self, major: u16, minor: u16) -> Self { - Rc::get_mut(&mut self.inner).unwrap().version_major = major; - Rc::get_mut(&mut self.inner).unwrap().version_minor = minor; + Arc::get_mut(&mut self.inner).unwrap().version_major = major; + Arc::get_mut(&mut self.inner).unwrap().version_minor = minor; self } } /// Default `Kind` for linux kernel and squashfs-tools/mksquashfs. Little-Endian v4.0 -pub const LE_V4_0: InnerKind = InnerKind { +pub const LE_V4_0: InnerKind = InnerKind { magic: *b"hsqs", type_endian: deku::ctx::Endian::Little, data_endian: deku::ctx::Endian::Little, @@ -255,7 +257,7 @@ pub const LE_V4_0: InnerKind = InnerKind { }; /// Big-Endian Superblock v4.0 -pub const BE_V4_0: InnerKind = InnerKind { +pub const BE_V4_0: InnerKind = InnerKind { magic: *b"sqsh", type_endian: deku::ctx::Endian::Big, data_endian: deku::ctx::Endian::Big, @@ -265,7 +267,7 @@ pub const BE_V4_0: InnerKind = InnerKind { }; /// AVM Fritz!OS firmware support. Tested with: -pub const AVM_BE_V4_0: InnerKind = InnerKind { +pub const AVM_BE_V4_0: InnerKind = InnerKind { magic: *b"sqsh", type_endian: deku::ctx::Endian::Big, data_endian: deku::ctx::Endian::Little, diff --git a/src/reader.rs b/src/reader.rs index 6c4a2e16..9576d2dc 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -80,8 +80,8 @@ pub trait BufReadRewind: BufRead + SeekRewind {} impl BufReadRewind for T {} /// Pseudo-Trait for BufRead + Seek -pub trait BufReadSeek: BufRead + Seek {} -impl BufReadSeek for T {} +pub trait BufReadSeek: BufRead + Seek + Send {} +impl BufReadSeek for T {} /// Pseudo-Trait for Write + Seek pub trait WriteSeek: Write + Seek {} diff --git a/src/squashfs.rs b/src/squashfs.rs index 99cf33f6..1720b914 100644 --- a/src/squashfs.rs +++ b/src/squashfs.rs @@ -1,11 +1,11 @@ //! Read from on-disk image -use std::cell::RefCell; use std::ffi::OsString; use std::io::{Seek, SeekFrom}; use std::os::unix::prelude::OsStringExt; use std::path::PathBuf; -use std::rc::Rc; +use std::sync::Arc; +use std::sync::Mutex; use deku::bitvec::{BitVec, BitView, Msb0}; use deku::prelude::*; @@ -299,7 +299,7 @@ impl Squashfs { reader, offset, Kind { - inner: Rc::new(LE_V4_0), + inner: Arc::new(LE_V4_0), }, ) } @@ -651,8 +651,8 @@ impl Squashfs { id_table: self.id, fragments: self.fragments, root, - reader: RefCell::new(Box::new(self.file)), - cache: RefCell::new(Cache::default()), + reader: Mutex::new(Box::new(self.file)), + cache: Mutex::new(Cache::default()), }; Ok(filesystem) } From 2ad4d283c16f7d2184a0cae70b80c557abf67cb4 Mon Sep 17 00:00:00 2001 From: wcampbell Date: Wed, 19 Jul 2023 19:49:21 -0400 Subject: [PATCH 3/6] unsquashfs: Extract with multiple threads --- Cargo.lock | 1 + Cargo.toml | 1 + src/bin/unsquashfs.rs | 77 ++++++++++++++++++++++--------------------- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 658c54d6..8f3da3bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,6 +117,7 @@ dependencies = [ "jemallocator", "libc", "nix", + "rayon", "rust-lzo", "rustc-hash", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 6191d6e9..75a25096 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ libc = "0.2.140" clap_complete = "4.2.1" indicatif = "0.17.5" console = "0.15.7" +rayon = "1.7.0" [features] default = ["xz", "gzip", "zstd"] diff --git a/src/bin/unsquashfs.rs b/src/bin/unsquashfs.rs index aadde518..0007319c 100644 --- a/src/bin/unsquashfs.rs +++ b/src/bin/unsquashfs.rs @@ -2,6 +2,7 @@ mod common; use std::fs::{self, File, Permissions}; use std::io::{self, BufReader, Read, Seek, SeekFrom}; +use std::iter::Iterator; use std::os::unix::prelude::{OsStrExt, PermissionsExt}; use std::path::{Component, Path, PathBuf}; use std::process::ExitCode; @@ -21,6 +22,7 @@ use libc::lchown; use nix::libc::geteuid; use nix::sys::stat::{dev_t, mknod, mode_t, umask, utimensat, utimes, Mode, SFlag, UtimensatFlags}; use nix::sys::time::{TimeSpec, TimeVal}; +use rayon::prelude::*; use std::time::{Duration, Instant}; // -musl malloc is slow, use jemalloc @@ -274,13 +276,22 @@ fn main() -> ExitCode { } else { None }; - extract_all(&args, &filesystem, root_process, nodes, n_nodes); + + extract_all( + &args, + &filesystem, + root_process, + nodes + .collect::>>() + .into_par_iter(), + n_nodes, + ); } ExitCode::SUCCESS } -fn list<'a>(nodes: impl std::iter::Iterator>) { +fn list<'a>(nodes: impl Iterator>) { for node in nodes { let path = &node.fullpath; println!("{}", path.display()); @@ -385,18 +396,15 @@ fn set_attributes( } } -fn extract_all<'a>( +fn extract_all<'a, S: ParallelIterator>>( args: &Args, filesystem: &'a FilesystemReader, root_process: bool, - nodes: impl std::iter::Iterator>, + nodes: S, n_nodes: Option, ) { let start = Instant::now(); - // alloc required space for file data readers - let (mut buf_read, mut buf_decompress) = filesystem.alloc_read_buffers(); - let pb = ProgressBar::new(n_nodes.unwrap_or(0) as u64); pb.set_style(ProgressStyle::default_spinner()); pb.set_style( @@ -414,23 +422,27 @@ fn extract_all<'a>( ); pb.set_prefix("Extracting"); pb.inc(1); - for node in nodes { + + nodes.for_each(|node| { let path = &node.fullpath; pb.set_message(path.as_path().to_str().unwrap().to_string()); pb.inc(1); - let path = path.strip_prefix(Component::RootDir).unwrap_or(path); + let filepath = Path::new(&args.dest).join(fullpath); + // create required dirs, we will fix permissions later + let _ = fs::create_dir_all(filepath.parent().unwrap()); + match &node.inner { InnerNode::File(file) => { - // read file - let filepath = Path::new(&args.dest).join(path); + // alloc required space for file data readers + let (mut buf_read, mut buf_decompress) = filesystem.alloc_read_buffers(); // check if file exists if !args.force && filepath.exists() { if !args.quiet { exists(&pb, filepath.to_str().unwrap()); } - continue; + return; } // write to file @@ -450,19 +462,17 @@ fn extract_all<'a>( let line = format!("{} : {e}", filepath.to_str().unwrap()); failed(&pb, &line); } - continue; + return; } } } InnerNode::Symlink(SquashfsSymlink { link }) => { // create symlink let link_display = link.display(); - let filepath = Path::new(&args.dest).join(path); - // check if file exists if !args.force && filepath.exists() { exists(&pb, filepath.to_str().unwrap()); - continue; + return; } match std::os::unix::fs::symlink(link, &filepath) { @@ -478,7 +488,7 @@ fn extract_all<'a>( format!("{}->{link_display} : {e}", filepath.to_str().unwrap()); failed(&pb, &line); } - continue; + return; } } @@ -509,27 +519,21 @@ fn extract_all<'a>( .unwrap(); } InnerNode::Dir(SquashfsDir { .. }) => { - // create dir - let path = Path::new(&args.dest).join(path); - // These permissions are corrected later (user default permissions for now) - if let Err(e) = std::fs::create_dir(&path) { - if !args.quiet { - let line = format!("{} : {e}", path.to_str().unwrap()); - failed(&pb, &line); - } - } else if args.info && !args.quiet { - created(&pb, path.to_str().unwrap()) + // + // don't display error if this was already created, we might have already + // created it in another thread to put down a file + if std::fs::create_dir(&filepath).is_ok() && args.info && !args.quiet { + created(&pb, filepath.to_str().unwrap()) } } InnerNode::CharacterDevice(SquashfsCharacterDevice { device_number }) => { - let path = Path::new(&args.dest).join(path); if root_process { match mknod( - &path, + path, SFlag::S_IFCHR, Mode::from_bits(mode_t::from(node.header.permissions)).unwrap(), - dev_t::from(*device_number), + dev_t::try_from(*device_number).unwrap(), ) { Ok(_) => { if args.info && !args.quiet { @@ -546,7 +550,7 @@ fn extract_all<'a>( ); failed(&pb, &line); } - continue; + return; } } } else { @@ -555,16 +559,15 @@ fn extract_all<'a>( format!("char device {}, are you superuser?", path.to_str().unwrap()); failed(&pb, &line); } - continue; + return; } } InnerNode::BlockDevice(SquashfsBlockDevice { device_number }) => { - let path = Path::new(&args.dest).join(path); match mknod( - &path, + path, SFlag::S_IFBLK, Mode::from_bits(mode_t::from(node.header.permissions)).unwrap(), - dev_t::from(*device_number), + dev_t::try_from(*device_number).unwrap(), ) { Ok(_) => { if args.info && !args.quiet { @@ -577,12 +580,12 @@ fn extract_all<'a>( if args.info && !args.quiet { created(&pb, path.to_str().unwrap()); } - continue; + return; } } } } - } + }); // fixup dir permissions for node in filesystem From 40f08d65bf093206fbda20a0da5cab55e5d92153 Mon Sep 17 00:00:00 2001 From: wcampbell Date: Wed, 19 Jul 2023 23:26:42 -0400 Subject: [PATCH 4/6] unsquashfs: Add MultiProgressBar * Since we are now multithreaded, show all current files being extracted --- src/bin/unsquashfs.rs | 99 +++++++++++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/src/bin/unsquashfs.rs b/src/bin/unsquashfs.rs index 0007319c..a6fc550b 100644 --- a/src/bin/unsquashfs.rs +++ b/src/bin/unsquashfs.rs @@ -1,11 +1,13 @@ #[path = "../../common/common.rs"] mod common; +use std::collections::HashSet; use std::fs::{self, File, Permissions}; use std::io::{self, BufReader, Read, Seek, SeekFrom}; use std::iter::Iterator; use std::os::unix::prelude::{OsStrExt, PermissionsExt}; use std::path::{Component, Path, PathBuf}; use std::process::ExitCode; +use std::sync::Mutex; use backhand::kind::Kind; use backhand::{ @@ -406,27 +408,42 @@ fn extract_all<'a, S: ParallelIterator>>( let start = Instant::now(); let pb = ProgressBar::new(n_nodes.unwrap_or(0) as u64); - pb.set_style(ProgressStyle::default_spinner()); - pb.set_style( - ProgressStyle::with_template( - // note that bar size is fixed unlike cargo which is dynamic - // and also the truncation in cargo uses trailers (`...`) - if Term::stdout().size().1 > 80 { - "{prefix:>16.cyan.bold} [{bar:57}] {pos}/{len} {wide_msg}" - } else { - "{prefix:>16.cyan.bold} [{bar:57}] {pos}/{len}" - }, - ) - .unwrap() - .progress_chars("=> "), - ); - pb.set_prefix("Extracting"); - pb.inc(1); + if !args.quiet { + pb.set_style(ProgressStyle::default_spinner()); + pb.set_style( + ProgressStyle::with_template( + // note that bar size is fixed unlike cargo which is dynamic + // and also the truncation in cargo uses trailers (`...`) + if Term::stdout().size().1 > 80 { + "{prefix:>16.cyan.bold} [{bar:57}] {pos}/{len} {wide_msg}" + } else { + "{prefix:>16.cyan.bold} [{bar:57}] {pos}/{len}" + }, + ) + .unwrap() + .progress_chars("=> "), + ); + pb.set_prefix("Extracting"); + pb.inc(1); + } + + let processing = Mutex::new(HashSet::new()); nodes.for_each(|node| { let path = &node.fullpath; - pb.set_message(path.as_path().to_str().unwrap().to_string()); - pb.inc(1); + let fullpath = path.strip_prefix(Component::RootDir).unwrap_or(path); + let mut p = processing.lock().unwrap(); + p.insert(fullpath.clone()); + if !args.quiet { + pb.set_message( + p.iter() + .map(|a| a.to_path_buf().into_os_string().into_string().unwrap()) + .collect::>() + .join(", "), + ); + pb.inc(1); + } + drop(p); let filepath = Path::new(&args.dest).join(fullpath); // create required dirs, we will fix permissions later @@ -442,6 +459,9 @@ fn extract_all<'a, S: ParallelIterator>>( if !args.quiet { exists(&pb, filepath.to_str().unwrap()); } + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); return; } @@ -462,6 +482,9 @@ fn extract_all<'a, S: ParallelIterator>>( let line = format!("{} : {e}", filepath.to_str().unwrap()); failed(&pb, &line); } + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); return; } } @@ -472,6 +495,9 @@ fn extract_all<'a, S: ParallelIterator>>( // check if file exists if !args.force && filepath.exists() { exists(&pb, filepath.to_str().unwrap()); + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); return; } @@ -488,6 +514,9 @@ fn extract_all<'a, S: ParallelIterator>>( format!("{}->{link_display} : {e}", filepath.to_str().unwrap()); failed(&pb, &line); } + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); return; } } @@ -530,61 +559,75 @@ fn extract_all<'a, S: ParallelIterator>>( InnerNode::CharacterDevice(SquashfsCharacterDevice { device_number }) => { if root_process { match mknod( - path, + &filepath, SFlag::S_IFCHR, Mode::from_bits(mode_t::from(node.header.permissions)).unwrap(), dev_t::try_from(*device_number).unwrap(), ) { Ok(_) => { if args.info && !args.quiet { - created(&pb, path.to_str().unwrap()); + created(&pb, filepath.to_str().unwrap()); } - set_attributes(&pb, args, &path, &node.header, root_process, true); + set_attributes(&pb, args, &filepath, &node.header, root_process, true); } Err(_) => { if !args.quiet { let line = format!( "char device {}, are you superuser?", - path.to_str().unwrap() + filepath.to_str().unwrap() ); failed(&pb, &line); } + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); return; } } } else { if !args.quiet { - let line = - format!("char device {}, are you superuser?", path.to_str().unwrap()); + let line = format!( + "char device {}, are you superuser?", + filepath.to_str().unwrap() + ); failed(&pb, &line); } + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); return; } } InnerNode::BlockDevice(SquashfsBlockDevice { device_number }) => { match mknod( - path, + &filepath, SFlag::S_IFBLK, Mode::from_bits(mode_t::from(node.header.permissions)).unwrap(), dev_t::try_from(*device_number).unwrap(), ) { Ok(_) => { if args.info && !args.quiet { - created(&pb, path.to_str().unwrap()); + created(&pb, filepath.to_str().unwrap()); } - set_attributes(&pb, args, &path, &node.header, root_process, true); + set_attributes(&pb, args, &filepath, &node.header, root_process, true); } Err(_) => { if args.info && !args.quiet { - created(&pb, path.to_str().unwrap()); + created(&pb, filepath.to_str().unwrap()); } + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); return; } } } } + let mut p = processing.lock().unwrap(); + p.remove(fullpath); + drop(p); }); // fixup dir permissions From 1a9b33a6bba4140b33e1b5cc36ec4b13d5edfe02 Mon Sep 17 00:00:00 2001 From: wcampbell Date: Wed, 19 Jul 2023 23:55:39 -0400 Subject: [PATCH 5/6] Update CHANGELOG --- CHANGELOG.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6763e83..d2d7c693 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # Unreleased ## backhand +### Changes +- Following changes were done to allow multi-threaded applications + - Change `RefCell>` into `Arc>` + - Change `RefCell` into `Mutex` + - Change `Rc` into `Arc` + - Change `dyn CompressionAction` to `dyn CompressionAction + Send + Sync` for `Kind` uses + - Change `BufReadSeek: BufRead + Seek {}` to `BufReadSeek: BufRead + Seek + Send {}` + ### Bug Fix - When creating an empty image using `FilesystemWriter::default()`, correctly create the ID table for UID and GID entries. Reported: ([@hwittenborn](https://github.com/hwittenborn)) ([!250](https://github.com/wcampbell0x2a/backhand/issues/275)), Fixed: ([#275](https://github.com/wcampbell0x2a/backhand/pull/275)) - Remove manual `Clone` impl for `FilesystemReaderFile` ([#277](https://github.com/wcampbell0x2a/backhand/pull/277)) @@ -16,8 +24,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `strip` and `LTO` are enabled for release binaries ## unsquashfs -- Add progress bar for a cleaner output when extracting files -- Add `--quiet` for not displaying progress bar and RUST_LOG output +- Add progress bar for a cleaner output when extracting files ([#272](https://github.com/wcampbell0x2a/backhand/pull/272)) +- Add `--quiet` for not displaying progress bar and RUST_LOG output ([#272](https://github.com/wcampbell0x2a/backhand/pull/272)) +- Add multiple threads for extracing files, giving us the same performance in most cases as `squashfs-tools/unsquashfs`! ## ci - Fix libc calls, add testing and release binaries for the following platforms:([#259](https://github.com/wcampbell0x2a/backhand/pull/259)) From e1b1872cf33871b07366fcc298e11f48055a2fba Mon Sep 17 00:00:00 2001 From: wcampbell Date: Thu, 20 Jul 2023 20:54:31 -0400 Subject: [PATCH 6/6] bins: Add after_text for RAYON_NUM_THREADS --- common/common.rs | 12 +++++++++--- src/bin/add.rs | 2 +- src/bin/replace.rs | 2 +- src/bin/unsquashfs.rs | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/common/common.rs b/common/common.rs index 3bb850e9..04f7751b 100644 --- a/common/common.rs +++ b/common/common.rs @@ -1,6 +1,6 @@ // Compiled for every binary, as this is not a workspce. Don't put many functinos in this file -pub fn after_help() -> String { +pub fn after_help(rayon_env: bool) -> String { let mut s = "Decompressors available:\n".to_string(); #[cfg(feature = "gzip")] @@ -15,7 +15,13 @@ pub fn after_help() -> String { #[cfg(feature = "zstd")] s.push_str("\tzstd\n"); - s.push_str("\nEnvironment Variables:\n\t"); - s.push_str(r#"RUST_LOG: See "https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#filtering-events-with-environment-variables""#); + s.push_str("\nEnvironment Variables:\n"); + s.push_str(" RUST_LOG:"); + s.push_str(r#" "https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#filtering-events-with-environment-variables""#); + if rayon_env { + s.push('\n'); + s.push_str(" RAYON_NUM_THREADS:"); + s.push_str(r#" "https://docs.rs/rayon/latest/rayon/struct.ThreadPoolBuilder.html#method.num_threads""#); + } s } diff --git a/src/bin/add.rs b/src/bin/add.rs index f9b16b49..e55d735b 100644 --- a/src/bin/add.rs +++ b/src/bin/add.rs @@ -20,7 +20,7 @@ static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; #[command(author, version, name = "add-backhand", - after_help = after_help(), + after_help = after_help(false), max_term_width = 98, )] struct Args { diff --git a/src/bin/replace.rs b/src/bin/replace.rs index 86554e12..fcc103ab 100644 --- a/src/bin/replace.rs +++ b/src/bin/replace.rs @@ -19,7 +19,7 @@ static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; #[command(author, version, name = "replace-backhand", - after_help = after_help(), + after_help = after_help(false), max_term_width = 98, )] struct Args { diff --git a/src/bin/unsquashfs.rs b/src/bin/unsquashfs.rs index a6fc550b..70e1cce5 100644 --- a/src/bin/unsquashfs.rs +++ b/src/bin/unsquashfs.rs @@ -84,7 +84,7 @@ pub fn failed(pb: &ProgressBar, s: &str) { #[command(author, version, name = "unsquashfs-backhand", - after_help = after_help(), + after_help = after_help(true), max_term_width = 98, )] struct Args {