prune implemented

src/cmd.rs

@@ -23,15 +23,19 @@ pub enum Cmd {
     /// Verifies the downloaded mirror(s) against the mirror configuration and outputs a report
     Verify,
     /// Removes unreferenced files in the downloaded mirror(s)  
-    Prune
+    Prune { 
+        #[clap(short, long, help = "Prints the files that the prune operation would delete")]
+        dry_run: bool 
+    }
 }
 
+
 impl Display for Cmd {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Cmd::Mirror => f.write_str("Mirroring"),
             Cmd::Verify => f.write_str("Verifying"),
-            Cmd::Prune => f.write_str("Pruning"),
+            Cmd::Prune { .. } => f.write_str("Pruning"),
         }
     }
 }
@@ -43,11 +47,11 @@ impl Cmd {
                 let ctxs = Context::<MirrorState>::create(opts, cli_opts, pgp_key_store)?;
                 self.run_all(ctxs).await;
             },
-            Cmd::Prune => {
-                let ctxs = Context::<PruneState>::create(opts, cli_opts)?;
+            Cmd::Prune { dry_run } => {
+                let ctxs = Context::<PruneState>::create(opts, cli_opts, dry_run)?;
                 self.run_all(ctxs).await;
             },
-            Cmd::Verify => todo!(),
+            Cmd::Verify => unimplemented!(),
         }
 
         Ok(())

src/error.rs

@@ -100,6 +100,9 @@ pub enum MirsError {
     #[error("error reading {path}: {inner}")]
     ReadingPackage { path: FilePath, inner: Box<MirsError> },
 
+    #[error("error reading path: {inner}")]
+    WalkDir { #[from]inner: walkdir::Error },
+
     #[error("PGP error: {inner}")]
     Pgp { #[from] inner: pgp::errors::Error },
 

src/metadata.rs

@@ -1,4 +1,4 @@
-use std::{fmt::Display, fs::Metadata, io::{BufRead, BufReader, Read}, path::Path, str::FromStr, sync::{atomic::{AtomicU64, Ordering}, Arc}};
+use std::{borrow::Borrow, fmt::Display, fs::Metadata, io::{BufRead, BufReader, Read}, path::Path, str::FromStr, sync::{atomic::{AtomicU64, Ordering}, Arc}};
 
 use compact_str::{format_compact, CompactString, ToCompactString};
 use metadata_file::MetadataFile;
@@ -19,6 +19,12 @@ pub mod metadata_file;
 #[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Default, Hash)]
 pub struct FilePath(pub CompactString);
 
+impl Borrow<str> for FilePath {
+    fn borrow(&self) -> &str {
+        self.as_str()
+    }
+}
+
 impl FromStr for FilePath {
     type Err = MirsError;
 

src/metadata/repository.rs

@@ -109,6 +109,17 @@ impl Repository {
             .map_err(MirsError::from)
     }
 
+    pub fn strip_root<'a>(&'a self, path: &'a str) -> &'a str {
+        let Some(path) = path.strip_prefix(self.root_dir.as_str()) else {
+            return path
+        };
+
+        match path.strip_prefix('/') {
+            Some(p) => p,
+            None => path
+        }
+    }
+
     pub fn rel_from_tmp<'a>(&self, path: &'a str) -> &'a str {
         path.strip_prefix(self.tmp_dir.as_str())
             .expect("input path should be in tmp dir")

src/progress.rs

@@ -97,6 +97,23 @@ impl Progress {
             .with_prefix(prefix)
     }
 
+    pub async fn create_unbounded_progress_bar(&self) -> ProgressBar {
+        let prefix = self.create_prefix().await;
+
+        ProgressBar::new(self.files.total())
+            .with_style(
+                ProgressStyle::default_bar()
+                    .template(
+                        "{prefix} [{elapsed_precise}] {pos}/{len} [{msg}]",
+                    )
+                    .expect("template string should follow the syntax")
+                    .progress_chars("###"),
+
+            )
+            .with_finish(ProgressFinish::AndLeave)
+            .with_prefix(prefix)
+    }
+
     pub async fn create_count_progress_bar(&self) -> ProgressBar {
         let prefix = self.create_prefix().await;
 
@@ -206,6 +223,10 @@ impl ProgressPart {
         self.success.load(Ordering::SeqCst)
     }
 
+    pub fn skipped(&self) -> u64 {
+        self.skipped.load(Ordering::SeqCst)
+    }
+
     pub fn remaining(&self) -> u64 {
         self.total.load(Ordering::SeqCst) -
             self.success.load(Ordering::SeqCst) -

src/prune.rs

@@ -3,6 +3,7 @@ use std::{collections::BTreeMap, fmt::Display, sync::Arc};
 use ahash::HashSet;
 use async_trait::async_trait;
 use compact_str::CompactString;
+use delete::Delete;
 use indicatif::HumanBytes;
 use inventory::Inventory;
 use thiserror::Error;
@@ -19,8 +20,8 @@ pub type PruneContext = Arc<Context<PruneState>>;
 
 #[derive(Error, Debug)]
 pub enum PruneResult { 
-    #[error("Ok: valid files found: {valid_files}, pruned {total_files} files, total: {}", HumanBytes(*.total_bytes))]
-    Pruned { valid_files: u64, total_files: u64, total_bytes: u64 },
+    #[error("Ok: valid {valid_files} ({}), pruned {deleted_files} ({})", HumanBytes(*.valid_bytes), HumanBytes(*.deleted_bytes))]
+    Pruned { valid_files: u64, valid_bytes: u64, deleted_files: u64, deleted_bytes: u64 },
     #[error("Fail: {0}")]
     Error(MirsError)
 }
@@ -30,6 +31,7 @@ impl CmdResult for PruneResult { }
 pub struct PruneState {
     pub mirrors: Vec<(MirrorOpts, Arc<Repository>)>,
     pub output: Arc<Mutex<PruneOutput>>,
+    pub dry_run: bool,
 }
 
 impl Display for PruneState {
@@ -59,9 +61,10 @@ impl Display for PruneState {
 #[derive(Default)]
 pub struct PruneOutput {
     pub files: HashSet<FilePath>, 
-    pub total_valid_files: u64,
-    pub total_files_deleted: u64,
-    pub total_bytes_deleted: u64,
+    pub total_valid: u64,
+    pub total_valid_bytes: u64,
+    pub total_deleted: u64,
+    pub total_deleted_bytes: u64,
 }
 
 #[async_trait]
@@ -72,9 +75,10 @@ impl CmdState for PruneState {
         let output = self.output.lock().await;
 
         PruneResult::Pruned {
-            valid_files: output.total_valid_files,
-            total_files: output.total_files_deleted,
-            total_bytes: output.total_bytes_deleted
+            valid_files: output.total_valid,
+            valid_bytes: output.total_valid_bytes,
+            deleted_files: output.total_deleted,
+            deleted_bytes: output.total_deleted_bytes
         }
     }
 
@@ -87,10 +91,11 @@ impl Context<PruneState> {
     fn create_steps() -> Vec<PruneDynStep> {
         vec![
             Box::new(Inventory),
+            Box::new(Delete),
         ]
     }
 
-    pub fn create(opts: Vec<MirrorOpts>, cli_opts: Arc<CliOpts>) -> Result<Vec<(PruneContext, Vec<PruneDynStep>)>> {
+    pub fn create(opts: Vec<MirrorOpts>, cli_opts: Arc<CliOpts>, dry_run: bool) -> Result<Vec<(PruneContext, Vec<PruneDynStep>)>> {
         let mut mirrors: BTreeMap<CompactString, Vec<(MirrorOpts, Arc<Repository>)>> = BTreeMap::new();
 
         for opt in opts {
@@ -105,7 +110,7 @@ impl Context<PruneState> {
 
         let ctxs: Vec<(PruneContext, Vec<PruneDynStep>)> = mirrors.into_values()
             .map(|mirrors| {
-                (Context::build(PruneState { mirrors, output: Default::default() }, cli_opts.clone(), Progress::new()), Self::create_steps())
+                (Context::build(PruneState { mirrors, output: Default::default(), dry_run }, cli_opts.clone(), Progress::new()), Self::create_steps())
             })
             .collect();
 

src/prune/delete.rs

@@ -1,17 +1,18 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
+use tokio::fs::remove_file;
+use walkdir::WalkDir;
 
 use crate::{context::Context, error::MirsError, step::{Step, StepResult}};
 use crate::error::Result;
 
 use super::{PruneResult, PruneState};
 
-
-pub struct Inventory;
+pub struct Delete;
 
 #[async_trait]
-impl Step<PruneState> for Inventory {
+impl Step<PruneState> for Delete {
     type Result = PruneResult;
 
     fn step_name(&self) -> &'static str {
@@ -23,6 +24,49 @@ impl Step<PruneState> for Inventory {
     }
 
     async fn execute(&self, ctx: Arc<Context<PruneState>>) -> Result<StepResult<Self::Result>> {
-        todo!()
+        let (_, repo) = ctx.state.mirrors.first().expect("there should be a mirror on prune");
+
+        let mut progress_bar = ctx.progress.create_unbounded_progress_bar().await;
+
+        let mut output = ctx.state.output.lock().await;
+
+        for entry in WalkDir::new(&repo.root_dir) {
+            let entry = entry?;
+
+            if entry.file_type().is_dir() {
+                continue
+            }
+
+            let path = repo.strip_root(entry.path().as_os_str().to_str().expect("path should be utf8"));
+
+            let size = entry.metadata()?.len();
+
+            ctx.progress.files.inc_total(1);
+
+            if output.files.contains(path) {
+                ctx.progress.files.inc_skipped(1);
+                ctx.progress.bytes.inc_skipped(size);
+            } else {
+                ctx.progress.files.inc_success(1);
+                ctx.progress.bytes.inc_success(size);
+
+                if ctx.state.dry_run {
+                    eprintln!("{path}");
+                } else {
+                    remove_file(repo.root_dir.join(path)).await?;
+                }
+            }
+
+            ctx.progress.update_for_files(&mut progress_bar);
+        }
+
+        progress_bar.finish_using_style();
+
+        output.total_valid = ctx.progress.files.skipped();
+        output.total_valid_bytes = ctx.progress.bytes.skipped();
+        output.total_deleted = ctx.progress.files.success();
+        output.total_deleted_bytes = ctx.progress.bytes.success();
+
+        Ok(StepResult::Continue)
     }
 }

src/prune/inventory.rs

@@ -4,7 +4,7 @@ use ahash::HashSet;
 use async_trait::async_trait;
 use compact_str::format_compact;
 
-use crate::{context::Context, error::MirsError, metadata::{metadata_file::{deduplicate_metadata, MetadataFile}, release::{FileEntry, Release}, FilePath}, mirror::verify_and_prune, progress::Progress, step::{Step, StepResult}};
+use crate::{context::Context, error::MirsError, metadata::{metadata_file::{deduplicate_metadata, MetadataFile}, release::{FileEntry, Release}, repository::Repository, FilePath}, mirror::verify_and_prune, progress::Progress, step::{Step, StepResult}};
 use crate::error::Result;
 
 use super::{PruneResult, PruneState};
@@ -47,18 +47,18 @@ impl Step<PruneState> for Inventory {
             let mut metadata: Vec<(MetadataFile, FileEntry)> = release.into_filtered_files(opts).collect();
 
             for f in release_files {
-                add_valid_metadata_file(&mut progress, &mut state.files, &f, repo.root_dir.as_str());
+                add_valid_metadata_file(&mut progress, &mut state.files, &f, repo);
             }
 
             for (metadata_file, file_entry) in &mut metadata {
                 metadata_file.prefix_with(dist_root.as_str());
 
                 let (_, primary, other) = file_entry.into_paths(metadata_file.path(), by_hash)?;
 
-                add_valid_metadata_file(&mut progress, &mut state.files, &primary, repo.root_dir.as_str());
+                add_valid_metadata_file(&mut progress, &mut state.files, &primary, repo);
 
                 for f in other {
-                    add_valid_metadata_file(&mut progress, &mut state.files, &f, repo.root_dir.as_str());
+                    add_valid_metadata_file(&mut progress, &mut state.files, &f, repo);
                 }
             }
 
@@ -87,12 +87,7 @@ impl Step<PruneState> for Inventory {
                     MetadataFile::Sources(..) => FilePath::from(""),
                     MetadataFile::SumFile(file_path) |
                     MetadataFile::DiffIndex(file_path) => {
-                        FilePath::from(
-                            file_path.parent()
-                                .expect("diff indicies should have parents")
-                                .strip_prefix(repo.root_dir.as_str())
-                                .expect("metadata paths should be rooted")
-                        )
+                        FilePath::from(repo.strip_root(file_path.parent().expect("diff indicies should have parents")))
                     },
                     MetadataFile::Other(..) => unreachable!()
                 };
@@ -112,17 +107,15 @@ impl Step<PruneState> for Inventory {
                 incremental_size_base += meta_file_size;
             }
         }
-
-        state.total_valid_files += state.files.len() as u64;
 
         progress_bar.finish_using_style();
 
         Ok(StepResult::Continue)
     }
 }
 
-fn add_valid_metadata_file(progress: &mut Progress, files: &mut HashSet<FilePath>, file: &FilePath, root_dir: &str) {
-    let path = file.as_str().strip_prefix(root_dir).expect("path is in root");
+fn add_valid_metadata_file(progress: &mut Progress, files: &mut HashSet<FilePath>, file: &FilePath, repo: &Repository) {
+    let path = repo.strip_root(file.as_str());
 
     add_valid_file(progress, files, path.into());
 }