diff --git a/.github/workflows/tests_and_checks.yml b/.github/workflows/tests_and_checks.yml index a363957..5787115 100644 --- a/.github/workflows/tests_and_checks.yml +++ b/.github/workflows/tests_and_checks.yml @@ -19,7 +19,7 @@ jobs: matrix: rust-toolchain: - stable - - nightly + # - nightly steps: - name: Checkout Repository uses: actions/checkout@v4 diff --git a/Cargo.lock b/Cargo.lock index cf20abc..6cab3c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,6 +207,15 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -317,6 +326,25 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.20.10" @@ -383,6 +411,16 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -560,6 +598,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -634,6 +682,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "home" version = "0.5.9" @@ -949,6 +1003,7 @@ dependencies = [ "dotenvy", "futures", "getset", + "hex", "home", "lazy_static", "oci-spec", @@ -957,6 +1012,7 @@ dependencies = [ "reqwest-retry", "serde", "serde_json", + "sha2", "structstruck", "test-log", "thiserror", @@ -1581,6 +1637,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2019,6 +2086,12 @@ dependencies = [ "syn", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.17" @@ -2085,6 +2158,12 @@ dependencies = [ "quote", ] +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" diff --git a/monocore/Cargo.toml b/monocore/Cargo.toml index 941d311..26dd516 100644 --- a/monocore/Cargo.toml +++ b/monocore/Cargo.toml @@ -30,6 +30,7 @@ clap.workspace = true dotenvy.workspace = true futures.workspace = true getset.workspace = true +hex = "0.4.3" home = "0.5.9" lazy_static = "1.5.0" oci-spec = "0.7.0" @@ -38,6 +39,7 @@ reqwest-middleware = "0.3.3" reqwest-retry = "0.6.1" serde.workspace = true serde_json = "1.0.128" +sha2 = "0.10.8" structstruck = "0.4.1" thiserror.workspace = true tokio.workspace = true diff --git a/monocore/lib/error.rs b/monocore/lib/error.rs index 866313a..ebed124 100644 --- a/monocore/lib/error.rs +++ b/monocore/lib/error.rs @@ -44,6 +44,14 @@ pub enum MonocoreError { /// An error that occurred when a join handle returned an error. #[error("join error: {0}")] JoinError(#[from] tokio::task::JoinError), + + /// An error that occurred when an unsupported image hash algorithm was used. + #[error("unsupported image hash algorithm: {0}")] + UnsupportedImageHashAlgorithm(String), + + /// An error that occurred when an image layer download failed. + #[error("image layer download failed: {0}")] + ImageLayerDownloadFailed(String), } /// An error that can represent any error. diff --git a/monocore/lib/oci/distribution/docker.rs b/monocore/lib/oci/distribution/docker.rs index e31510c..752bc1c 100644 --- a/monocore/lib/oci/distribution/docker.rs +++ b/monocore/lib/oci/distribution/docker.rs @@ -19,7 +19,7 @@ use tokio::{ }; use crate::{ - utils::{self, IMAGE_LAYERS_SUBDIR, MONOCORE_PATH}, + utils::{self, MONOCORE_IMAGE_LAYERS_PATH, MONOCORE_PATH}, MonocoreError, MonocoreResult, }; @@ -74,7 +74,7 @@ pub struct DockerRegistry { client: ClientWithMiddleware, /// The path to the where files are downloaded. - download_path: PathBuf, + path: PathBuf, } /// Stores authentication credentials obtained from the Docker registry, including tokens and expiration details. @@ -130,22 +130,19 @@ impl DockerRegistry { Self { client, - download_path: MONOCORE_PATH.join(REGISTRY_PATH), + path: MONOCORE_PATH.join(REGISTRY_PATH), } } /// Creates a new DockerRegistry instance with a custom path. - pub fn with_download_path(download_path: PathBuf) -> Self { + pub fn with_path(path: PathBuf) -> Self { let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); let client_builder = ClientBuilder::new(Client::new()); let client = client_builder .with(RetryTransientMiddleware::new_with_policy(retry_policy)) .build(); - Self { - client, - download_path, - } + Self { client, path } } /// Gets the size of a downloaded file if it exists. @@ -158,12 +155,12 @@ impl DockerRegistry { path.metadata().unwrap().len() } - // TODO: Can this accept references instead. /// Downloads a blob from the registry, supports download resumption if the file already partially exists. async fn download_image_blob( &self, - repository: String, - digest: Digest, + repository: &str, + digest: &Digest, + download_size: u64, destination: PathBuf, ) -> MonocoreResult<()> { // Ensure the destination directory exists @@ -174,23 +171,28 @@ impl DockerRegistry { // Get the size of the already downloaded file if it exists let downloaded_size = self.get_downloaded_file_size(&destination); - // Create the request with a range header for resumption - let mut stream = self - .fetch_image_blob(&repository, &digest, downloaded_size..) - .await?; - // Open the file for writing, create if it doesn't exist - let mut file = if downloaded_size > 0 { - OpenOptions::new().append(true).open(&destination).await? - } else { + let mut file = if downloaded_size == 0 { OpenOptions::new() .create(true) .truncate(true) .write(true) .open(&destination) .await? + } else if downloaded_size < download_size { + OpenOptions::new().append(true).open(&destination).await? + } else { + tracing::info!( + "file already exists skipping download: {}", + destination.display() + ); + return Ok(()); }; + let mut stream = self + .fetch_image_blob(repository, digest, downloaded_size..) + .await?; + // Write the stream to the file while let Some(chunk) = stream.next().await { let bytes = chunk?; @@ -199,18 +201,17 @@ impl DockerRegistry { // TODO: Check that the downloaded file has the same digest as the one we wanted // TODO: Use the hash method derived from the digest to verify the download - // let algorithm = digest.algorithm(); - // let expected_hash = digest.to_string(); - // let actual_hash = utils::get_file_hash(destination, algorithm).await?; - - // if actual_hash != expected_hash { - // // Delete the file if the hash does not match - // fs::remove_file(destination).await?; - // return Err(MonocoreError::DownloadFailed(format!( - // "Downloaded file hash {} does not match expected hash {}", - // actual_hash, expected_hash - // ))); - // } + let algorithm = digest.algorithm(); + let expected_hash = digest.digest(); + let actual_hash = hex::encode(utils::get_file_hash(&destination, algorithm).await?); + + // Delete the already downloaded file if the hash does not match + if actual_hash != expected_hash { + fs::remove_file(destination).await?; + return Err(MonocoreError::ImageLayerDownloadFailed(format!( + "({repository}:{digest}) file hash {actual_hash} does not match expected hash {expected_hash}", + ))); + } Ok(()) } @@ -278,13 +279,8 @@ impl OciRegistryPull for DockerRegistry { .layers() .iter() .map(|layer| { - let layer_path = self - .download_path - .join(IMAGE_LAYERS_SUBDIR) - .join(layer.digest().to_string()); - - // TODO: Can this accept references instead. - self.download_image_blob(repository.to_string(), layer.digest().clone(), layer_path) + let layer_path = MONOCORE_IMAGE_LAYERS_PATH.join(layer.digest().to_string()); + self.download_image_blob(repository, layer.digest(), layer.size(), layer_path) }) .collect(); @@ -531,7 +527,9 @@ mod tests { async fn test_pull_image() -> anyhow::Result<()> { let registry = DockerRegistry::new(); - registry.pull_image("library/alpine", None).await?; + let result = registry.pull_image("library/alpine", None).await; + + assert!(result.is_ok()); Ok(()) } diff --git a/monocore/lib/utils/file.rs b/monocore/lib/utils/file.rs new file mode 100644 index 0000000..cd718bc --- /dev/null +++ b/monocore/lib/utils/file.rs @@ -0,0 +1,32 @@ +use std::path::Path; + +use oci_spec::image::DigestAlgorithm; +use sha2::{Digest, Sha256, Sha384, Sha512}; +use tokio::{fs::File, io::AsyncReadExt}; + +use crate::{MonocoreError, MonocoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Gets the hash of a file. +pub async fn get_file_hash(path: &Path, algorithm: &DigestAlgorithm) -> MonocoreResult> { + let mut file = File::open(path).await?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).await?; + + let hash = match algorithm { + DigestAlgorithm::Sha256 => Sha256::digest(&buffer).to_vec(), + DigestAlgorithm::Sha384 => Sha384::digest(&buffer).to_vec(), + DigestAlgorithm::Sha512 => Sha512::digest(&buffer).to_vec(), + _ => { + return Err(MonocoreError::UnsupportedImageHashAlgorithm(format!( + "Unsupported algorithm: {}", + algorithm + ))); + } + }; + + Ok(hash) +} diff --git a/monocore/lib/utils/mod.rs b/monocore/lib/utils/mod.rs index d1dec88..a25c510 100644 --- a/monocore/lib/utils/mod.rs +++ b/monocore/lib/utils/mod.rs @@ -1,6 +1,7 @@ //! Utility functions and types. mod conversion; +mod file; mod path; //-------------------------------------------------------------------------------------------------- @@ -8,4 +9,5 @@ mod path; //-------------------------------------------------------------------------------------------------- pub use conversion::*; +pub use file::*; pub use path::*; diff --git a/monocore/lib/utils/path.rs b/monocore/lib/utils/path.rs index 5fb3886..22d1a74 100644 --- a/monocore/lib/utils/path.rs +++ b/monocore/lib/utils/path.rs @@ -10,12 +10,15 @@ use home::home_dir; pub const MONOCORE_SUBDIR: &str = ".monocore"; /// The sub directory where monocore OCI image layers are cached. -pub const IMAGE_LAYERS_SUBDIR: &str = "image_layers"; +pub const IMAGE_LAYERS_SUBDIR: &str = "layers"; /// The sub directory where monocore OCI image index configurations are cached. -pub const IMAGE_DEFS_SUBDIR: &str = "image_defs"; +pub const IMAGE_DESCRIPTION_SUBDIR: &str = "descriptions"; lazy_static::lazy_static! { /// The path where all monocore artifacts, configs, etc are stored. pub static ref MONOCORE_PATH: PathBuf = home_dir().unwrap().join(MONOCORE_SUBDIR); + + /// The path where all monocore OCI image layers are cached. + pub static ref MONOCORE_IMAGE_LAYERS_PATH: PathBuf = MONOCORE_PATH.join(IMAGE_LAYERS_SUBDIR); }