Skip to content

Commit

Permalink
write zarrs version/link to array metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
LDeakin committed Oct 17, 2023
1 parent 69d3432 commit d8ff25c
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `ArrayBuilder::build()` and `GroupBuilder::build` now accept unsized storage
- **Breaking**: `StoragePartialDecoder` now takes a `ReadableStorage` input
- `sharded_array_write_read` example now prints storage operations and demonstrates retrieving inner chunks directly from a partial decoder
- the zarrs version and a link to the source code is now written to the `_zarrs` attribute in array metadata, this can be disabled with `set_include_zarrs_metadata(false)`

### Fixed
- Bytes codec handling of complex and raw bits data types
Expand Down
37 changes: 36 additions & 1 deletion src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
//! - [`ReadableStorageTraits`] storage can read array data and metadata.
//! - [`WritableStorageTraits`] storage can write array data and metadata, and
//! - both traits are needed to update chunk subsets, such as with [`Array::store_array_subset`].
//!
//! By default, the `zarrs` version and a link to its source code is written to the `_zarrs` attribute in array metadata.
//! This functionality can be disabled with [`set_include_zarrs_metadata(false)`](Array::set_include_zarrs_metadata).
mod array_builder;
mod array_errors;
Expand Down Expand Up @@ -60,6 +63,7 @@ pub use self::{
use parking_lot::Mutex;
use rayon::prelude::{ParallelBridge, ParallelIterator};
use safe_transmute::TriviallyTransmutable;
use serde::Serialize;

use crate::{
array_subset::{validate_array_subset, ArraySubset},
Expand Down Expand Up @@ -173,6 +177,8 @@ pub struct Array<TStorage: ?Sized> {
parallel_chunks: bool,
/// Chunk locks.
chunk_locks: Mutex<HashMap<Vec<u64>, Arc<Mutex<()>>>>,
/// Zarrs metadata.
include_zarrs_metadata: bool,
}

impl<TStorage: ?Sized> Array<TStorage> {
Expand Down Expand Up @@ -247,6 +253,7 @@ impl<TStorage: ?Sized> Array<TStorage> {
parallel_codecs: true,
parallel_chunks: true,
chunk_locks: Mutex::default(),
include_zarrs_metadata: true,
})
}

Expand Down Expand Up @@ -353,17 +360,45 @@ impl<TStorage: ?Sized> Array<TStorage> {
self.parallel_chunks = parallel_chunks;
}

/// Enable or disable the inclusion of zarrs metadata in the array attributes. Enabled by default.
///
/// Zarrs metadata includes the zarrs version and some parameters.
pub fn set_include_zarrs_metadata(&mut self, include_zarrs_metadata: bool) {
self.include_zarrs_metadata = include_zarrs_metadata;
}

/// Create [`ArrayMetadata`].
#[must_use]
pub fn metadata(&self) -> ArrayMetadata {
let attributes = if self.include_zarrs_metadata {
#[derive(Serialize)]
struct ZarrsMetadata {
description: String,
repository: String,
version: String,
}
let zarrs_metadata = ZarrsMetadata {
description: "This array was created with zarrs".to_string(),
repository: env!("CARGO_PKG_REPOSITORY").to_string(),
version: env!("CARGO_PKG_VERSION").to_string(),
};
let mut attributes = self.attributes().clone();
attributes.insert("_zarrs".to_string(), unsafe {
serde_json::to_value(zarrs_metadata).unwrap_unchecked()
});
attributes
} else {
self.attributes().clone()
};

ArrayMetadataV3::new(
self.shape().to_vec(),
self.data_type().metadata(),
self.chunk_grid().create_metadata(),
self.chunk_key_encoding().create_metadata(),
self.data_type().metadata_fill_value(self.fill_value()),
self.codecs().create_metadatas(),
self.attributes().clone(),
attributes,
self.storage_transformers().create_metadatas(),
self.dimension_names().clone(),
self.additional_fields().clone(),
Expand Down
1 change: 1 addition & 0 deletions src/array/array_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ impl ArrayBuilder {
parallel_codecs: self.parallel_codecs,
parallel_chunks: self.parallel_chunks,
chunk_locks: parking_lot::Mutex::default(),
include_zarrs_metadata: true,
})
}
}

0 comments on commit d8ff25c

Please sign in to comment.