From 09eb5b8651a4d53d4864a8c23a52d695867f68fb Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Wed, 8 Jan 2025 08:59:50 +1100 Subject: [PATCH] fletcher32 cleanup --- CHANGELOG.md | 3 ++- zarrs/doc/status/codecs_experimental.md | 2 ++ zarrs/src/array/codec/bytes_to_bytes/fletcher32.rs | 9 +++++++++ zarrs/src/config.rs | 2 +- zarrs/tests/zarr_python_compat.rs | 2 +- zarrs_metadata/src/v3/array/codec/fletcher32.rs | 8 +++++++- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 126dead1..39cf709b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `Group::[set_]consolidated_metadata` - Add `Node::consolidate_metadata` - Consolidated metadata is not currently used to optimise node hierarchy requests -- Add `fletcher32` checksum codec based on the numcodecs implementation +- Add experimental `fletcher32` checksum codec based on the numcodecs implementation + - Adds `fletcher32` feature flag ### Changed - **Breaking**: Seal `Array` extension traits: `ArraySharded[Readable]Ext` and `ArrayChunkCacheExt` diff --git a/zarrs/doc/status/codecs_experimental.md b/zarrs/doc/status/codecs_experimental.md index f1ec44f1..bdc7a136 100644 --- a/zarrs/doc/status/codecs_experimental.md +++ b/zarrs/doc/status/codecs_experimental.md @@ -13,6 +13,7 @@ This is configurable with [`Config::experimental_codec_names_mut`](config::Confi | | [vlen-utf8] | | ✓ | ✓ | | | Bytes to Bytes | [bz2] | | ✓ | ✓ | bz2 | | | [gdeflate] | | ✓ | | gdeflate | +| | [fletcher32] | | ✓ | ✓ | fletcher32 | [bitround]: (crate::array::codec::array_to_array::bitround) [zfp]: crate::array::codec::array_to_bytes::zfp @@ -23,3 +24,4 @@ This is configurable with [`Config::experimental_codec_names_mut`](config::Confi [vlen-utf8]: crate::array::codec::array_to_bytes::vlen_utf8 [bz2]: crate::array::codec::bytes_to_bytes::bz2 [gdeflate]: crate::array::codec::bytes_to_bytes::gdeflate +[fletcher32]: crate::array::codec::bytes_to_bytes::fletcher32 diff --git a/zarrs/src/array/codec/bytes_to_bytes/fletcher32.rs b/zarrs/src/array/codec/bytes_to_bytes/fletcher32.rs index d79c244e..044509d9 100644 --- a/zarrs/src/array/codec/bytes_to_bytes/fletcher32.rs +++ b/zarrs/src/array/codec/bytes_to_bytes/fletcher32.rs @@ -2,7 +2,16 @@ //! //! Appends a fletcher32 checksum of the input bytestream. //! +//! This is based on the `numcodecs` implementation. //! See . +//! +//!
+//! This codec is experimental and may be incompatible with other Zarr V3 implementations. +//!
+//! +//! This codec requires the `fletcher32` feature, which is disabled by default. +//! +//! See [`Fletcher32CodecConfigurationV1`] for example `JSON` metadata. mod fletcher32_codec; diff --git a/zarrs/src/config.rs b/zarrs/src/config.rs index d1bab970..424c4c97 100644 --- a/zarrs/src/config.rs +++ b/zarrs/src/config.rs @@ -22,7 +22,7 @@ use crate::array::{codec::CodecOptions, ArrayMetadataOptions}; /// /// [`CodecOptions::validate_checksums()`] defaults to [`Config::validate_checksums()`]. /// -/// If validate checksums is enabled, checksum codecs (e.g. `crc32c`) will validate that encoded data matches stored checksums, otherwise validation is skipped. +/// If validate checksums is enabled, checksum codecs (e.g. `crc32c`, `fletcher32`) will validate that encoded data matches stored checksums, otherwise validation is skipped. /// Note that regardless of this configuration option, checksum codecs may skip validation when partial decoding. /// /// ### Store Empty Chunks diff --git a/zarrs/tests/zarr_python_compat.rs b/zarrs/tests/zarr_python_compat.rs index d009fb01..f9e76e26 100644 --- a/zarrs/tests/zarr_python_compat.rs +++ b/zarrs/tests/zarr_python_compat.rs @@ -33,7 +33,7 @@ fn zarr_python_compat_fletcher32() -> Result<(), Box> { let elements = array.retrieve_array_subset_elements::(&ArraySubset::new_with_shape( array.shape().to_vec(), ))?; - assert_eq!(elements, (0..100*100).collect::>()); + assert_eq!(elements, (0..100 * 100).collect::>()); Ok(()) } diff --git a/zarrs_metadata/src/v3/array/codec/fletcher32.rs b/zarrs_metadata/src/v3/array/codec/fletcher32.rs index 48869807..31521dc6 100644 --- a/zarrs_metadata/src/v3/array/codec/fletcher32.rs +++ b/zarrs_metadata/src/v3/array/codec/fletcher32.rs @@ -14,7 +14,13 @@ pub enum Fletcher32CodecConfiguration { /// `fletcher32` (checksum) codec configuration parameters (version 1.0 draft). /// -/// See . +/// ### Example (Zarr V3) +/// ```json +/// { +/// "name": "fletcher32", +/// "configuration": {} +/// } +/// ``` #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Debug, Display)] #[serde(deny_unknown_fields)] #[display("{}", serde_json::to_string(self).unwrap_or_default())]