diff --git a/CHANGELOG.md b/CHANGELOG.md index b20c452e..136195a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - `Array::store_chunk` now erases empty chunks - Fixed a race in `Array::store_chunk_subset` and add a fast path if the subset spans the whole chunk + - Fix complex number handling in `fill_value_from_metadata` + - Fix byte arrays being interpreted as complex number fill value metadata ## [0.5.0] - 2023-10-08 diff --git a/src/array/data_type.rs b/src/array/data_type.rs index d4d142f6..319cf940 100644 --- a/src/array/data_type.rs +++ b/src/array/data_type.rs @@ -235,14 +235,14 @@ impl DataType { Self::UInt64 => Ok(FV::from(fill_value.try_as_uint::().ok_or_else(err)?)), Self::Float32 => Ok(FV::from(fill_value.try_as_float::().ok_or_else(err)?)), Self::Float64 => Ok(FV::from(fill_value.try_as_float::().ok_or_else(err)?)), - Self::Complex64 => Ok(FV::from(num::complex::Complex32::new( - fill_value.try_as_float::().ok_or_else(err)?, - fill_value.try_as_float::().ok_or_else(err)?, - ))), - Self::Complex128 => Ok(FV::from(num::complex::Complex64::new( - fill_value.try_as_float::().ok_or_else(err)?, - fill_value.try_as_float::().ok_or_else(err)?, - ))), + Self::Complex64 => { + let (re, im) = fill_value.try_as_float_pair::().ok_or_else(err)?; + Ok(FV::from(num::complex::Complex32::new(re, im))) + } + Self::Complex128 => { + let (re, im) = fill_value.try_as_float_pair::().ok_or_else(err)?; + Ok(FV::from(num::complex::Complex64::new(re, im))) + } Self::Extension(extension) => extension.fill_value_from_metadata(fill_value), } } @@ -349,10 +349,26 @@ mod tests { let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); - match data_type { - DataType::Bool => {} - _ => panic!(), - } + assert_eq!(data_type, DataType::Bool); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#"true"#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + &[true as u8] + ); + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#"false"#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + &[false as u8] + ); } #[test] @@ -361,10 +377,23 @@ mod tests { let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); - match data_type { - DataType::Int8 => {} - _ => panic!(), - } + assert_eq!(data_type, DataType::Int8); + + assert_eq!( + data_type + .fill_value_from_metadata(&serde_json::from_str::("7").unwrap()) + .unwrap() + .as_ne_bytes(), + 7i8.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata(&serde_json::from_str::("-7").unwrap()) + .unwrap() + .as_ne_bytes(), + (-7i8).to_ne_bytes() + ); } #[test] @@ -373,10 +402,15 @@ mod tests { let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); - match data_type { - DataType::UInt8 => {} - _ => panic!(), - } + assert_eq!(data_type, DataType::UInt8); + + assert_eq!( + data_type + .fill_value_from_metadata(&serde_json::from_str::("7").unwrap()) + .unwrap() + .as_ne_bytes(), + 7u8.to_ne_bytes() + ); } #[test] @@ -385,30 +419,210 @@ mod tests { let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); - match data_type { - DataType::Float32 => {} - _ => panic!(), - } + assert_eq!(data_type, DataType::Float32); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::("-7.0").unwrap() + ) + .unwrap() + .as_ne_bytes(), + (-7.0f32).to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""NaN""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f32::NAN.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""0x7fc00000""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f32::NAN.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f32::INFINITY.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""-Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f32::NEG_INFINITY.to_ne_bytes() + ); + } + + #[test] + fn data_type_float64() { + let json = r#""float64""#; + let metadata: Metadata = serde_json::from_str(json).unwrap(); + let data_type = DataType::from_metadata(&metadata).unwrap(); + assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); + assert_eq!(data_type, DataType::Float64); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::("-7.0").unwrap() + ) + .unwrap() + .as_ne_bytes(), + (-7.0f64).to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""NaN""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f64::NAN.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f64::INFINITY.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""-Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f64::NEG_INFINITY.to_ne_bytes() + ); } #[cfg(feature = "float16")] #[test] fn data_type_float16() { + use half::f16; + let json = r#""float16""#; let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); assert_eq!(data_type.identifier(), "float16"); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::("-7.0").unwrap() + ) + .unwrap() + .as_ne_bytes(), + (f16::from_f32_const(-7.0)).to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""NaN""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f16::NAN.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f16::INFINITY.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""-Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + f16::NEG_INFINITY.to_ne_bytes() + ); } #[cfg(feature = "bfloat16")] #[test] fn data_type_bfloat16() { + use half::bf16; + let json = r#""bfloat16""#; let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); assert_eq!(data_type.identifier(), "bfloat16"); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::("-7.0").unwrap() + ) + .unwrap() + .as_ne_bytes(), + (bf16::from_f32_const(-7.0)).to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""NaN""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + bf16::NAN.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + bf16::INFINITY.to_ne_bytes() + ); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#""-Infinity""#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + bf16::NEG_INFINITY.to_ne_bytes() + ); } #[test] @@ -417,10 +631,22 @@ mod tests { let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); - match data_type { - DataType::Complex64 => {} - _ => panic!(), - } + assert_eq!(data_type, DataType::Complex64); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#"[-7.0, "Infinity"]"#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + (-7.0f32) + .to_ne_bytes() + .iter() + .chain(f32::INFINITY.to_ne_bytes().iter()) + .map(|b| *b) + .collect::>() + ); } #[test] @@ -429,10 +655,22 @@ mod tests { let metadata: Metadata = serde_json::from_str(json).unwrap(); let data_type = DataType::from_metadata(&metadata).unwrap(); assert_eq!(json, serde_json::to_string(&data_type.metadata()).unwrap()); - match data_type { - DataType::Complex128 => {} - _ => panic!(), - } + assert_eq!(data_type, DataType::Complex128); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#"[-7.0, "Infinity"]"#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + (-7.0f64) + .to_ne_bytes() + .iter() + .chain(f64::INFINITY.to_ne_bytes().iter()) + .map(|b| *b) + .collect::>() + ); } #[cfg(feature = "raw_bits")] @@ -445,6 +683,16 @@ mod tests { assert_eq!(data_type.identifier(), "r*"); assert_eq!(data_type.name().as_str(), "r8"); assert_eq!(data_type.size(), 1); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#"[7]"#).unwrap() + ) + .unwrap() + .as_ne_bytes(), + (7u8).to_ne_bytes() + ); } #[cfg(feature = "raw_bits")] @@ -457,6 +705,16 @@ mod tests { assert_eq!(data_type.identifier(), "r*"); assert_eq!(data_type.name().as_str(), "r16"); assert_eq!(data_type.size(), 2); + + assert_eq!( + data_type + .fill_value_from_metadata( + &serde_json::from_str::(r#"[0, 255]"#).unwrap() + ) + .unwrap() + .as_ne_bytes(), // FIXME: All other fill values can go straight to ne bytes, but the endianness of the bytes array is unknown. This depends on the array->bytes codec? + &[0u8, 255u8] + ); } #[test] diff --git a/src/array/data_type/raw_bits.rs b/src/array/data_type/raw_bits.rs index c70c9a46..f190493c 100644 --- a/src/array/data_type/raw_bits.rs +++ b/src/array/data_type/raw_bits.rs @@ -1,5 +1,7 @@ //! `r*` raw bits data type. Variable size in bits given by *. +// TODO: Make this a standard part of DataType and don't lock behind a feature + use crate::{ array::{data_type::DataTypePlugin, FillValue, FillValueMetadata}, metadata::{ConfigurationInvalidError, Metadata}, diff --git a/src/array/fill_value_metadata.rs b/src/array/fill_value_metadata.rs index 1e323dc9..f397a7b1 100644 --- a/src/array/fill_value_metadata.rs +++ b/src/array/fill_value_metadata.rs @@ -23,12 +23,12 @@ pub enum FillValueMetadata { Int(i64), /// A float. Float(FillValueFloat), - /// A complex number. - #[display(fmt = "{{re:{_0}, im:{_1}}}")] - Complex(FillValueFloat, FillValueFloat), /// A raw data type. #[display(fmt = "{_0:?}")] ByteArray(Vec), + /// A complex number. + #[display(fmt = "{{re:{_0}, im:{_1}}}")] + Complex(FillValueFloat, FillValueFloat), } impl TryFrom<&str> for FillValueMetadata { @@ -51,6 +51,37 @@ pub enum FillValueFloat { NonFinite(FillValueFloatStringNonFinite), } +impl FillValueFloat { + fn to_float(&self) -> Option { + match self { + Self::Float(float) => T::from(*float), + Self::HexString(hex_string) => { + let bytes: &[u8] = hex_string.as_bytes(); + if bytes.len() == core::mem::size_of::() { + // NOTE: Cleaner way of doing this? + if core::mem::size_of::() == core::mem::size_of::() { + T::from(f32::from_be_bytes(bytes.try_into().unwrap_or_default())) + } else if core::mem::size_of::() == core::mem::size_of::() { + T::from(f64::from_be_bytes(bytes.try_into().unwrap_or_default())) + } else { + None + } + } else { + None + } + } + Self::NonFinite(nonfinite) => { + use FillValueFloatStringNonFinite as NF; + match nonfinite { + NF::PosInfinity => Some(T::infinity()), + NF::NegInfinity => Some(T::neg_infinity()), + NF::NaN => Some(T::nan()), + } + } + } + } +} + /// A hex string. #[derive(Debug, Clone, Eq, PartialEq, From)] pub struct HexString(Vec); @@ -123,7 +154,7 @@ pub enum FillValueFloatStringNonFinite { } impl FillValueMetadata { - /// Convert the fill value to bool. + /// Convert the fill value to a bool. #[must_use] pub fn try_as_bool(&self) -> Option { match self { @@ -132,7 +163,7 @@ impl FillValueMetadata { } } - /// Convert the fill value to int. + /// Convert the fill value to an int. #[must_use] pub fn try_as_int + std::convert::TryFrom>( &self, @@ -144,7 +175,7 @@ impl FillValueMetadata { } } - /// Convert the fill value to uint. + /// Convert the fill value to a uint. #[must_use] pub fn try_as_uint + std::convert::TryFrom>( &self, @@ -156,7 +187,7 @@ impl FillValueMetadata { } } - /// Convert the fill value to float. + /// Convert the fill value to a float. #[must_use] pub fn try_as_float(&self) -> Option { match self { @@ -192,6 +223,21 @@ impl FillValueMetadata { _ => None, } } + + /// Convert the fill value to a complex number (float pair). + #[must_use] + pub fn try_as_float_pair(&self) -> Option<(T, T)> { + match self { + FillValueMetadata::Complex(re, im) => { + if let (Some(re), Some(im)) = (re.to_float::(), im.to_float::()) { + Some((re, im)) + } else { + None + } + } + _ => None, + } + } } #[cfg(test)]