From cde9dacbb763408b987b19978feeee32b683c322 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Thu, 13 Apr 2023 16:03:27 -0700 Subject: [PATCH 1/4] fix for incorrect implementation of KeyDeserialize for (T, U) --- src/de.rs | 89 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/src/de.rs b/src/de.rs index 046d8b9..04da951 100644 --- a/src/de.rs +++ b/src/de.rs @@ -8,6 +8,10 @@ use crate::int_key::IntKey; pub trait KeyDeserialize { type Output: Sized; + /// This key length is used for the deserialization of compound keys. + /// It should be equal to PrimaryKey::key().len() + const KEY_LEN: u16; + fn from_vec(value: Vec) -> StdResult; fn from_slice(value: &[u8]) -> StdResult { @@ -18,6 +22,8 @@ pub trait KeyDeserialize { impl KeyDeserialize for () { type Output = (); + const KEY_LEN: u16 = 0; + #[inline(always)] fn from_vec(_value: Vec) -> StdResult { Ok(()) @@ -27,6 +33,8 @@ impl KeyDeserialize for () { impl KeyDeserialize for Vec { type Output = Vec; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Ok(value) @@ -36,6 +44,8 @@ impl KeyDeserialize for Vec { impl KeyDeserialize for &Vec { type Output = Vec; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Ok(value) @@ -45,6 +55,8 @@ impl KeyDeserialize for &Vec { impl KeyDeserialize for &[u8] { type Output = Vec; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Ok(value) @@ -54,6 +66,8 @@ impl KeyDeserialize for &[u8] { impl KeyDeserialize for String { type Output = String; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { String::from_utf8(value).map_err(StdError::invalid_utf8) @@ -63,6 +77,8 @@ impl KeyDeserialize for String { impl KeyDeserialize for &String { type Output = String; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Self::Output::from_vec(value) @@ -72,6 +88,8 @@ impl KeyDeserialize for &String { impl KeyDeserialize for &str { type Output = String; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Self::Output::from_vec(value) @@ -81,6 +99,8 @@ impl KeyDeserialize for &str { impl KeyDeserialize for Addr { type Output = Addr; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Ok(Addr::unchecked(String::from_vec(value)?)) @@ -90,6 +110,8 @@ impl KeyDeserialize for Addr { impl KeyDeserialize for &Addr { type Output = Addr; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Self::Output::from_vec(value) @@ -101,6 +123,8 @@ macro_rules! integer_de { $(impl KeyDeserialize for $t { type Output = $t; + const KEY_LEN: u16 = 1; + #[inline(always)] fn from_vec(value: Vec) -> StdResult { Ok(<$t>::from_cw_bytes(value.as_slice().try_into() @@ -121,33 +145,52 @@ fn parse_length(value: &[u8]) -> StdResult { .into()) } +/// This will split off the first key from the value based on the provided key length. +/// Since from_vec expects that the last key is not length prefixed, we need to remove the length prefix. +/// This should not be called on the last key within a compound key. +/// The return value is ordered as (first_key, remainder) +fn split_off_first_key(key_len: u16, value: &[u8]) -> StdResult<(Vec, &[u8])> { + let mut slice_index: usize = 0; + let mut first_key = Vec::new(); + // First iterate over the sub keys + for key_index in 0..key_len { + // Key length is always 2 bytes + let key_start_index = slice_index + 2; + let len_slice = &value[slice_index..key_start_index]; + // If this is not the last key, we need to add the length prefix + if key_index != key_len - 1 { + first_key.extend_from_slice(len_slice); + } + let subkey_len = parse_length(len_slice)?; + first_key.extend_from_slice(&value[key_start_index..key_start_index + subkey_len]); + slice_index += subkey_len + 2; + } + let remainder = &value[slice_index..]; + Ok((first_key, remainder)) +} + impl KeyDeserialize for (T, U) { type Output = (T::Output, U::Output); - #[inline(always)] - fn from_vec(mut value: Vec) -> StdResult { - let mut tu = value.split_off(2); - let t_len = parse_length(&value)?; - let u = tu.split_off(t_len); + const KEY_LEN: u16 = T::KEY_LEN + U::KEY_LEN; - Ok((T::from_vec(tu)?, U::from_vec(u)?)) + #[inline(always)] + fn from_vec(value: Vec) -> StdResult { + let (t, u) = split_off_first_key(T::KEY_LEN, value.as_ref())?; + Ok((T::from_vec(t)?, U::from_vec(u.to_vec())?)) } } impl KeyDeserialize for (T, U, V) { type Output = (T::Output, U::Output, V::Output); - #[inline(always)] - fn from_vec(mut value: Vec) -> StdResult { - let mut tuv = value.split_off(2); - let t_len = parse_length(&value)?; - let mut len_uv = tuv.split_off(t_len); + const KEY_LEN: u16 = T::KEY_LEN + U::KEY_LEN + V::KEY_LEN; - let mut uv = len_uv.split_off(2); - let u_len = parse_length(&len_uv)?; - let v = uv.split_off(u_len); - - Ok((T::from_vec(tuv)?, U::from_vec(uv)?, V::from_vec(v)?)) + #[inline(always)] + fn from_vec(value: Vec) -> StdResult { + let (t, remainder) = split_off_first_key(T::KEY_LEN, value.as_ref())?; + let (u, v) = split_off_first_key(U::KEY_LEN, remainder)?; + Ok((T::from_vec(t)?, U::from_vec(u)?, V::from_vec(v.to_vec())?)) } } @@ -257,6 +300,20 @@ mod test { ); } + #[test] + fn deserialize_tuple_of_tuples_works() { + assert_eq!( + <((&[u8], &str), (&[u8], &str))>::from_slice( + ((BYTES, STRING), (BYTES, STRING)).joined_key().as_slice() + ) + .unwrap(), + ( + (BYTES.to_vec(), STRING.to_string()), + (BYTES.to_vec(), STRING.to_string()) + ) + ); + } + #[test] fn deserialize_triple_works() { assert_eq!( From 86040d3423ea1148342fe1da8cc005da7b377e12 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 17 Apr 2023 09:45:05 -0700 Subject: [PATCH 2/4] changed KEY_LEN to KEY_ELEMS --- src/de.rs | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/de.rs b/src/de.rs index 04da951..c6b6dcb 100644 --- a/src/de.rs +++ b/src/de.rs @@ -8,9 +8,9 @@ use crate::int_key::IntKey; pub trait KeyDeserialize { type Output: Sized; - /// This key length is used for the deserialization of compound keys. + /// The number of key elements is used for the deserialization of compound keys. /// It should be equal to PrimaryKey::key().len() - const KEY_LEN: u16; + const KEY_ELEMS: u16; fn from_vec(value: Vec) -> StdResult; @@ -22,7 +22,7 @@ pub trait KeyDeserialize { impl KeyDeserialize for () { type Output = (); - const KEY_LEN: u16 = 0; + const KEY_ELEMS: u16 = 0; #[inline(always)] fn from_vec(_value: Vec) -> StdResult { @@ -33,7 +33,7 @@ impl KeyDeserialize for () { impl KeyDeserialize for Vec { type Output = Vec; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -44,7 +44,7 @@ impl KeyDeserialize for Vec { impl KeyDeserialize for &Vec { type Output = Vec; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -55,7 +55,7 @@ impl KeyDeserialize for &Vec { impl KeyDeserialize for &[u8] { type Output = Vec; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -66,7 +66,7 @@ impl KeyDeserialize for &[u8] { impl KeyDeserialize for String { type Output = String; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -77,7 +77,7 @@ impl KeyDeserialize for String { impl KeyDeserialize for &String { type Output = String; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -88,7 +88,7 @@ impl KeyDeserialize for &String { impl KeyDeserialize for &str { type Output = String; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -99,7 +99,7 @@ impl KeyDeserialize for &str { impl KeyDeserialize for Addr { type Output = Addr; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -110,7 +110,7 @@ impl KeyDeserialize for Addr { impl KeyDeserialize for &Addr { type Output = Addr; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -123,7 +123,7 @@ macro_rules! integer_de { $(impl KeyDeserialize for $t { type Output = $t; - const KEY_LEN: u16 = 1; + const KEY_ELEMS: u16 = 1; #[inline(always)] fn from_vec(value: Vec) -> StdResult { @@ -145,20 +145,20 @@ fn parse_length(value: &[u8]) -> StdResult { .into()) } -/// This will split off the first key from the value based on the provided key length. +/// This will split off the first key from the value based on the provided number of key elements. /// Since from_vec expects that the last key is not length prefixed, we need to remove the length prefix. /// This should not be called on the last key within a compound key. /// The return value is ordered as (first_key, remainder) -fn split_off_first_key(key_len: u16, value: &[u8]) -> StdResult<(Vec, &[u8])> { +fn split_off_first_key(key_elems: u16, value: &[u8]) -> StdResult<(Vec, &[u8])> { let mut slice_index: usize = 0; let mut first_key = Vec::new(); // First iterate over the sub keys - for key_index in 0..key_len { + for key_index in 0..key_elems { // Key length is always 2 bytes let key_start_index = slice_index + 2; let len_slice = &value[slice_index..key_start_index]; // If this is not the last key, we need to add the length prefix - if key_index != key_len - 1 { + if key_index != key_elems - 1 { first_key.extend_from_slice(len_slice); } let subkey_len = parse_length(len_slice)?; @@ -172,11 +172,11 @@ fn split_off_first_key(key_len: u16, value: &[u8]) -> StdResult<(Vec, &[u8]) impl KeyDeserialize for (T, U) { type Output = (T::Output, U::Output); - const KEY_LEN: u16 = T::KEY_LEN + U::KEY_LEN; + const KEY_ELEMS: u16 = T::KEY_ELEMS + U::KEY_ELEMS; #[inline(always)] fn from_vec(value: Vec) -> StdResult { - let (t, u) = split_off_first_key(T::KEY_LEN, value.as_ref())?; + let (t, u) = split_off_first_key(T::KEY_ELEMS, value.as_ref())?; Ok((T::from_vec(t)?, U::from_vec(u.to_vec())?)) } } @@ -184,12 +184,12 @@ impl KeyDeserialize for (T, U) { impl KeyDeserialize for (T, U, V) { type Output = (T::Output, U::Output, V::Output); - const KEY_LEN: u16 = T::KEY_LEN + U::KEY_LEN + V::KEY_LEN; + const KEY_ELEMS: u16 = T::KEY_ELEMS + U::KEY_ELEMS + V::KEY_ELEMS; #[inline(always)] fn from_vec(value: Vec) -> StdResult { - let (t, remainder) = split_off_first_key(T::KEY_LEN, value.as_ref())?; - let (u, v) = split_off_first_key(U::KEY_LEN, remainder)?; + let (t, remainder) = split_off_first_key(T::KEY_ELEMS, value.as_ref())?; + let (u, v) = split_off_first_key(U::KEY_ELEMS, remainder)?; Ok((T::from_vec(t)?, U::from_vec(u)?, V::from_vec(v.to_vec())?)) } } From 238b4e1e43ddbfb73661e94b5fbd6adae63a407b Mon Sep 17 00:00:00 2001 From: Mauro Lacy Date: Mon, 17 Apr 2023 20:02:45 +0200 Subject: [PATCH 3/4] Add more tuple / triple de tests --- src/de.rs | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/de.rs b/src/de.rs index c6b6dcb..6e056a9 100644 --- a/src/de.rs +++ b/src/de.rs @@ -314,6 +314,60 @@ mod test { ); } + #[test] + fn deserialize_tuple_of_triples_works() { + assert_eq!( + <((&[u8], &str, u32), (&[u8], &str, u16))>::from_slice( + ((BYTES, STRING, 1234u32), (BYTES, STRING, 567u16)) + .joined_key() + .as_slice() + ) + .unwrap(), + ( + (BYTES.to_vec(), STRING.to_string(), 1234), + (BYTES.to_vec(), STRING.to_string(), 567) + ) + ); + } + + #[test] + fn deserialize_triple_of_tuples_works() { + assert_eq!( + <((u32, &str), (&str, &[u8]), (i32, i32))>::from_slice( + ((1234u32, STRING), (STRING, BYTES), (1234i32, 567i32)) + .joined_key() + .as_slice() + ) + .unwrap(), + ( + (1234, STRING.to_string()), + (STRING.to_string(), BYTES.to_vec()), + (1234, 567) + ) + ); + } + + #[test] + fn deserialize_triple_of_triples_works() { + assert_eq!( + <((u32, &str, &str), (&str, &[u8], u8), (i32, u8, i32))>::from_slice( + ( + (1234u32, STRING, STRING), + (STRING, BYTES, 123u8), + (4567i32, 89u8, 10i32) + ) + .joined_key() + .as_slice() + ) + .unwrap(), + ( + (1234, STRING.to_string(), STRING.to_string()), + (STRING.to_string(), BYTES.to_vec(), 123), + (4567, 89, 10) + ) + ); + } + #[test] fn deserialize_triple_works() { assert_eq!( From 403eff662ce6d5fa0c5a07baa745204b3b566e6d Mon Sep 17 00:00:00 2001 From: Mauro Lacy Date: Mon, 17 Apr 2023 23:10:19 +0200 Subject: [PATCH 4/4] Slightly better version of first key split --- src/de.rs | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/de.rs b/src/de.rs index 6e056a9..65b3f57 100644 --- a/src/de.rs +++ b/src/de.rs @@ -145,27 +145,29 @@ fn parse_length(value: &[u8]) -> StdResult { .into()) } -/// This will split off the first key from the value based on the provided number of key elements. -/// Since from_vec expects that the last key is not length prefixed, we need to remove the length prefix. -/// This should not be called on the last key within a compound key. -/// The return value is ordered as (first_key, remainder) -fn split_off_first_key(key_elems: u16, value: &[u8]) -> StdResult<(Vec, &[u8])> { - let mut slice_index: usize = 0; +/// Splits the first key from the value based on the provided number of key elements. +/// The return value is ordered as (first_key, remainder). +/// +fn split_first_key(key_elems: u16, value: &[u8]) -> StdResult<(Vec, &[u8])> { + let mut index = 0; let mut first_key = Vec::new(); - // First iterate over the sub keys - for key_index in 0..key_elems { - // Key length is always 2 bytes - let key_start_index = slice_index + 2; - let len_slice = &value[slice_index..key_start_index]; - // If this is not the last key, we need to add the length prefix - if key_index != key_elems - 1 { + + // Iterate over the sub keys + for i in 0..key_elems { + let len_slice = &value[index..index + 2]; + index += 2; + let is_last_key = i == key_elems - 1; + + if !is_last_key { first_key.extend_from_slice(len_slice); } + let subkey_len = parse_length(len_slice)?; - first_key.extend_from_slice(&value[key_start_index..key_start_index + subkey_len]); - slice_index += subkey_len + 2; + first_key.extend_from_slice(&value[index..index + subkey_len]); + index += subkey_len; } - let remainder = &value[slice_index..]; + + let remainder = &value[index..]; Ok((first_key, remainder)) } @@ -176,7 +178,7 @@ impl KeyDeserialize for (T, U) { #[inline(always)] fn from_vec(value: Vec) -> StdResult { - let (t, u) = split_off_first_key(T::KEY_ELEMS, value.as_ref())?; + let (t, u) = split_first_key(T::KEY_ELEMS, value.as_ref())?; Ok((T::from_vec(t)?, U::from_vec(u.to_vec())?)) } } @@ -188,8 +190,8 @@ impl KeyDeserialize for #[inline(always)] fn from_vec(value: Vec) -> StdResult { - let (t, remainder) = split_off_first_key(T::KEY_ELEMS, value.as_ref())?; - let (u, v) = split_off_first_key(U::KEY_ELEMS, remainder)?; + let (t, remainder) = split_first_key(T::KEY_ELEMS, value.as_ref())?; + let (u, v) = split_first_key(U::KEY_ELEMS, remainder)?; Ok((T::from_vec(t)?, U::from_vec(u)?, V::from_vec(v.to_vec())?)) } }