Merge pull request #728 from epage/stream

fix(stream)!: Don't return input on peek

src/_topic/language.rs

@@ -136,7 +136,7 @@
 #![doc = include_str!("../../examples/string/parser.rs")]
 //! ```
 //!
-//! See also [`take_escaped`] and [`escaped_transform`].
+//! See also [`take_escaped`] and [`escaped`].
 //!
 //! ### Integers
 //!
@@ -320,7 +320,7 @@
 #![allow(unused_imports)]
 use crate::ascii::dec_int;
 use crate::ascii::dec_uint;
-use crate::ascii::escaped_transform;
+use crate::ascii::escaped;
 use crate::ascii::float;
 use crate::ascii::hex_uint;
 use crate::ascii::take_escaped;

src/stream/bstr.rs

@@ -77,6 +77,15 @@ impl<'i> Stream for &'i BStr {
         }
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        if self.is_empty() {
+            None
+        } else {
+            Some(self[0])
+        }
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -98,6 +107,11 @@ impl<'i> Stream for &'i BStr {
         *self = BStr::from_bytes(next);
         slice
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        let (slice, _next) = self.split_at(offset);
+        slice
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {

src/stream/bytes.rs

@@ -77,6 +77,15 @@ impl<'i> Stream for &'i Bytes {
         }
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        if self.is_empty() {
+            None
+        } else {
+            Some(self[0])
+        }
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -98,6 +107,11 @@ impl<'i> Stream for &'i Bytes {
         *self = Bytes::from_bytes(next);
         slice
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        let (slice, _next) = self.split_at(offset);
+        slice
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {

src/stream/locating.rs

@@ -128,6 +128,11 @@ impl<I: Stream> Stream for LocatingSlice<I> {
         self.input.next_token()
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        self.input.peek_token()
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -143,6 +148,10 @@ impl<I: Stream> Stream for LocatingSlice<I> {
     fn next_slice(&mut self, offset: usize) -> Self::Slice {
         self.input.next_slice(offset)
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        self.input.peek_slice(offset)
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {

src/stream/mod.rs

@@ -154,15 +154,7 @@ pub trait Stream: Offset<<Self as Stream>::Checkpoint> + crate::lib::std::fmt::D
     /// Split off the next token from the input
     fn next_token(&mut self) -> Option<Self::Token>;
     /// Split off the next token from the input
-    #[inline(always)]
-    fn peek_token(&self) -> Option<(Self, Self::Token)>
-    where
-        Self: Clone,
-    {
-        let mut peek = self.clone();
-        let token = peek.next_token()?;
-        Some((peek, token))
-    }
+    fn peek_token(&self) -> Option<Self::Token>;
 
     /// Finds the offset of the next matching token
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
@@ -195,15 +187,7 @@ pub trait Stream: Offset<<Self as Stream>::Checkpoint> + crate::lib::std::fmt::D
     ///
     fn next_slice(&mut self, offset: usize) -> Self::Slice;
     /// Split off a slice of tokens from the input
-    #[inline(always)]
-    fn peek_slice(&self, offset: usize) -> (Self, Self::Slice)
-    where
-        Self: Clone,
-    {
-        let mut peek = self.clone();
-        let slice = peek.next_slice(offset);
-        (peek, slice)
-    }
+    fn peek_slice(&self, offset: usize) -> Self::Slice;
 
     /// Advance to the end of the stream
     #[inline(always)]
@@ -212,13 +196,11 @@ pub trait Stream: Offset<<Self as Stream>::Checkpoint> + crate::lib::std::fmt::D
     }
     /// Advance to the end of the stream
     #[inline(always)]
-    fn peek_finish(&self) -> (Self, Self::Slice)
+    fn peek_finish(&self) -> Self::Slice
     where
         Self: Clone,
     {
-        let mut peek = self.clone();
-        let slice = peek.finish();
-        (peek, slice)
+        self.peek_slice(self.eof_offset())
     }
 
     /// Save the current parse location within the stream
@@ -261,6 +243,15 @@ where
         Some(token.clone())
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        if self.is_empty() {
+            None
+        } else {
+            Some(self[0].clone())
+        }
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -282,6 +273,11 @@ where
         *self = next;
         slice
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        let (slice, _next) = self.split_at(offset);
+        slice
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {
@@ -323,6 +319,11 @@ impl<'i> Stream for &'i str {
         Some(c)
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        self.chars().next()
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -357,6 +358,11 @@ impl<'i> Stream for &'i str {
         *self = next;
         slice
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        let (slice, _next) = self.split_at(offset);
+        slice
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {
@@ -406,6 +412,11 @@ where
         next_bit(self)
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        peek_bit(self)
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -434,6 +445,14 @@ where
         self.1 = end_offset;
         (s, start_offset, end_offset)
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        let byte_offset = (offset + self.1) / 8;
+        let end_offset = (offset + self.1) % 8;
+        let s = self.0.peek_slice(byte_offset);
+        let start_offset = self.1;
+        (s, start_offset, end_offset)
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {
@@ -496,6 +515,27 @@ where
     }
 }
 
+fn peek_bit<I>(i: &(I, usize)) -> Option<bool>
+where
+    I: Stream<Token = u8> + Clone,
+{
+    if i.eof_offset() == 0 {
+        return None;
+    }
+    let offset = i.1;
+
+    let mut next_i = i.0.clone();
+    let byte = next_i.next_token()?;
+    let bit = (byte >> offset) & 0x1 == 0x1;
+
+    let next_offset = offset + 1;
+    if next_offset == 8 {
+        Some(bit)
+    } else {
+        Some(bit)
+    }
+}
+
 /// Current parse locations offset
 ///
 /// See [`LocatingSlice`] for adding location tracking to your [`Stream`]

src/stream/partial.rs

@@ -165,6 +165,11 @@ impl<I: Stream> Stream for Partial<I> {
         self.input.next_token()
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        self.input.peek_token()
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -180,6 +185,10 @@ impl<I: Stream> Stream for Partial<I> {
     fn next_slice(&mut self, offset: usize) -> Self::Slice {
         self.input.next_slice(offset)
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        self.input.peek_slice(offset)
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {

src/stream/recoverable.rs

@@ -138,6 +138,11 @@ where
         self.input.next_token()
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        self.input.peek_token()
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -153,6 +158,10 @@ where
     fn next_slice(&mut self, offset: usize) -> Self::Slice {
         self.input.next_slice(offset)
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        self.input.peek_slice(offset)
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {

src/stream/stateful.rs

@@ -116,6 +116,11 @@ impl<I: Stream, S: crate::lib::std::fmt::Debug> Stream for Stateful<I, S> {
         self.input.next_token()
     }
 
+    #[inline(always)]
+    fn peek_token(&self) -> Option<Self::Token> {
+        self.input.peek_token()
+    }
+
     #[inline(always)]
     fn offset_for<P>(&self, predicate: P) -> Option<usize>
     where
@@ -131,6 +136,10 @@ impl<I: Stream, S: crate::lib::std::fmt::Debug> Stream for Stateful<I, S> {
     fn next_slice(&mut self, offset: usize) -> Self::Slice {
         self.input.next_slice(offset)
     }
+    #[inline(always)]
+    fn peek_slice(&self, offset: usize) -> Self::Slice {
+        self.input.peek_slice(offset)
+    }
 
     #[inline(always)]
     fn checkpoint(&self) -> Self::Checkpoint {

src/stream/tests.rs

@@ -67,8 +67,7 @@ fn test_bit_stream_empty() {
     let actual = i.offset_at(1);
     assert_eq!(actual, Err(Needed::new(1)));
 
-    let (actual_input, actual_slice) = i.peek_slice(0);
-    assert_eq!(actual_input, (&b""[..], 0));
+    let actual_slice = i.peek_slice(0);
     assert_eq!(actual_slice, (&b""[..], 0, 0));
 }
 
@@ -101,24 +100,24 @@ fn bit_stream_inner(byte_len: usize, start: usize) {
 
     let mut curr_i = i;
     let mut curr_offset = 0;
-    while let Some((next_i, _token)) = curr_i.peek_token() {
+    while let Some(_token) = curr_i.peek_token() {
         let to_offset = curr_i.offset_from(&i);
         assert_eq!(curr_offset, to_offset);
 
-        let (slice_i, _) = i.peek_slice(curr_offset);
-        assert_eq!(curr_i, slice_i);
+        let actual_slice = i.peek_slice(curr_offset);
+        let expected_slice = i.clone().peek_slice(curr_offset);
+        assert_eq!(actual_slice, expected_slice);
 
         let at_offset = i.offset_at(curr_offset).unwrap();
         assert_eq!(curr_offset, at_offset);
 
         let eof_offset = curr_i.eof_offset();
-        let (next_eof_i, eof_slice) = curr_i.peek_slice(eof_offset);
-        assert_eq!(next_eof_i, (&b""[..], 0));
+        let eof_slice = curr_i.peek_slice(eof_offset);
         let eof_slice_i = (eof_slice.0, eof_slice.1);
         assert_eq!(eof_slice_i, curr_i);
 
         curr_offset += 1;
-        curr_i = next_i;
+        let _ = curr_i.next_token();
     }
     assert_eq!(i.eof_offset(), curr_offset);
 }