Skip to content

Commit

Permalink
fix(parse): Don't assume boundary characters are one byte
Browse files Browse the repository at this point in the history
This was inspired by heck.  They have an invariant to ensure this isn't
a problem (only accept `_` as boundary) while on the other hand we
accept a lot of things as boundaries.
  • Loading branch information
epage committed Jul 7, 2019
1 parent 60372ef commit 166e263
Showing 1 changed file with 2 additions and 3 deletions.
5 changes: 2 additions & 3 deletions src/tokens.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,13 +176,12 @@ fn split_ident(ident: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
while let Some((i, c)) = char_indices.next() {
let cur_mode = WordMode::classify(c);
if cur_mode == WordMode::Boundary {
if start == i {
start += 1;
}
assert!(start_mode == WordMode::Boundary);
continue;
}
if start_mode == WordMode::Boundary {
start_mode = cur_mode;
start = i;
}

if let Some(&(next_i, next)) = char_indices.peek() {
Expand Down

0 comments on commit 166e263

Please sign in to comment.