perf(parser): Speed up UTF-8 validation

crate-ci · Apr 28, 2021 · 6b92e34 · 6b92e34
1 parent 819702c
commit 6b92e34
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 1 deletion.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/typos/Cargo.toml b/crates/typos/Cargo.toml
@@ -20,6 +20,7 @@ thiserror = "1.0"
 regex = "1.3"
 once_cell = "1.2.0"
 serde = { version = "1.0", features = ["derive"] }
+simdutf8 = "0.1.1"
 itertools = "0.10"
 log = "0.4"
 unicode-segmentation = "1.7.1"
diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs
@@ -158,7 +158,7 @@ impl<'s> Iterator for Utf8Chunks<'s> {
             if self.source.is_empty() {
                 return None;
             }
-            match std::str::from_utf8(self.source) {
+            match simdutf8::compat::from_utf8(self.source) {
                 Ok(valid) => {
                     self.source = b"";
                     return Some(valid);