feat: Check and replace UTF-16 files

We don't have good detection for non-UTF encodings and don't have
encoding support for UTF-32, so limiting it to just UTF-16.

Fixes #17

crates/typos/Cargo.toml

@@ -28,3 +28,4 @@ unicode-segmentation = "1.6.0"
 derive_more = "0.99.11"
 derive_setters = "0.1"
 content_inspector = "0.2.4"
+encoding = "0.2"

crates/typos/src/checks.rs

@@ -1,4 +1,5 @@
 use bstr::ByteSlice;
+use encoding::Encoding;
 
 use crate::report;
 use crate::tokens;
@@ -190,18 +191,31 @@ impl ParseWords {
 
         let buffer = std::fs::read(path)
             .map_err(|e| crate::ErrorKind::IoError.into_error().with_source(e))?;
-        if !explicit && !self.binary {
-            let content_type = content_inspector::inspect(&buffer);
-            // HACK: We only support UTF-8 at the moment
-            if content_type.is_binary()
-                || (content_type != content_inspector::ContentType::UTF_8_BOM
-                    && content_type != content_inspector::ContentType::UTF_8)
-            {
-                let msg = report::BinaryFile { path };
-                reporter.report(msg.into());
-                return Ok(typos_found);
-            }
-        }
+        let content_type = content_inspector::inspect(&buffer);
+
+        let buffer = match content_type {
+            content_inspector::ContentType::BINARY |
+            // HACK: We don't support UTF-32 yet
+            content_inspector::ContentType::UTF_32LE |
+            content_inspector::ContentType::UTF_32BE
+                => {
+                if !explicit && !self.binary {
+                    let msg = report::BinaryFile { path };
+                    reporter.report(msg.into());
+                    return Ok(typos_found);
+                } else {
+                    buffer
+                }
+            },
+            content_inspector::ContentType::UTF_8 |
+            content_inspector::ContentType::UTF_8_BOM
+                => { buffer
+                },
+            content_inspector::ContentType::UTF_16LE |
+            content_inspector::ContentType::UTF_16BE => {
+                buffer
+            },
+        };
 
         for line in buffer.lines() {
             let msg = report::Parse {