Fix encoding of EIJIRO.TXT

yuichiro-s · Mar 7, 2019 · 56ec1d6 · 56ec1d6
1 parent ccb14da
commit 56ec1d6
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/app/scripts/options/importer.ts b/app/scripts/options/importer.ts
@@ -5,7 +5,7 @@ import { IPackage, ISettings } from "../common/package";
 import { sendCommand } from "../content/command";
 import { EIJIRO_PKG_ID } from "../preprocess/eijiro";
 
-export function loadFile(file: File): Promise<string> {
+function loadFile(file: File): Promise<string> {
     return new Promise((resolve, reject) => {
         console.log(`Loading ${file.name} ...`);
         const reader = new FileReader();

diff --git a/app/scripts/preprocess/eijiro.worker.ts b/app/scripts/preprocess/eijiro.worker.ts
@@ -45,7 +45,7 @@ async function loadEijiro(
     chunkSize: number,
     progressFn: (progress: IProgress) => void) {
 
-    const eijiroContent = loadText(eijiroURL);
+    const eijiroContent = loadText(eijiroURL, "Shift_JIS");
     const inflectionContent = loadText(inflectionURL);
     const frequencyContent = loadText(frequencyURL);
     const whitelistContent = loadText(whitelistURL);

diff --git a/app/scripts/preprocess/loader-browser.ts b/app/scripts/preprocess/loader-browser.ts
@@ -1,4 +1,4 @@
-function loadFile(url: string, type: XMLHttpRequestResponseType): Promise<any> {
+function loadFile(url: string, type: XMLHttpRequestResponseType, encoding?: string): Promise<any> {
     return new Promise((resolve, reject) => {
         console.log(`Loading ${url} ...`);
         const x = new XMLHttpRequest();
@@ -9,8 +9,11 @@ function loadFile(url: string, type: XMLHttpRequestResponseType): Promise<any> {
         x.onerror = reject;
         x.open("GET", url);
         x.responseType = type;
+        if (encoding !== undefined) {
+            x.overrideMimeType(`text/plain; charset=${encoding}`);
+        }
         x.send();
     });
 }
-export const loadText = (url: string) => loadFile(url, "text");
+export const loadText = (url: string, encoding?: string) => loadFile(url, "text", encoding);
 export const loadJSON = (url: string) => loadFile(url, "json");