Merge branch 'master' into ar

Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Matt Ickstadt <[email protected]>"]
 license = "MIT/Apache-2.0"
 name = "elasticlunr-rs"
-version = "2.3.13"
+version = "2.3.14"
 description = "A partial port of elasticlunr.js to Rust for generating static document search indexes"
 documentation = "https://docs.rs/elasticlunr-rs"
 repository = "https://github.com/mattico/elasticlunr-rs"
@@ -28,8 +28,9 @@ serde_derive = "1.0.34" # First verstion to support #[serde(flatten)]
 serde_json = "1"
 strum = "0.21"
 strum_macros = "0.21"
-jieba-rs = { version = "0.5.0", optional = true }
-lindera = { version = "0.3.4", optional = true }
+jieba-rs = { version = "0.6", optional = true }
+lindera = { version = "0.8", optional = true }
+lindera-core = { version = "0.8", optional = true }
 
 [features]
 default = ["languages"]
@@ -51,5 +52,5 @@ ru = ["rust-stemmers"]
 sv = ["rust-stemmers"]
 tr = ["rust-stemmers"]
 zh = ["jieba-rs"]
-ja = ["lindera"]
-ar = []
+ja = ["lindera", "lindera-core"]
+ar = []

src/lib.rs

@@ -47,6 +47,8 @@ extern crate maplit;
 extern crate jieba_rs;
 #[cfg(feature = "ja")]
 extern crate lindera;
+#[cfg(feature = "ja")]
+extern crate lindera_core;
 
 /// The version of elasticlunr.js this library was designed for.
 pub const ELASTICLUNR_VERSION: &str = "0.9.5";

src/pipeline.rs

@@ -1,10 +1,6 @@
 //! Defines the pipeline which processes text for inclusion in the index. Most users do not need
 //! to use this module directly.
 
-#[cfg(feature = "zh")]
-use jieba_rs::Jieba;
-#[cfg(feature = "ja")]
-use lindera::tokenizer::Tokenizer;
 use serde::ser::{Serialize, SerializeSeq, Serializer};
 
 /// Splits a text string into a vector of individual tokens.
@@ -17,7 +13,7 @@ pub fn tokenize(text: &str) -> Vec<String> {
 
 #[cfg(feature = "zh")]
 pub fn tokenize_chinese(text: &str) -> Vec<String> {
-    let jieba = Jieba::new();
+    let jieba = jieba_rs::Jieba::new();
 
     jieba
         .cut_for_search(text.as_ref(), false)
@@ -28,9 +24,17 @@ pub fn tokenize_chinese(text: &str) -> Vec<String> {
 
 #[cfg(feature = "ja")]
 pub fn tokenize_japanese(text: &str) -> Vec<String> {
-    let mut tokenizer = Tokenizer::new("decompose", "");
+    use lindera_core::viterbi::Mode;
+    use lindera::tokenizer::{Tokenizer, TokenizerConfig};
+    let config = TokenizerConfig {
+        mode: Mode::Decompose(Default::default()),
+        ..Default::default()
+    };
+    // NB: unwrap() is okay since the errors are only related to user-supplied dictionaries.
+    let mut tokenizer = Tokenizer::with_config(config).unwrap();
     tokenizer
         .tokenize(text)
+        .unwrap()
         .into_iter()
         .filter_map(|tok| match tok.detail.get(0).map(|d| d.as_str()) {
             Some("助詞") | Some("助動詞") | Some("記号") | Some("UNK") => None,