ik-rs

Usage

add to Cargo.toml

[dependencies]
ik-rs = "0.7.0"

Chinese Segment

#[cfg(test)]
mod test {
 use ik_rs::core::ik_segmenter::{IKSegmenter, TokenMode};
 #[test]
 pub fn test_ik() {
 let mut ik = IKSegmenter::new();
 let text = "中华人民共和国";
 let tokens = ik.tokenize(text, TokenMode::INDEX); // TokenMode::SEARCH
 let mut token_texts = Vec::new();
 for token in tokens.iter() {
 println!("{:?}", token);
 token_texts.push(token.get_lexeme_text());
 }
 assert_eq!(
 token_texts,
 vec![
 "中华人民共和国",
 "中华人民",
 "中华",
 "华人",
 "人民共和国",
 "人民",
 "共和国",
 "共和",
 "国"
 ]
 )
 }
}

BenchMark

cargo bench
ik_tokenize_benchmark time: [19.366 μs 19.572 μs 19.850 μs]
 change: [-1.5364% -0.4029% +0.7357%] (p = 0.51 > 0.05)

Usage for Tantivy