Cargo.toml

[package]
name = "charabia"
version = "0.4.0"
license = "MIT"
authors = ["Many <many@meilisearch.com>"]
edition = "2021"
description = "A simple library to detect the language, tokenize the text and normalize the tokens"
documentation = "https://docs.rs/charabia"
repository = "https://github.com/meilisearch/charabia"
keywords = ["segmenter", "tokenizer", "normalize", "language"]
categories = ["text-processing"]

[dependencies]
character_converter = { version = "2.1.0", optional = true }
cow-utils = "0.1"
deunicode = "1.1.1"
fst = "0.4"
jieba-rs = { version = "0.6", optional = true }
once_cell = "1.5.2"
slice-group-by = "0.3.0"
unicode-segmentation = "1.6.0"
whatlang = "0.13.0"
lindera = { version = "0.13.5", features = ["ipadic", "unidic"], optional = true }
lindera-core = { version = "0.13.4", optional = true }

[features]
default = ["chinese", "hebrew", "japanese"]

# allow chinese specialized tokenization
chinese = ["dep:character_converter", "dep:jieba-rs"]

# allow hebrew specialized tokenization
hebrew = []

# allow japanese specialized tokenization
japanese = ["dep:lindera", "dep:lindera-core"]

[dev-dependencies]
criterion = "0.3"
jemallocator = "0.3.0"

[[bench]]
name = "bench"
harness = false