diff --git a/charabia/src/separators.rs b/charabia/src/separators.rs index 60a0048..6d0d1e9 100644 --- a/charabia/src/separators.rs +++ b/charabia/src/separators.rs @@ -64,6 +64,7 @@ pub const DEFAULT_SEPARATORS: &[&str] = &[ #[rustfmt::skip] pub const CONTEXT_SEPARATORS: &[&str] = &[ + "\0", // Null byte, can be used as artificial separator "᠆", // Mongolian Todo Soft Hyphen, mark the end of a paragraph. "᚛", "᚜", // Oghams, mark start and end of text "!", ". ", ", ", ";", "?", "¡", "§", "¶", "¿", ";", // Latin