icu_capi/
segmenter_word.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5#[diplomat::bridge]
6#[diplomat::abi_rename = "icu4x_{0}_mv1"]
7pub mod ffi {
8    use alloc::boxed::Box;
9    use icu_segmenter::scaffold::{Latin1, PotentiallyIllFormedUtf8, Utf16};
10
11    #[cfg(feature = "buffer_provider")]
12    use crate::unstable::provider::ffi::DataProvider;
13    #[cfg(any(feature = "compiled_data", feature = "buffer_provider"))]
14    use crate::unstable::{errors::ffi::DataError, locale_core::ffi::Locale};
15
16    #[diplomat::enum_convert(icu_segmenter::options::WordType, needs_wildcard)]
17    #[diplomat::rust_link(icu::segmenter::options::WordType, Enum)]
18    #[non_exhaustive]
19    pub enum SegmenterWordType {
20        // This is an output type, so the default mostly impacts deferred initialization.
21        #[diplomat::attr(auto, default)]
22        None = 0,
23        Number = 1,
24        Letter = 2,
25    }
26
27    #[diplomat::opaque]
28    /// An ICU4X word-break segmenter, capable of finding word breakpoints in strings.
29    #[diplomat::rust_link(icu::segmenter::WordSegmenter, Struct)]
30    #[diplomat::rust_link(icu::segmenter::WordSegmenterBorrowed, Struct, hidden)]
31    #[diplomat::demo(custom_func = "../../../tools/web-demo/custom/WordSegmenter.mjs")]
32    pub struct WordSegmenter(icu_segmenter::WordSegmenter);
33
34    #[diplomat::opaque]
35    #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator, Struct)]
36    #[diplomat::attr(demo_gen, disable)] // iterator type
37    pub struct WordBreakIteratorUtf8<'a>(
38        icu_segmenter::iterators::WordBreakIterator<'a, 'a, PotentiallyIllFormedUtf8>,
39    );
40
41    #[diplomat::opaque]
42    #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator, Struct)]
43    #[diplomat::attr(demo_gen, disable)] // iterator type
44    pub struct WordBreakIteratorUtf16<'a>(
45        icu_segmenter::iterators::WordBreakIterator<'a, 'a, Utf16>,
46    );
47    #[diplomat::opaque]
48    #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator, Struct)]
49    #[diplomat::attr(demo_gen, disable)] // iterator type
50    pub struct WordBreakIteratorLatin1<'a>(
51        icu_segmenter::iterators::WordBreakIterator<'a, 'a, Latin1>,
52    );
53
54    impl SegmenterWordType {
55        #[diplomat::rust_link(icu::segmenter::options::WordType::is_word_like, FnInEnum)]
56        #[diplomat::attr(auto, getter)]
57        pub fn is_word_like(self) -> bool {
58            icu_segmenter::options::WordType::from(self).is_word_like()
59        }
60    }
61
62    impl WordSegmenter {
63        /// Construct a [`WordSegmenter`] with automatically selecting the best available LSTM
64        /// or dictionary payload data, using compiled data. This does not assume any content locale.
65        ///
66        /// Note: currently, it uses dictionary for Chinese and Japanese, and LSTM for Burmese,
67        /// Khmer, Lao, and Thai.
68        #[diplomat::rust_link(icu::segmenter::WordSegmenter::new_auto, FnInStruct)]
69        #[diplomat::rust_link(icu::segmenter::options::WordBreakInvariantOptions, Struct, hidden)]
70        #[diplomat::rust_link(
71            icu::segmenter::options::WordBreakInvariantOptions::default,
72            FnInStruct,
73            hidden
74        )]
75        #[diplomat::attr(auto, named_constructor = "auto")]
76        #[cfg(feature = "compiled_data")]
77        pub fn create_auto() -> Box<WordSegmenter> {
78            Box::new(WordSegmenter(
79                icu_segmenter::WordSegmenter::new_auto(Default::default()).static_to_owned(),
80            ))
81        }
82
83        /// Construct a [`WordSegmenter`] with automatically selecting the best available LSTM
84        /// or dictionary payload data, using compiled data.
85        ///
86        /// Note: currently, it uses dictionary for Chinese and Japanese, and LSTM for Burmese,
87        /// Khmer, Lao, and Thai.
88        #[diplomat::rust_link(icu::segmenter::WordSegmenter::try_new_auto, FnInStruct)]
89        #[diplomat::rust_link(icu::segmenter::options::WordBreakOptions, Struct, hidden)]
90        #[diplomat::rust_link(
91            icu::segmenter::options::WordBreakOptions::default,
92            FnInStruct,
93            hidden
94        )]
95        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "auto_with_content_locale")]
96        #[cfg(feature = "compiled_data")]
97        pub fn create_auto_with_content_locale(
98            locale: &Locale,
99        ) -> Result<Box<WordSegmenter>, DataError> {
100            Ok(Box::new(WordSegmenter(
101                icu_segmenter::WordSegmenter::try_new_auto(locale.into())?,
102            )))
103        }
104
105        /// Construct a [`WordSegmenter`] with automatically selecting the best available LSTM
106        /// or dictionary payload data, using a particular data source.
107        ///
108        /// Note: currently, it uses dictionary for Chinese and Japanese, and LSTM for Burmese,
109        /// Khmer, Lao, and Thai.
110        #[diplomat::rust_link(icu::segmenter::WordSegmenter::try_new_auto, FnInStruct)]
111        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "auto_with_content_locale_and_provider")]
112        #[cfg(feature = "buffer_provider")]
113        pub fn create_auto_with_content_locale_and_provider(
114            provider: &DataProvider,
115            locale: &Locale,
116        ) -> Result<Box<WordSegmenter>, DataError> {
117            Ok(Box::new(WordSegmenter(
118                icu_segmenter::WordSegmenter::try_new_auto_with_buffer_provider(
119                    provider.get()?,
120                    locale.into(),
121                )?,
122            )))
123        }
124
125        /// Construct a [`WordSegmenter`] with LSTM payload data for Burmese, Khmer, Lao, and
126        /// Thai, using compiled data. This does not assume any content locale.
127        ///
128        /// Note: currently, it uses dictionary for Chinese and Japanese, and LSTM for Burmese,
129        /// Khmer, Lao, and Thai.
130        #[diplomat::rust_link(icu::segmenter::WordSegmenter::new_lstm, FnInStruct)]
131        #[diplomat::attr(auto, named_constructor = "lstm")]
132        #[cfg(feature = "compiled_data")]
133        pub fn create_lstm() -> Box<WordSegmenter> {
134            Box::new(WordSegmenter(
135                icu_segmenter::WordSegmenter::new_lstm(Default::default()).static_to_owned(),
136            ))
137        }
138
139        /// Construct a [`WordSegmenter`] with LSTM payload data for Burmese, Khmer, Lao, and
140        /// Thai, using compiled data.
141        ///
142        /// Note: currently, it uses dictionary for Chinese and Japanese, and LSTM for Burmese,
143        /// Khmer, Lao, and Thai.
144        #[diplomat::rust_link(icu::segmenter::WordSegmenter::try_new_lstm, FnInStruct)]
145        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "lstm_with_content_locale")]
146        #[cfg(feature = "compiled_data")]
147        pub fn create_lstm_with_content_locale(
148            locale: &Locale,
149        ) -> Result<Box<WordSegmenter>, DataError> {
150            Ok(Box::new(WordSegmenter(
151                icu_segmenter::WordSegmenter::try_new_lstm(locale.into())?,
152            )))
153        }
154
155        /// Construct a [`WordSegmenter`] with LSTM payload data for Burmese, Khmer, Lao, and
156        /// Thai, using a particular data source.
157        ///
158        /// Note: currently, it uses dictionary for Chinese and Japanese, and LSTM for Burmese,
159        /// Khmer, Lao, and Thai.
160        #[diplomat::rust_link(icu::segmenter::WordSegmenter::try_new_lstm, FnInStruct)]
161        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "lstm_with_content_locale_and_provider")]
162        #[cfg(feature = "buffer_provider")]
163        pub fn create_lstm_with_content_locale_and_provider(
164            provider: &DataProvider,
165            locale: &Locale,
166        ) -> Result<Box<WordSegmenter>, DataError> {
167            Ok(Box::new(WordSegmenter(
168                icu_segmenter::WordSegmenter::try_new_lstm_with_buffer_provider(
169                    provider.get()?,
170                    locale.into(),
171                )?,
172            )))
173        }
174
175        /// Construct a [`WordSegmenter`] with dictionary payload data for Chinese, Japanese,
176        /// Burmese, Khmer, Lao, and Thai, using compiled data. This does not assume any content locale.
177        ///
178        /// Note: currently, it uses dictionary for Chinese and Japanese, and dictionary for Burmese,
179        /// Khmer, Lao, and Thai.
180        #[diplomat::rust_link(icu::segmenter::WordSegmenter::new_dictionary, FnInStruct)]
181        #[diplomat::attr(auto, named_constructor = "dictionary")]
182        #[cfg(feature = "compiled_data")]
183        pub fn create_dictionary() -> Box<WordSegmenter> {
184            Box::new(WordSegmenter(
185                icu_segmenter::WordSegmenter::new_dictionary(Default::default()).static_to_owned(),
186            ))
187        }
188
189        /// Construct a [`WordSegmenter`] with dictionary payload data for Chinese, Japanese,
190        /// Burmese, Khmer, Lao, and Thai, using compiled data.
191        ///
192        /// Note: currently, it uses dictionary for Chinese and Japanese, and dictionary for Burmese,
193        /// Khmer, Lao, and Thai.
194        #[diplomat::rust_link(icu::segmenter::WordSegmenter::try_new_dictionary, FnInStruct)]
195        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "dictionary_with_content_locale")]
196        #[cfg(feature = "compiled_data")]
197        pub fn create_dictionary_with_content_locale(
198            locale: &Locale,
199        ) -> Result<Box<WordSegmenter>, DataError> {
200            Ok(Box::new(WordSegmenter(
201                icu_segmenter::WordSegmenter::try_new_dictionary(locale.into())?,
202            )))
203        }
204
205        /// Construct a [`WordSegmenter`] with dictionary payload data for Chinese, Japanese,
206        /// Burmese, Khmer, Lao, and Thai, using a particular data source.
207        ///
208        /// Note: currently, it uses dictionary for Chinese and Japanese, and dictionary for Burmese,
209        /// Khmer, Lao, and Thai.
210        #[diplomat::rust_link(icu::segmenter::WordSegmenter::try_new_dictionary, FnInStruct)]
211        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "dictionary_with_content_locale_and_provider")]
212        #[cfg(feature = "buffer_provider")]
213        pub fn create_dictionary_with_content_locale_and_provider(
214            provider: &DataProvider,
215            locale: &Locale,
216        ) -> Result<Box<WordSegmenter>, DataError> {
217            Ok(Box::new(WordSegmenter(
218                icu_segmenter::WordSegmenter::try_new_dictionary_with_buffer_provider(
219                    provider.get()?,
220                    locale.into(),
221                )?,
222            )))
223        }
224
225        /// Construct a [`WordSegmenter`] with no support for scripts requiring complex context dependent word breaks (Chinese, Japanese,
226        /// Burmese, Khmer, Lao, and Thai), using compiled data. This does not assume any content locale.
227        #[diplomat::rust_link(
228            icu::segmenter::WordSegmenter::new_for_non_complex_scripts,
229            FnInStruct
230        )]
231        #[diplomat::attr(auto, named_constructor = "for_non_complex_scripts")]
232        #[cfg(feature = "compiled_data")]
233        pub fn create_for_non_complex_scripts() -> Box<WordSegmenter> {
234            Box::new(WordSegmenter(
235                icu_segmenter::WordSegmenter::new_for_non_complex_scripts(Default::default())
236                    .static_to_owned(),
237            ))
238        }
239
240        /// Construct a [`WordSegmenter`] with no support for scripts requiring complex context dependent word breaks (Chinese, Japanese,
241        /// Burmese, Khmer, Lao, and Thai), using compiled data.
242        #[diplomat::rust_link(
243            icu::segmenter::WordSegmenter::try_new_for_non_complex_scripts,
244            FnInStruct
245        )]
246        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "for_non_complex_scripts_with_content_locale")]
247        #[cfg(feature = "compiled_data")]
248        pub fn create_for_non_complex_scripts_with_content_locale(
249            locale: &Locale,
250        ) -> Result<Box<WordSegmenter>, DataError> {
251            Ok(Box::new(WordSegmenter(
252                icu_segmenter::WordSegmenter::try_new_for_non_complex_scripts(locale.into())?,
253            )))
254        }
255
256        /// Construct a [`WordSegmenter`] with no support for scripts requiring complex context dependent word breaks (Chinese, Japanese,
257        /// Burmese, Khmer, Lao, and Thai), using a particular data source.
258        #[diplomat::rust_link(
259            icu::segmenter::WordSegmenter::try_new_for_non_complex_scripts,
260            FnInStruct
261        )]
262        #[diplomat::attr(all(supports = fallible_constructors, supports = named_constructors), named_constructor = "for_non_complex_scripts_with_content_locale_and_provider")]
263        #[cfg(feature = "buffer_provider")]
264        pub fn create_for_non_complex_scripts_with_content_locale_and_provider(
265            provider: &DataProvider,
266            locale: &Locale,
267        ) -> Result<Box<WordSegmenter>, DataError> {
268            Ok(Box::new(WordSegmenter(
269                icu_segmenter::WordSegmenter::try_new_for_non_complex_scripts_with_buffer_provider(
270                    provider.get()?,
271                    locale.into(),
272                )?,
273            )))
274        }
275        /// Segments a string.
276        ///
277        /// Ill-formed input is treated as if errors had been replaced with REPLACEMENT CHARACTERs according
278        /// to the WHATWG Encoding Standard.
279        #[diplomat::rust_link(icu::segmenter::WordSegmenterBorrowed::segment_utf8, FnInStruct)]
280        #[diplomat::rust_link(
281            icu::segmenter::WordSegmenterBorrowed::segment_str,
282            FnInStruct,
283            hidden
284        )]
285        #[diplomat::attr(not(supports = utf8_strings), disable)]
286        #[diplomat::attr(*, rename = "segment")]
287        pub fn segment_utf8<'a>(
288            &'a self,
289            input: &'a DiplomatStr,
290        ) -> Box<WordBreakIteratorUtf8<'a>> {
291            Box::new(WordBreakIteratorUtf8(
292                self.0.as_borrowed().segment_utf8(input),
293            ))
294        }
295
296        /// Segments a string.
297        ///
298        /// Ill-formed input is treated as if errors had been replaced with REPLACEMENT CHARACTERs according
299        /// to the WHATWG Encoding Standard.
300        #[diplomat::rust_link(icu::segmenter::WordSegmenterBorrowed::segment_utf16, FnInStruct)]
301        #[diplomat::attr(not(supports = utf8_strings), rename = "segment")]
302        #[diplomat::attr(supports = utf8_strings, rename = "segment16")]
303        pub fn segment_utf16<'a>(
304            &'a self,
305            input: &'a DiplomatStr16,
306        ) -> Box<WordBreakIteratorUtf16<'a>> {
307            Box::new(WordBreakIteratorUtf16(
308                self.0.as_borrowed().segment_utf16(input),
309            ))
310        }
311
312        /// Segments a Latin-1 string.
313        #[diplomat::rust_link(icu::segmenter::WordSegmenterBorrowed::segment_latin1, FnInStruct)]
314        #[diplomat::attr(not(supports = utf8_strings), disable)]
315        pub fn segment_latin1<'a>(&'a self, input: &'a [u8]) -> Box<WordBreakIteratorLatin1<'a>> {
316            Box::new(WordBreakIteratorLatin1(
317                self.0.as_borrowed().segment_latin1(input),
318            ))
319        }
320    }
321
322    impl<'a> WordBreakIteratorUtf8<'a> {
323        /// Finds the next breakpoint. Returns -1 if at the end of the string or if the index is
324        /// out of range of a 32-bit signed integer.
325        #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator::next, FnInStruct)]
326        pub fn next(&mut self) -> i32 {
327            self.0
328                .next()
329                .and_then(|u| i32::try_from(u).ok())
330                .unwrap_or(-1)
331        }
332
333        /// Return the status value of break boundary.
334        #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator::word_type, FnInStruct)]
335        #[diplomat::rust_link(
336            icu::segmenter::iterators::WordBreakIteratorWithWordType,
337            Struct,
338            hidden
339        )]
340        #[diplomat::rust_link(
341            icu::segmenter::iterators::WordBreakIteratorWithWordType::next,
342            FnInStruct,
343            hidden
344        )]
345        #[diplomat::attr(auto, getter)]
346        pub fn word_type(&self) -> SegmenterWordType {
347            self.0.word_type().into()
348        }
349
350        /// Return true when break boundary is word-like such as letter/number/CJK
351        #[diplomat::rust_link(
352            icu::segmenter::iterators::WordBreakIterator::is_word_like,
353            FnInStruct
354        )]
355        #[diplomat::attr(auto, getter)]
356        pub fn is_word_like(&self) -> bool {
357            self.0.is_word_like()
358        }
359    }
360
361    impl<'a> WordBreakIteratorUtf16<'a> {
362        /// Finds the next breakpoint. Returns -1 if at the end of the string or if the index is
363        /// out of range of a 32-bit signed integer.
364        #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator::next, FnInStruct)]
365        #[diplomat::rust_link(
366            icu::segmenter::iterators::WordBreakIterator::Item,
367            AssociatedTypeInStruct,
368            hidden
369        )]
370        pub fn next(&mut self) -> i32 {
371            self.0
372                .next()
373                .and_then(|u| i32::try_from(u).ok())
374                .unwrap_or(-1)
375        }
376
377        /// Return the status value of break boundary.
378        #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator::word_type, FnInStruct)]
379        #[diplomat::rust_link(
380            icu::segmenter::iterators::WordBreakIterator::iter_with_word_type,
381            FnInStruct,
382            hidden
383        )]
384        #[diplomat::attr(auto, getter)]
385        pub fn word_type(&self) -> SegmenterWordType {
386            self.0.word_type().into()
387        }
388
389        /// Return true when break boundary is word-like such as letter/number/CJK
390        #[diplomat::rust_link(
391            icu::segmenter::iterators::WordBreakIterator::is_word_like,
392            FnInStruct
393        )]
394        #[diplomat::attr(auto, getter)]
395        pub fn is_word_like(&self) -> bool {
396            self.0.is_word_like()
397        }
398    }
399
400    impl<'a> WordBreakIteratorLatin1<'a> {
401        /// Finds the next breakpoint. Returns -1 if at the end of the string or if the index is
402        /// out of range of a 32-bit signed integer.
403        #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator::next, FnInStruct)]
404        #[diplomat::rust_link(
405            icu::segmenter::iterators::WordBreakIterator::Item,
406            AssociatedTypeInStruct,
407            hidden
408        )]
409        pub fn next(&mut self) -> i32 {
410            self.0
411                .next()
412                .and_then(|u| i32::try_from(u).ok())
413                .unwrap_or(-1)
414        }
415
416        /// Return the status value of break boundary.
417        #[diplomat::rust_link(icu::segmenter::iterators::WordBreakIterator::word_type, FnInStruct)]
418        #[diplomat::attr(auto, getter)]
419        pub fn word_type(&self) -> SegmenterWordType {
420            self.0.word_type().into()
421        }
422
423        /// Return true when break boundary is word-like such as letter/number/CJK
424        #[diplomat::rust_link(
425            icu::segmenter::iterators::WordBreakIterator::is_word_like,
426            FnInStruct
427        )]
428        #[diplomat::attr(auto, getter)]
429        pub fn is_word_like(&self) -> bool {
430            self.0.is_word_like()
431        }
432    }
433}
434
435impl<'a> From<&'a crate::unstable::locale_core::ffi::Locale>
436    for icu_segmenter::options::WordBreakOptions<'a>
437{
438    fn from(other: &'a crate::unstable::locale_core::ffi::Locale) -> Self {
439        let mut options = icu_segmenter::options::WordBreakOptions::default();
440        options.content_locale = Some(&other.0.id);
441        options
442    }
443}
icu_capi/segmenter_word.rs

icu_capi/
segmenter_word.rs
