icu_capi/
segmenter_line.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5#[diplomat::bridge]
6#[diplomat::abi_rename = "icu4x_{0}_mv1"]
7pub mod ffi {
8    use alloc::boxed::Box;
9    use icu_segmenter::scaffold::{Latin1, PotentiallyIllFormedUtf8, Utf16};
10
11    #[cfg(any(feature = "compiled_data", feature = "buffer_provider"))]
12    use crate::unstable::locale_core::ffi::Locale;
13    #[cfg(feature = "buffer_provider")]
14    use crate::unstable::{errors::ffi::DataError, provider::ffi::DataProvider};
15    use diplomat_runtime::DiplomatOption;
16    #[cfg(any(feature = "compiled_data", feature = "buffer_provider"))]
17    use icu_segmenter::options::LineBreakOptions;
18
19    #[diplomat::opaque]
20    /// An ICU4X line-break segmenter, capable of finding breakpoints in strings.
21    #[diplomat::rust_link(icu::segmenter::LineSegmenter, Struct)]
22    #[diplomat::rust_link(icu::segmenter::LineSegmenterBorrowed, Struct, hidden)]
23    pub struct LineSegmenter(icu_segmenter::LineSegmenter);
24
25    #[diplomat::rust_link(icu::segmenter::options::LineBreakStrictness, Enum)]
26    #[diplomat::enum_convert(icu_segmenter::options::LineBreakStrictness, needs_wildcard)]
27    #[non_exhaustive]
28    pub enum LineBreakStrictness {
29        Loose,
30        Normal,
31        #[diplomat::attr(auto, default)]
32        Strict,
33        Anywhere,
34    }
35
36    #[diplomat::rust_link(icu::segmenter::options::LineBreakWordOption, Enum)]
37    #[diplomat::enum_convert(icu_segmenter::options::LineBreakWordOption, needs_wildcard)]
38    #[non_exhaustive]
39    pub enum LineBreakWordOption {
40        #[diplomat::attr(auto, default)]
41        Normal,
42        BreakAll,
43        KeepAll,
44    }
45
46    #[diplomat::rust_link(icu::segmenter::options::LineBreakOptions, Struct)]
47    #[diplomat::rust_link(icu::segmenter::options::LineBreakOptions::default, FnInStruct, hidden)]
48    #[diplomat::attr(supports = non_exhaustive_structs, rename = "LineBreakOptions")]
49    pub struct LineBreakOptionsV2 {
50        pub strictness: DiplomatOption<LineBreakStrictness>,
51        pub word_option: DiplomatOption<LineBreakWordOption>,
52    }
53
54    #[diplomat::opaque]
55    #[diplomat::rust_link(icu::segmenter::iterators::LineBreakIterator, Struct)]
56    #[diplomat::attr(demo_gen, disable)] // iterator type
57    pub struct LineBreakIteratorUtf8<'a>(
58        icu_segmenter::iterators::LineBreakIterator<'a, 'a, PotentiallyIllFormedUtf8>,
59    );
60
61    #[diplomat::opaque]
62    #[diplomat::rust_link(icu::segmenter::iterators::LineBreakIterator, Struct)]
63    #[diplomat::attr(demo_gen, disable)] // iterator type
64    pub struct LineBreakIteratorUtf16<'a>(
65        icu_segmenter::iterators::LineBreakIterator<'a, 'a, Utf16>,
66    );
67
68    #[diplomat::opaque]
69    #[diplomat::rust_link(icu::segmenter::iterators::LineBreakIterator, Struct)]
70    #[diplomat::attr(demo_gen, disable)] // iterator type
71    pub struct LineBreakIteratorLatin1<'a>(
72        icu_segmenter::iterators::LineBreakIterator<'a, 'a, Latin1>,
73    );
74
75    impl LineSegmenter {
76        /// Construct a [`LineSegmenter`] with default options (no locale-based tailoring) using compiled data. It automatically loads the best
77        /// available payload data for Burmese, Khmer, Lao, and Thai.
78        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_auto, FnInStruct)]
79        #[diplomat::attr(auto, named_constructor = "auto")]
80        #[cfg(feature = "compiled_data")]
81        pub fn create_auto() -> Box<LineSegmenter> {
82            Box::new(LineSegmenter(
83                icu_segmenter::LineSegmenter::new_auto(Default::default()).static_to_owned(),
84            ))
85        }
86
87        /// Construct a [`LineSegmenter`] with default options (no locale-based tailoring) and LSTM payload data for
88        /// Burmese, Khmer, Lao, and Thai, using compiled data.
89        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_lstm, FnInStruct)]
90        #[diplomat::attr(auto, named_constructor = "lstm")]
91        #[cfg(feature = "compiled_data")]
92        pub fn create_lstm() -> Box<LineSegmenter> {
93            Box::new(LineSegmenter(
94                icu_segmenter::LineSegmenter::new_lstm(Default::default()).static_to_owned(),
95            ))
96        }
97
98        /// Construct a [`LineSegmenter`] with default options (no locale-based tailoring) and dictionary payload data for
99        /// Burmese, Khmer, Lao, and Thai, using compiled data
100        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_dictionary, FnInStruct)]
101        #[diplomat::attr(auto, named_constructor = "dictionary")]
102        #[cfg(feature = "compiled_data")]
103        pub fn create_dictionary() -> Box<LineSegmenter> {
104            Box::new(LineSegmenter(
105                icu_segmenter::LineSegmenter::new_dictionary(Default::default()).static_to_owned(),
106            ))
107        }
108
109        /// Construct a [`LineSegmenter`] with default options (no locale-based tailoring) and no support for scripts requiring complex context dependent line breaks
110        /// (Burmese, Khmer, Lao, and Thai), using compiled data
111        #[diplomat::rust_link(
112            icu::segmenter::LineSegmenter::new_for_non_complex_scripts,
113            FnInStruct
114        )]
115        #[diplomat::attr(auto, named_constructor = "for_non_complex_scripts")]
116        #[cfg(feature = "compiled_data")]
117        pub fn create_for_non_complex_scripts() -> Box<LineSegmenter> {
118            Box::new(LineSegmenter(
119                icu_segmenter::LineSegmenter::new_for_non_complex_scripts(Default::default())
120                    .static_to_owned(),
121            ))
122        }
123
124        /// Construct a [`LineSegmenter`] with custom options using compiled data. It automatically loads the best
125        /// available payload data for Burmese, Khmer, Lao, and Thai.
126        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_auto, FnInStruct)]
127        #[diplomat::attr(supports = non_exhaustive_structs, rename = "auto_with_options")]
128        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = named_constructors), named_constructor = "auto_with_options")]
129        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = named_constructors), named_constructor = "auto_with_options_v2")]
130        #[cfg(feature = "compiled_data")]
131        pub fn create_auto_with_options_v2(
132            content_locale: Option<&Locale>,
133            options: LineBreakOptionsV2,
134        ) -> Box<LineSegmenter> {
135            let mut options: LineBreakOptions = options.into();
136            options.content_locale = content_locale.map(|c| &c.0.id);
137            Box::new(LineSegmenter(
138                icu_segmenter::LineSegmenter::new_auto(options).static_to_owned(),
139            ))
140        }
141        /// Construct a [`LineSegmenter`] with custom options. It automatically loads the best
142        /// available payload data for Burmese, Khmer, Lao, and Thai, using a particular data source.
143        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_auto, FnInStruct)]
144        #[diplomat::attr(supports = non_exhaustive_structs, rename = "auto_with_options_and_provider")]
145        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = fallible_constructors, supports = named_constructors), named_constructor = "auto_with_options_and_provider")]
146        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = fallible_constructors, supports = named_constructors), named_constructor = "auto_with_options_v2_and_provider")]
147        #[cfg(feature = "buffer_provider")]
148        pub fn create_auto_with_options_v2_and_provider(
149            provider: &DataProvider,
150            content_locale: Option<&Locale>,
151            options: LineBreakOptionsV2,
152        ) -> Result<Box<LineSegmenter>, DataError> {
153            let mut options: LineBreakOptions = options.into();
154            options.content_locale = content_locale.map(|c| &c.0.id);
155
156            Ok(Box::new(LineSegmenter(
157                icu_segmenter::LineSegmenter::try_new_auto_with_buffer_provider(
158                    provider.get()?,
159                    options,
160                )?,
161            )))
162        }
163        /// Construct a [`LineSegmenter`] with custom options and LSTM payload data for
164        /// Burmese, Khmer, Lao, and Thai, using compiled data.
165        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_lstm, FnInStruct)]
166        #[diplomat::attr(supports = non_exhaustive_structs, rename = "lstm_with_options")]
167        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = named_constructors), named_constructor = "lstm_with_options")]
168        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = named_constructors), named_constructor = "lstm_with_options_v2")]
169        #[cfg(feature = "compiled_data")]
170        pub fn create_lstm_with_options_v2(
171            content_locale: Option<&Locale>,
172            options: LineBreakOptionsV2,
173        ) -> Box<LineSegmenter> {
174            let mut options: LineBreakOptions = options.into();
175            options.content_locale = content_locale.map(|c| &c.0.id);
176
177            Box::new(LineSegmenter(
178                icu_segmenter::LineSegmenter::new_lstm(options).static_to_owned(),
179            ))
180        }
181        /// Construct a [`LineSegmenter`] with custom options and LSTM payload data for
182        /// Burmese, Khmer, Lao, and Thai, using a particular data source.
183        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_lstm, FnInStruct)]
184        #[diplomat::attr(supports = non_exhaustive_structs, rename = "lstm_with_options_and_provider")]
185        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = fallible_constructors, supports = named_constructors), named_constructor = "lstm_with_options_and_provider")]
186        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = fallible_constructors, supports = named_constructors), named_constructor = "lstm_with_options_v2_and_provider")]
187        #[cfg(feature = "buffer_provider")]
188        pub fn create_lstm_with_options_v2_and_provider(
189            provider: &DataProvider,
190            content_locale: Option<&Locale>,
191            options: LineBreakOptionsV2,
192        ) -> Result<Box<LineSegmenter>, DataError> {
193            let mut options: LineBreakOptions = options.into();
194            options.content_locale = content_locale.map(|c| &c.0.id);
195
196            Ok(Box::new(LineSegmenter(
197                icu_segmenter::LineSegmenter::try_new_lstm_with_buffer_provider(
198                    provider.get()?,
199                    options,
200                )?,
201            )))
202        }
203        /// Construct a [`LineSegmenter`] with custom options and dictionary payload data for
204        /// Burmese, Khmer, Lao, and Thai, using compiled data.
205        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_dictionary, FnInStruct)]
206        #[diplomat::attr(supports = non_exhaustive_structs, rename = "dictionary_with_options")]
207        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = named_constructors), named_constructor = "dictionary_with_options")]
208        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = named_constructors), named_constructor = "dictionary_with_options_v2")]
209        #[cfg(feature = "compiled_data")]
210        pub fn create_dictionary_with_options_v2(
211            content_locale: Option<&Locale>,
212            options: LineBreakOptionsV2,
213        ) -> Box<LineSegmenter> {
214            let mut options: LineBreakOptions = options.into();
215            options.content_locale = content_locale.map(|c| &c.0.id);
216
217            Box::new(LineSegmenter(
218                icu_segmenter::LineSegmenter::new_dictionary(options).static_to_owned(),
219            ))
220        }
221        /// Construct a [`LineSegmenter`] with custom options and dictionary payload data for
222        /// Burmese, Khmer, Lao, and Thai, using a particular data source.
223        #[diplomat::rust_link(icu::segmenter::LineSegmenter::new_dictionary, FnInStruct)]
224        #[diplomat::attr(supports = non_exhaustive_structs, rename = "dictionary_with_options_and_provider")]
225        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = fallible_constructors, supports = named_constructors), named_constructor = "dictionary_with_options_and_provider")]
226        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = fallible_constructors, supports = named_constructors), named_constructor = "dictionary_with_options_v2_and_provider")]
227        #[cfg(feature = "buffer_provider")]
228        pub fn create_dictionary_with_options_v2_and_provider(
229            provider: &DataProvider,
230            content_locale: Option<&Locale>,
231            options: LineBreakOptionsV2,
232        ) -> Result<Box<LineSegmenter>, DataError> {
233            let mut options: LineBreakOptions = options.into();
234            options.content_locale = content_locale.map(|c| &c.0.id);
235
236            Ok(Box::new(LineSegmenter(
237                icu_segmenter::LineSegmenter::try_new_dictionary_with_buffer_provider(
238                    provider.get()?,
239                    options,
240                )?,
241            )))
242        }
243        /// Construct a [`LineSegmenter`] with custom options and no support for scripts requiring complex context dependent line breaks
244        /// (Burmese, Khmer, Lao, and Thai), using compiled data.
245        #[diplomat::rust_link(
246            icu::segmenter::LineSegmenter::new_for_non_complex_scripts,
247            FnInStruct
248        )]
249        #[diplomat::attr(supports = non_exhaustive_structs, rename = "for_non_complex_scripts_with_options")]
250        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = named_constructors), named_constructor = "for_non_complex_scripts_with_options")]
251        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = named_constructors), named_constructor = "for_non_complex_scripts_with_options_v2")]
252        #[cfg(feature = "compiled_data")]
253        pub fn create_for_non_complex_scripts_with_options_v2(
254            content_locale: Option<&Locale>,
255            options: LineBreakOptionsV2,
256        ) -> Box<LineSegmenter> {
257            let mut options: LineBreakOptions = options.into();
258            options.content_locale = content_locale.map(|c| &c.0.id);
259
260            Box::new(LineSegmenter(
261                icu_segmenter::LineSegmenter::new_for_non_complex_scripts(options)
262                    .static_to_owned(),
263            ))
264        }
265        /// Construct a [`LineSegmenter`] with custom options and no support for complex languages
266        /// (Burmese, Khmer, Lao, and Thai), using a particular data source.
267        #[diplomat::rust_link(
268            icu::segmenter::LineSegmenter::new_for_non_complex_scripts,
269            FnInStruct
270        )]
271        #[diplomat::attr(supports = non_exhaustive_structs, rename = "for_non_complex_scripts_with_options_and_provider")]
272        #[diplomat::attr(all(supports = non_exhaustive_structs, supports = fallible_constructors, supports = named_constructors), named_constructor = "for_non_complex_scripts_with_options_and_provider")]
273        #[diplomat::attr(all(not(supports = non_exhaustive_structs), supports = fallible_constructors, supports = named_constructors), named_constructor = "for_non_complex_scripts_with_options_v2_and_provider")]
274        #[cfg(feature = "buffer_provider")]
275        pub fn create_for_non_complex_scripts_with_options_v2_and_provider(
276            provider: &DataProvider,
277            content_locale: Option<&Locale>,
278            options: LineBreakOptionsV2,
279        ) -> Result<Box<LineSegmenter>, DataError> {
280            let mut options: LineBreakOptions = options.into();
281            options.content_locale = content_locale.map(|c| &c.0.id);
282
283            Ok(Box::new(LineSegmenter(
284                icu_segmenter::LineSegmenter::try_new_for_non_complex_scripts_with_buffer_provider(
285                    provider.get()?,
286                    options,
287                )?,
288            )))
289        }
290        /// Segments a string.
291        ///
292        /// Ill-formed input is treated as if errors had been replaced with REPLACEMENT CHARACTERs according
293        /// to the WHATWG Encoding Standard.
294        #[diplomat::rust_link(icu::segmenter::LineSegmenterBorrowed::segment_utf8, FnInStruct)]
295        #[diplomat::rust_link(
296            icu::segmenter::LineSegmenterBorrowed::segment_str,
297            FnInStruct,
298            hidden
299        )]
300        #[diplomat::attr(not(supports = utf8_strings), disable)]
301        #[diplomat::attr(*, rename = "segment")]
302        pub fn segment_utf8<'a>(
303            &'a self,
304            input: &'a DiplomatStr,
305        ) -> Box<LineBreakIteratorUtf8<'a>> {
306            Box::new(LineBreakIteratorUtf8(
307                self.0.as_borrowed().segment_utf8(input),
308            ))
309        }
310
311        /// Segments a string.
312        ///
313        /// Ill-formed input is treated as if errors had been replaced with REPLACEMENT CHARACTERs according
314        /// to the WHATWG Encoding Standard.
315        #[diplomat::rust_link(icu::segmenter::LineSegmenterBorrowed::segment_utf16, FnInStruct)]
316        #[diplomat::attr(not(supports = utf8_strings), rename = "segment")]
317        #[diplomat::attr(supports = utf8_strings, rename = "segment16")]
318        pub fn segment_utf16<'a>(
319            &'a self,
320            input: &'a DiplomatStr16,
321        ) -> Box<LineBreakIteratorUtf16<'a>> {
322            Box::new(LineBreakIteratorUtf16(
323                self.0.as_borrowed().segment_utf16(input),
324            ))
325        }
326
327        /// Segments a Latin-1 string.
328        #[diplomat::rust_link(icu::segmenter::LineSegmenterBorrowed::segment_latin1, FnInStruct)]
329        #[diplomat::attr(not(supports = utf8_strings), disable)]
330        pub fn segment_latin1<'a>(&'a self, input: &'a [u8]) -> Box<LineBreakIteratorLatin1<'a>> {
331            Box::new(LineBreakIteratorLatin1(
332                self.0.as_borrowed().segment_latin1(input),
333            ))
334        }
335    }
336
337    impl<'a> LineBreakIteratorUtf8<'a> {
338        /// Finds the next breakpoint. Returns -1 if at the end of the string or if the index is
339        /// out of range of a 32-bit signed integer.
340        #[diplomat::rust_link(icu::segmenter::iterators::LineBreakIterator::next, FnInStruct)]
341        pub fn next(&mut self) -> i32 {
342            self.0
343                .next()
344                .and_then(|u| i32::try_from(u).ok())
345                .unwrap_or(-1)
346        }
347    }
348
349    impl<'a> LineBreakIteratorUtf16<'a> {
350        /// Finds the next breakpoint. Returns -1 if at the end of the string or if the index is
351        /// out of range of a 32-bit signed integer.
352        #[diplomat::rust_link(icu::segmenter::iterators::LineBreakIterator::next, FnInStruct)]
353        #[diplomat::rust_link(
354            icu::segmenter::iterators::LineBreakIterator::Item,
355            AssociatedTypeInStruct,
356            hidden
357        )]
358        pub fn next(&mut self) -> i32 {
359            self.0
360                .next()
361                .and_then(|u| i32::try_from(u).ok())
362                .unwrap_or(-1)
363        }
364    }
365
366    impl<'a> LineBreakIteratorLatin1<'a> {
367        /// Finds the next breakpoint. Returns -1 if at the end of the string or if the index is
368        /// out of range of a 32-bit signed integer.
369        #[diplomat::rust_link(icu::segmenter::iterators::LineBreakIterator::next, FnInStruct)]
370        #[diplomat::rust_link(
371            icu::segmenter::iterators::LineBreakIterator::Item,
372            AssociatedTypeInStruct,
373            hidden
374        )]
375        pub fn next(&mut self) -> i32 {
376            self.0
377                .next()
378                .and_then(|u| i32::try_from(u).ok())
379                .unwrap_or(-1)
380        }
381    }
382}
383
384impl From<ffi::LineBreakOptionsV2> for icu_segmenter::options::LineBreakOptions<'_> {
385    fn from(other: ffi::LineBreakOptionsV2) -> Self {
386        let mut options = icu_segmenter::options::LineBreakOptions::default();
387        options.strictness = other.strictness.into_converted_option();
388        options.word_option = other.word_option.into_converted_option();
389        options
390    }
391}
icu_capi/segmenter_line.rs

icu_capi/
segmenter_line.rs
