icu_normalizer/
provider.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5//! 🚧 \[Unstable\] Data provider struct definitions for this ICU4X component.
6//!
7//! <div class="stab unstable">
8//! 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
9//! including in SemVer minor releases. While the serde representation of data structs is guaranteed
10//! to be stable, their Rust representation might not be. Use with caution.
11//! </div>
12//!
13//! Read more about data providers: [`icu_provider`]
14
15// Provider structs must be stable
16#![allow(clippy::exhaustive_structs, clippy::exhaustive_enums)]
17
18use icu_collections::char16trie::Char16Trie;
19use icu_collections::codepointtrie::CodePointTrie;
20use icu_provider::prelude::*;
21use zerovec::ZeroVec;
22
23#[cfg(feature = "compiled_data")]
24#[derive(Debug)]
25/// Baked data
26///
27/// <div class="stab unstable">
28/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
29/// including in SemVer minor releases. In particular, the `DataProvider` implementations are only
30/// guaranteed to match with this version's `*_unstable` providers. Use with caution.
31/// </div>
32pub struct Baked;
33
34#[cfg(feature = "compiled_data")]
35#[allow(unused_imports)]
36const _: () = {
37    use icu_normalizer_data::*;
38    pub mod icu {
39        pub use crate as normalizer;
40        pub use icu_collections as collections;
41    }
42    make_provider!(Baked);
43    impl_normalizer_nfc_v1!(Baked);
44    impl_normalizer_nfc_v2!(Baked);
45    impl_normalizer_nfd_data_v1!(Baked);
46    impl_normalizer_nfd_supplement_v1!(Baked);
47    impl_normalizer_nfd_tables_v1!(Baked);
48    impl_normalizer_nfkd_data_v1!(Baked);
49    impl_normalizer_nfkd_tables_v1!(Baked);
50    impl_normalizer_uts46_data_v1!(Baked);
51};
52
53icu_provider::data_marker!(
54    /// Marker for data for canonical decomposition.
55    NormalizerNfdDataV1,
56    "normalizer/nfd/data/v1",
57    DecompositionData<'static>,
58    is_singleton = true
59);
60icu_provider::data_marker!(
61    /// Marker for additional data for canonical decomposition.
62    NormalizerNfdTablesV1,
63    "normalizer/nfd/tables/v1",
64    DecompositionTables<'static>,
65    is_singleton = true
66);
67icu_provider::data_marker!(
68    /// Marker for data for compatibility decomposition.
69    NormalizerNfkdDataV1,
70    "normalizer/nfkd/data/v1",
71    DecompositionData<'static>,
72    is_singleton = true
73);
74icu_provider::data_marker!(
75    /// Marker for additional data for compatibility decomposition.
76    NormalizerNfkdTablesV1,
77    "normalizer/nfkd/tables/v1",
78    DecompositionTables<'static>,
79    is_singleton = true
80);
81icu_provider::data_marker!(
82    /// Marker for data for UTS-46 decomposition.
83    NormalizerUts46DataV1,
84    "normalizer/uts46/data/v1",
85    DecompositionData<'static>,
86    is_singleton = true
87);
88icu_provider::data_marker!(
89    /// Marker for data for composition.
90    NormalizerNfcV1,
91    "normalizer/nfc/v1",
92    CanonicalCompositions<'static>,
93    is_singleton = true
94);
95icu_provider::data_marker!(
96    /// Marker for data for composition.
97    NormalizerNfcV2,
98    "normalizer/nfc/v2",
99    CanonicalCompositionsNew<'static>,
100    is_singleton = true
101);
102icu_provider::data_marker!(
103    /// Marker for additional data for non-recusrsive composition.
104    NormalizerNfdSupplementV1,
105    "normalizer/nfd/supplement/v1",
106    NonRecursiveDecompositionSupplement<'static>,
107    is_singleton = true
108);
109
110#[cfg(feature = "datagen")]
111/// The latest minimum set of markers required by this component.
112pub const MARKERS: &[DataMarkerInfo] = &[
113    NormalizerNfcV1::INFO,
114    NormalizerNfcV2::INFO,
115    NormalizerNfdDataV1::INFO,
116    NormalizerNfdTablesV1::INFO,
117    NormalizerNfkdDataV1::INFO,
118    NormalizerNfkdTablesV1::INFO,
119    NormalizerNfdSupplementV1::INFO,
120    NormalizerUts46DataV1::INFO,
121];
122
123/// Decomposition data
124///
125/// <div class="stab unstable">
126/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
127/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
128/// to be stable, their Rust representation might not be. Use with caution.
129/// </div>
130#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
131#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
132#[cfg_attr(feature = "datagen", databake(path = icu_normalizer::provider))]
133#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
134pub struct DecompositionData<'data> {
135    /// Trie for decomposition.
136    #[cfg_attr(feature = "serde", serde(borrow))]
137    pub trie: CodePointTrie<'data, u32>,
138    /// The passthrough bounds of NFD/NFC are lowered to this
139    /// maximum instead. (16-bit, because cannot be higher
140    /// than 0x0300, which is the bound for NFC.)
141    pub passthrough_cap: u16,
142}
143
144icu_provider::data_struct!(
145    DecompositionData<'_>,
146    #[cfg(feature = "datagen")]
147);
148
149/// The expansion tables for cases where the decomposition isn't
150/// contained in the trie value
151///
152/// <div class="stab unstable">
153/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
154/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
155/// to be stable, their Rust representation might not be. Use with caution.
156/// </div>
157#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
158#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
159#[cfg_attr(feature = "datagen", databake(path = icu_normalizer::provider))]
160#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
161pub struct DecompositionTables<'data> {
162    /// Decompositions that are fully within the BMP
163    #[cfg_attr(feature = "serde", serde(borrow))]
164    pub scalars16: ZeroVec<'data, u16>,
165    /// Decompositions with at least one character outside
166    /// the BMP
167    #[cfg_attr(feature = "serde", serde(borrow))]
168    pub scalars24: ZeroVec<'data, char>,
169}
170
171icu_provider::data_struct!(
172    DecompositionTables<'_>,
173    #[cfg(feature = "datagen")]
174);
175
176/// Non-Hangul canonical compositions, old format
177///
178/// <div class="stab unstable">
179/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
180/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
181/// to be stable, their Rust representation might not be. Use with caution.
182/// </div>
183#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
184#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
185#[cfg_attr(feature = "datagen", databake(path = icu_normalizer::provider))]
186#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
187pub struct CanonicalCompositions<'data> {
188    /// Trie keys are two-`char` strings with the second
189    /// character coming first. The value, if any, is the
190    /// (non-Hangul) canonical composition.
191    #[cfg_attr(feature = "serde", serde(borrow))]
192    pub canonical_compositions: Char16Trie<'data>,
193}
194
195icu_provider::data_struct!(
196    CanonicalCompositions<'_>,
197    #[cfg(feature = "datagen")]
198);
199
200/// Non-Hangul canonical compositions, new format
201///
202/// <div class="stab unstable">
203/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
204/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
205/// to be stable, their Rust representation might not be. Use with caution.
206/// </div>
207#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
208#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
209#[cfg_attr(feature = "datagen", databake(path = icu_normalizer::provider))]
210#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
211pub struct CanonicalCompositionsNew<'data> {
212    /// Trie from primary character to index into `linear`.
213    ///
214    /// If the first character of the pair is
215    /// `'a' | 'e' | 'i' | 'o' | 'u' | 'A' | 'E' | 'I' | 'O' | 'U'`,
216    /// the primary character is the second item in the pair. Otherwise,
217    /// the primary character is the first item in the pair.
218    #[cfg_attr(feature = "serde", serde(borrow))]
219    pub trie: CodePointTrie<'data, u16>,
220
221    /// Compositions that are fully within the BMP
222    #[cfg_attr(feature = "serde", serde(borrow))]
223    pub linear16: ZeroVec<'data, (u16, u16)>,
224    /// Compositions with at least one character outside
225    /// the BMP
226    #[cfg_attr(feature = "serde", serde(borrow))]
227    pub linear24: ZeroVec<'data, (char, char)>,
228}
229
230icu_provider::data_struct!(
231    CanonicalCompositionsNew<'_>,
232    #[cfg(feature = "datagen")]
233);
234
235/// Non-recursive canonical decompositions that differ from
236/// `DecompositionData`.
237///
238/// <div class="stab unstable">
239/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
240/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
241/// to be stable, their Rust representation might not be. Use with caution.
242/// </div>
243#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
244#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
245#[cfg_attr(feature = "datagen", databake(path = icu_normalizer::provider))]
246#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
247pub struct NonRecursiveDecompositionSupplement<'data> {
248    /// Trie for the supplementary non-recursive decompositions
249    #[cfg_attr(feature = "serde", serde(borrow))]
250    pub trie: CodePointTrie<'data, u32>,
251    /// Decompositions with at least one character outside
252    /// the BMP
253    #[cfg_attr(feature = "serde", serde(borrow))]
254    pub scalars24: ZeroVec<'data, char>,
255}
256
257icu_provider::data_struct!(
258    NonRecursiveDecompositionSupplement<'_>,
259    #[cfg(feature = "datagen")]
260);
icu_normalizer/provider.rs

icu_normalizer/
provider.rs
