icu_collections/codepointtrie/cptrie.rs
1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use crate::codepointtrie::error::Error;
6use crate::codepointtrie::impl_const::*;
7
8#[cfg(feature = "alloc")]
9use crate::codepointinvlist::CodePointInversionList;
10use core::char::CharTryFromError;
11use core::convert::Infallible;
12use core::convert::TryFrom;
13use core::fmt::Display;
14#[cfg(feature = "alloc")]
15use core::iter::FromIterator;
16use core::num::TryFromIntError;
17use core::ops::RangeInclusive;
18use yoke::Yokeable;
19use zerofrom::ZeroFrom;
20use zerovec::ule::AsULE;
21#[cfg(feature = "alloc")]
22use zerovec::ule::UleError;
23use zerovec::ZeroSlice;
24use zerovec::ZeroVec;
25
26/// The type of trie represents whether the trie has an optimization that
27/// would make it smaller or faster.
28///
29/// Regarding performance, a trie being a small or fast type affects the number of array lookups
30/// needed for code points in the range `[0x1000, 0x10000)`. In this range, `Small` tries use 4 array lookups,
31/// while `Fast` tries use 2 array lookups.
32/// Code points before the interval (in `[0, 0x1000)`) will always use 2 array lookups.
33/// Code points after the interval (in `[0x10000, 0x10FFFF]`) will always use 4 array lookups.
34///
35/// Regarding size, `Fast` type tries are larger than `Small` type tries because the minimum size of
36/// the index array is larger. The minimum size is the "fast max" limit, which is the limit of the range
37/// of code points with 2 array lookups.
38///
39/// See the document [Unicode Properties and Code Point Tries in ICU4X](https://github.com/unicode-org/icu4x/blob/main/documents/design/properties_code_point_trie.md).
40///
41/// Also see [`UCPTrieType`](https://unicode-org.github.io/icu-docs/apidoc/dev/icu4c/ucptrie_8h.html) in ICU4C.
42#[derive(Clone, Copy, PartialEq, Debug, Eq)]
43#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
44#[cfg_attr(feature = "databake", derive(databake::Bake))]
45#[cfg_attr(feature = "databake", databake(path = icu_collections::codepointtrie))]
46#[allow(clippy::exhaustive_enums)] // based on a stable serialized form
47pub enum TrieType {
48 /// Represents the "fast" type code point tries for the
49 /// [`TrieType`] trait. The "fast max" limit is set to `0xffff`.
50 Fast = 0,
51 /// Represents the "small" type code point tries for the
52 /// [`TrieType`] trait. The "fast max" limit is set to `0x0fff`.
53 Small = 1,
54}
55
56// TrieValue trait
57
58// AsULE is AsUnalignedLittleEndian, i.e. "allowed in a zerovec"
59
60/// A trait representing the values stored in the data array of a [`CodePointTrie`].
61/// This trait is used as a type parameter in constructing a `CodePointTrie`.
62///
63/// This trait can be implemented on anything that can be represented as a u32s worth of data.
64pub trait TrieValue: Copy + Eq + PartialEq + AsULE + 'static {
65 /// Last-resort fallback value to return if we cannot read data from the trie.
66 ///
67 /// In most cases, the error value is read from the last element of the `data` array,
68 /// this value is used for empty codepointtrie arrays
69 /// Error type when converting from a u32 to this `TrieValue`.
70 type TryFromU32Error: Display;
71 /// A parsing function that is primarily motivated by deserialization contexts.
72 /// When the serialization type width is smaller than 32 bits, then it is expected
73 /// that the call site will widen the value to a `u32` first.
74 fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error>;
75
76 /// A method for converting back to a `u32` that can roundtrip through
77 /// [`Self::try_from_u32()`]. The default implementation of this trait
78 /// method panics in debug mode and returns 0 in release mode.
79 ///
80 /// This method is allowed to have GIGO behavior when fed a value that has
81 /// no corresponding `u32` (since such values cannot be stored in the trie)
82 fn to_u32(self) -> u32;
83}
84
85macro_rules! impl_primitive_trie_value {
86 ($primitive:ty, $error:ty) => {
87 impl TrieValue for $primitive {
88 type TryFromU32Error = $error;
89 fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
90 Self::try_from(i)
91 }
92
93 #[allow(trivial_numeric_casts)]
94 fn to_u32(self) -> u32 {
95 // bitcast when the same size, zero-extend/sign-extend
96 // when not the same size
97 self as u32
98 }
99 }
100 };
101}
102
103impl_primitive_trie_value!(u8, TryFromIntError);
104impl_primitive_trie_value!(u16, TryFromIntError);
105impl_primitive_trie_value!(u32, Infallible);
106impl_primitive_trie_value!(i8, TryFromIntError);
107impl_primitive_trie_value!(i16, TryFromIntError);
108impl_primitive_trie_value!(i32, TryFromIntError);
109impl_primitive_trie_value!(char, CharTryFromError);
110
111/// Helper function used by [`get_range`]. Converts occurrences of trie's null
112/// value into the provided `null_value`.
113///
114/// Note: the ICU version of this helper function uses a `ValueFilter` function
115/// to apply a transform on a non-null value. But currently, this implementation
116/// stops short of that functionality, and instead leaves the non-null trie value
117/// untouched. This is equivalent to having a `ValueFilter` function that is the
118/// identity function.
119fn maybe_filter_value<T: TrieValue>(value: T, trie_null_value: T, null_value: T) -> T {
120 if value == trie_null_value {
121 null_value
122 } else {
123 value
124 }
125}
126
127/// This struct represents a de-serialized [`CodePointTrie`] that was exported from
128/// ICU binary data.
129///
130/// For more information:
131/// - [ICU Site design doc](https://unicode-org.github.io/icu/design/struct/utrie)
132/// - [ICU User Guide section on Properties lookup](https://unicode-org.github.io/icu/userguide/strings/properties.html#lookup)
133// serde impls in crate::serde
134#[derive(Debug, Eq, PartialEq, Yokeable, ZeroFrom)]
135pub struct CodePointTrie<'trie, T: TrieValue> {
136 /// # Safety Invariant
137 ///
138 /// The value of `header.trie_type` must not change after construction.
139 pub(crate) header: CodePointTrieHeader,
140 /// # Safety Invariant
141 ///
142 /// If `header.trie_type == TrieType::Fast`, `index.len()` must be greater
143 /// than `FAST_TYPE_FAST_INDEXING_MAX`. Otherwise, `index.len()`
144 /// must be greater than `SMALL_TYPE_FAST_INDEXING_MAX`. Furthermore,
145 /// this field must not change after construction. (Strictly: It must
146 /// not become shorter than the length requirement stated above and the
147 /// values within the prefix up to the length requirement must not change.)
148 pub(crate) index: ZeroVec<'trie, u16>,
149 /// # Safety Invariant
150 ///
151 /// If `header.trie_type == TrieType::Fast`, `data.len()` must be greater
152 /// than `FAST_TYPE_DATA_MASK` plus the largest value in
153 /// `index[0..FAST_TYPE_FAST_INDEXING_MAX + 1]`. Otherwise, `data.len()`
154 /// must be greater than `FAST_TYPE_DATA_MASK` plus the largest value in
155 /// `index[0..SMALL_TYPE_FAST_INDEXING_MAX + 1]`. Furthermore, this field
156 /// must not change after construction. (Strictly: The stated length
157 /// requirement must continue to hold.)
158 pub(crate) data: ZeroVec<'trie, T>,
159 // serde impl skips this field
160 #[zerofrom(clone)] // TrieValue is Copy, this allows us to avoid
161 // a T: ZeroFrom bound
162 pub(crate) error_value: T,
163}
164
165/// This struct contains the fixed-length header fields of a [`CodePointTrie`].
166///
167/// # Safety Invariant
168///
169/// The `trie_type` field must not change after construction.
170///
171/// (In practice, all the fields here remain unchanged during the lifetime
172/// of the trie.)
173#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
174#[cfg_attr(feature = "databake", derive(databake::Bake))]
175#[cfg_attr(feature = "databake", databake(path = icu_collections::codepointtrie))]
176#[derive(Copy, Clone, Debug, Eq, PartialEq, Yokeable, ZeroFrom)]
177#[allow(clippy::exhaustive_structs)] // based on a stable serialized form
178pub struct CodePointTrieHeader {
179 /// The code point of the start of the last range of the trie. A
180 /// range is defined as a partition of the code point space such that the
181 /// value in this trie associated with all code points of the same range is
182 /// the same.
183 ///
184 /// For the property value data for many Unicode properties,
185 /// often times, `high_start` is `U+10000` or lower. In such cases, not
186 /// reserving space in the `index` array for duplicate values is a large
187 /// savings. The "highValue" associated with the `high_start` range is
188 /// stored at the second-to-last position of the `data` array.
189 /// (See `impl_const::HIGH_VALUE_NEG_DATA_OFFSET`.)
190 pub high_start: u32,
191 /// A version of the `high_start` value that is right-shifted 12 spaces,
192 /// but is rounded up to a multiple `0x1000` for easy testing from UTF-8
193 /// lead bytes.
194 pub shifted12_high_start: u16,
195 /// Offset for the null block in the "index-3" table of the `index` array.
196 /// Set to an impossibly high value (e.g., `0xffff`) if there is no
197 /// dedicated index-3 null block.
198 pub index3_null_offset: u16,
199 /// Internal data null block offset, not shifted.
200 /// Set to an impossibly high value (e.g., `0xfffff`) if there is no
201 /// dedicated data null block.
202 pub data_null_offset: u32,
203 /// The value stored in the trie that represents a null value being
204 /// associated to a code point.
205 pub null_value: u32,
206 /// The enum value representing the type of trie, where trie type is as it
207 /// is defined in ICU (ex: Fast, Small).
208 ///
209 /// # Safety Invariant
210 ///
211 /// Must not change after construction.
212 pub trie_type: TrieType,
213}
214
215impl TryFrom<u8> for TrieType {
216 type Error = Error;
217
218 fn try_from(trie_type_int: u8) -> Result<TrieType, Error> {
219 match trie_type_int {
220 0 => Ok(TrieType::Fast),
221 1 => Ok(TrieType::Small),
222 _ => Err(Error::FromDeserialized {
223 reason: "Cannot parse value for trie_type",
224 }),
225 }
226 }
227}
228
229// Helper macro that turns arithmetic into wrapping-in-release, checked-in-debug arithmetic
230//
231// This is rustc's default behavior anyway, however some projects like Android deliberately
232// enable overflow checks. CodePointTrie::get() is intended to be used in Android bionic which
233// cares about codesize and we don't want the pile of panicking infrastructure brought in by overflow
234// checks, so we force wrapping in release.
235// See #6052
236macro_rules! w(
237 // Note: these matchers are not perfect since you cannot have an operator after an expr matcher
238 // Use variables if you need complex first operands.
239 ($a:tt + $b:expr) => {
240 {
241 #[allow(unused_parens)]
242 let a = $a;
243 let b = $b;
244 debug_assert!(a.checked_add(b).is_some());
245 $a.wrapping_add($b)
246 }
247 };
248 ($a:tt - $b:expr) => {
249
250 {
251 #[allow(unused_parens)]
252 let a = $a;
253 let b = $b;
254 debug_assert!(a.checked_sub(b).is_some());
255 $a.wrapping_sub($b)
256 }
257 };
258 ($a:tt * $b:expr) => {
259 {
260 #[allow(unused_parens)]
261 let a = $a;
262 let b = $b;
263 debug_assert!(a.checked_mul(b).is_some());
264 $a.wrapping_mul($b)
265 }
266 };
267);
268
269impl<'trie, T: TrieValue> CodePointTrie<'trie, T> {
270 #[doc(hidden)] // databake internal
271 /// # Safety
272 ///
273 /// `header.trie_type`, `index`, and `data` must
274 /// satisfy the invariants for the fields of the
275 /// same names on `CodePointTrie`.
276 pub const unsafe fn from_parts_unstable_unchecked_v1(
277 header: CodePointTrieHeader,
278 index: ZeroVec<'trie, u16>,
279 data: ZeroVec<'trie, T>,
280 error_value: T,
281 ) -> Self {
282 // Field invariants upheld: The caller is responsible.
283 // In practice, this means that datagen in the databake
284 // mode upholds these invariants when constructing the
285 // `CodePointTrie` that is then baked.
286 Self {
287 header,
288 index,
289 data,
290 error_value,
291 }
292 }
293
294 /// Returns a new [`CodePointTrie`] backed by borrowed data for the `index`
295 /// array and `data` array, whose data values have width `W`.
296 pub fn try_new(
297 header: CodePointTrieHeader,
298 index: ZeroVec<'trie, u16>,
299 data: ZeroVec<'trie, T>,
300 ) -> Result<CodePointTrie<'trie, T>, Error> {
301 let error_value = Self::validate_fields(&header, &index, &data)?;
302 // Field invariants upheld: Checked by `validate_fields` above.
303 let trie: CodePointTrie<'trie, T> = CodePointTrie {
304 header,
305 index,
306 data,
307 error_value,
308 };
309 Ok(trie)
310 }
311
312 /// Checks the invariant on the fields that fast-path access relies on for
313 /// safety in order to omit slice bound checks and upon success returns the
314 /// `error_value` for the trie.
315 ///
316 /// # Safety Usable Invariant
317 ///
318 /// Iff this function returns `Ok(T)`, the arguments satisfy the invariants
319 /// for corresponding fields of `CodePointTrie`. (Other than proving that
320 /// nothing else changes the fields subsequently.)
321 pub(crate) fn validate_fields(
322 header: &CodePointTrieHeader,
323 index: &ZeroSlice<u16>,
324 data: &ZeroSlice<T>,
325 ) -> Result<T, Error> {
326 let error_value = data.last().ok_or(Error::EmptyDataVector)?;
327
328 // `CodePointTrie` lookup has two stages: fast and small (the trie types
329 // are also fast and small; they affect where the boundary between fast
330 // and small lookups is).
331 //
332 // The length requirements for `index` and `data` are checked here only
333 // for the fast lookup case so that the fast lookup can omit bound checks
334 // at the time of access. In the small lookup case, bounds are checked at
335 // the time of access.
336 //
337 // The fast lookup happens on the prefixes of `index` and `data` with
338 // more items for the small lookup case afterwards. The check here
339 // only looks at the prefixes relevant to the fast case.
340 //
341 // In the fast lookup case, the bits of the of the code point are
342 // partitioned into a bit prefix and a bit suffix. First, a value
343 // is read from `index` by indexing into it using the bit prefix.
344 // Then `data` is accessed by the value just read from `index` plus
345 // the bit suffix.
346 //
347 // Therefore, the length of `index` needs to accommodate access
348 // by the maximum possible bit prefix, and the length of `data`
349 // needs to accommodate access by the largest value stored in the part
350 // of `data` reachable by the bit prefix plus the maximum possible bit
351 // suffix.
352 //
353 // The maximum possible bit prefix depends on the trie type.
354
355 // The maximum code point that can be used for fast-path access:
356 let fast_max = match header.trie_type {
357 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
358 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
359 };
360 // Keep only the prefix bits:
361 let max_bit_prefix = fast_max >> FAST_TYPE_SHIFT;
362 // Attempt slice the part of `index` that the fast path can index into.
363 // Since `max_bit_prefix` is the largest possible value used for
364 // indexing (inclusive bound), we need to add one to get the exclusive
365 // bound, which is what `get_subslice` wants.
366 let fast_index = index
367 .get_subslice(0..(max_bit_prefix as usize) + 1)
368 .ok_or(Error::IndexTooShortForFastAccess)?;
369 // Invariant upheld for `index`: If we got this far, the length of `index`
370 // satisfies its length invariant on the assumption that `header.trie_type`
371 // will not change subsequently.
372
373 // Now find the largest offset in the part of `index` reachable by the
374 // bit prefix. `max` can never actually return `None`, since we already
375 // know the slice isn't empty. Hence, reusing the error kind instead of
376 // minting a new one for this check.
377 let max_offset = fast_index
378 .iter()
379 .max()
380 .ok_or(Error::IndexTooShortForFastAccess)?;
381 // `FAST_TYPE_DATA_MASK` is the maximum possible bit suffix, since the
382 // maximum is when all the bits in the suffix are set, and the mask
383 // has that many bits set.
384 if (max_offset) as usize + (FAST_TYPE_DATA_MASK as usize) >= data.len() {
385 return Err(Error::DataTooShortForFastAccess);
386 }
387
388 // The builder is supposed to support direct indexing to the data array
389 // by ASCII.
390 if data.len() < 128 {
391 return Err(Error::DataTooShortForAsciiAccess);
392 }
393
394 // Invariant upheld for `data`: If we got this far, the length of `data`
395 // satisfies `data`'s length invariant on the assumption that the contents
396 // of `fast_index` subslice of `index` and `header.trie_type` will not
397 // change subsequently.
398
399 Ok(error_value)
400 }
401
402 /// Turns this trie into a version whose trie type is encoded in the Rust type.
403 #[inline]
404 pub const fn to_typed(
405 self,
406 ) -> Typed<FastCodePointTrie<'trie, T>, SmallCodePointTrie<'trie, T>> {
407 match self.header.trie_type {
408 TrieType::Fast => Typed::Fast(FastCodePointTrie { inner: self }),
409 TrieType::Small => Typed::Small(SmallCodePointTrie { inner: self }),
410 }
411 }
412
413 /// Obtains a reference to this trie as a Rust type that encodes the trie type in the Rust type.
414 #[inline]
415 pub fn as_typed_ref(
416 &self,
417 ) -> Typed<&FastCodePointTrie<'trie, T>, &SmallCodePointTrie<'trie, T>> {
418 // SAFETY: `FastCodePointTrie` and `SmallCodePointTrie` are `repr(transparent)`
419 // with `CodePointTrie`, so transmuting between the references is OK when the
420 // actual trie type agrees with the semantics of the typed wrapper.
421 match self.header.trie_type {
422 TrieType::Fast => Typed::Fast(unsafe {
423 core::mem::transmute::<&CodePointTrie<'trie, T>, &FastCodePointTrie<'trie, T>>(self)
424 }),
425 TrieType::Small => Typed::Small(unsafe {
426 core::mem::transmute::<&CodePointTrie<'trie, T>, &SmallCodePointTrie<'trie, T>>(
427 self,
428 )
429 }),
430 }
431 }
432
433 /// Returns the position in the data array containing the trie's stored
434 /// error value.
435 #[inline(always)] // `always` was based on previous normalizer benchmarking
436 fn trie_error_val_index(&self) -> u32 {
437 // We use wrapping_sub here to avoid panicky overflow checks.
438 // len should always be > 1, but if it isn't this will just cause GIGO behavior of producing
439 // None on `.get()`
440 debug_assert!(self.data.len() as u32 >= ERROR_VALUE_NEG_DATA_OFFSET);
441 w!((self.data.len() as u32) - ERROR_VALUE_NEG_DATA_OFFSET)
442 }
443
444 fn internal_small_index(&self, code_point: u32) -> u32 {
445 // We use wrapping arithmetic here to avoid overflow checks making their way into binaries
446 // with overflow checks enabled. Ultimately this code ends up as a checked index, so any
447 // bugs here will cause GIGO
448 let mut index1_pos: u32 = code_point >> SHIFT_1;
449 if self.header.trie_type == TrieType::Fast {
450 debug_assert!(
451 FAST_TYPE_FAST_INDEXING_MAX < code_point && code_point < self.header.high_start
452 );
453 index1_pos = w!(index1_pos + BMP_INDEX_LENGTH - OMITTED_BMP_INDEX_1_LENGTH);
454 } else {
455 assert!(code_point < self.header.high_start && self.header.high_start > SMALL_LIMIT);
456 index1_pos = w!(index1_pos + SMALL_INDEX_LENGTH);
457 }
458 let index1_val = if let Some(index1_val) = self.index.get(index1_pos as usize) {
459 index1_val
460 } else {
461 return self.trie_error_val_index();
462 };
463 let index3_block_idx: u32 =
464 w!((index1_val as u32) + (code_point >> SHIFT_2) & INDEX_2_MASK);
465 let mut index3_block: u32 =
466 if let Some(index3_block) = self.index.get(index3_block_idx as usize) {
467 index3_block as u32
468 } else {
469 return self.trie_error_val_index();
470 };
471 let mut index3_pos: u32 = (code_point >> SHIFT_3) & INDEX_3_MASK;
472 let mut data_block: u32;
473 if index3_block & 0x8000 == 0 {
474 // 16-bit indexes
475 data_block =
476 if let Some(data_block) = self.index.get(w!(index3_block + index3_pos) as usize) {
477 data_block as u32
478 } else {
479 return self.trie_error_val_index();
480 };
481 } else {
482 // 18-bit indexes stored in groups of 9 entries per 8 indexes.
483 index3_block = w!((index3_block & 0x7fff) + w!((index3_pos & !7) + index3_pos >> 3));
484 index3_pos &= 7;
485 data_block = if let Some(data_block) = self.index.get(index3_block as usize) {
486 data_block as u32
487 } else {
488 return self.trie_error_val_index();
489 };
490 data_block = (data_block << w!(2u32 + w!(2u32 * index3_pos))) & 0x30000;
491 index3_block += 1;
492 data_block =
493 if let Some(index3_val) = self.index.get(w!(index3_block + index3_pos) as usize) {
494 data_block | (index3_val as u32)
495 } else {
496 return self.trie_error_val_index();
497 };
498 }
499 // Returns data_pos == data_block (offset) +
500 // portion of code_point bit field for last (4th) lookup
501 w!(data_block + code_point & SMALL_DATA_MASK)
502 }
503
504 /// Returns the position in the `data` array for the given code point,
505 /// where this code point is at or above the fast limit associated for the
506 /// `trie_type`. We will refer to that limit as "`fastMax`" here.
507 ///
508 /// A lookup of the value in the code point trie for a code point in the
509 /// code point space range [`fastMax`, `high_start`) will be a 4-step
510 /// lookup: 3 lookups in the `index` array and one lookup in the `data`
511 /// array. Lookups for code points in the range [`high_start`,
512 /// `CODE_POINT_MAX`] are short-circuited to be a single lookup, see
513 /// [`CodePointTrieHeader::high_start`].
514 fn small_index(&self, code_point: u32) -> u32 {
515 if code_point >= self.header.high_start {
516 w!((self.data.len() as u32) - HIGH_VALUE_NEG_DATA_OFFSET)
517 } else {
518 self.internal_small_index(code_point) // helper fn
519 }
520 }
521
522 /// Returns the position in the `data` array for the given code point,
523 /// where this code point is below the fast limit associated for the
524 /// `trie type`. We will refer to that limit as "`fastMax`" here.
525 ///
526 /// A lookup of the value in the code point trie for a code point in the
527 /// code point space range [0, `fastMax`) will be a 2-step lookup: 1
528 /// lookup in the `index` array and one lookup in the `data` array. By
529 /// design, for trie type `T`, there is an element allocated in the `index`
530 /// array for each block of code points in [0, `fastMax`), which in
531 /// turn guarantees that those code points are represented and only need 1
532 /// lookup.
533 fn fast_index(&self, code_point: u32) -> u32 {
534 let index_array_pos: u32 = code_point >> FAST_TYPE_SHIFT;
535 let index_array_val: u16 =
536 if let Some(index_array_val) = self.index.get(index_array_pos as usize) {
537 index_array_val
538 } else {
539 return self.trie_error_val_index();
540 };
541 let masked_cp = code_point & FAST_TYPE_DATA_MASK;
542 let index_array_val = index_array_val as u32;
543 let fast_index_val: u32 = w!(index_array_val + masked_cp);
544 fast_index_val
545 }
546
547 /// Returns the value that is associated with `code_point` in this [`CodePointTrie`]
548 /// if `code_point` uses fast-path lookup or `None` if `code_point`
549 /// should use small-path lookup or is above the supported range.
550 #[inline(always)] // "always" to make the `Option` collapse away
551 fn get32_by_fast_index(&self, code_point: u32) -> Option<T> {
552 let fast_max = match self.header.trie_type {
553 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
554 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
555 };
556 if code_point <= fast_max {
557 // SAFETY: We just checked the invariant of
558 // `get32_assuming_fast_index`,
559 // which is
560 // "If `self.header.trie_type == TrieType::Small`, `code_point` must be at most
561 // `SMALL_TYPE_FAST_INDEXING_MAX`. If `self.header.trie_type ==
562 // TrieType::Fast`, `code_point` must be at most `FAST_TYPE_FAST_INDEXING_MAX`."
563 Some(unsafe { self.get32_assuming_fast_index(code_point) })
564 } else {
565 // The caller needs to call `get32_by_small_index` or determine
566 // that the argument is above the permitted range.
567 None
568 }
569 }
570
571 /// Performs the actual fast-mode lookup
572 ///
573 /// # Safety
574 ///
575 /// If `self.header.trie_type == TrieType::Small`, `code_point` must be at most
576 /// `SMALL_TYPE_FAST_INDEXING_MAX`. If `self.header.trie_type ==
577 /// TrieType::Fast`, `code_point` must be at most `FAST_TYPE_FAST_INDEXING_MAX`.
578 #[inline(always)]
579 unsafe fn get32_assuming_fast_index(&self, code_point: u32) -> T {
580 // Check the safety invariant of this method.
581 debug_assert!(
582 code_point
583 <= match self.header.trie_type {
584 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
585 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
586 }
587 );
588
589 let bit_prefix = (code_point as usize) >> FAST_TYPE_SHIFT;
590 let bit_suffix = (code_point & FAST_TYPE_DATA_MASK) as usize;
591 self.get_bit_prefix_suffix_assuming_fast_index(bit_prefix, bit_suffix)
592 }
593
594 #[inline(always)]
595 unsafe fn get_bit_prefix_suffix_assuming_fast_index(
596 &self,
597 bit_prefix: usize,
598 bit_suffix: usize,
599 ) -> T {
600 debug_assert!(bit_prefix < self.index.len());
601 // SAFETY: Relying on the length invariant of `self.index` having
602 // been checked and on the unchangedness invariant of `self.index`
603 // and `self.header.trie_type` after construction.
604 let base_offset_to_data: usize = usize::from(u16::from_unaligned(*unsafe {
605 self.index.as_ule_slice().get_unchecked(bit_prefix)
606 }));
607 // SAFETY: Cannot overflow with supported (32-bit and 64-bit) `usize`
608 // sizes, since `base_offset_to_data` was extended from `u16` and
609 // `bit_suffix` is at most `FAST_TYPE_DATA_MASK`, which is well
610 // under what it takes to reach the 32-bit (or 64-bit) max with
611 // additon from the max of `u16`.
612 let offset_to_data = w!(base_offset_to_data + bit_suffix);
613 debug_assert!(offset_to_data < self.data.len());
614 // SAFETY: Relying on the length invariant of `self.data` having
615 // been checked and on the unchangedness invariant of `self.data`,
616 // `self.index`, and `self.header.trie_type` after construction.
617 T::from_unaligned(*unsafe { self.data.as_ule_slice().get_unchecked(offset_to_data) })
618 }
619
620 /// Coldness wrapper for `get32_by_small_index` to also allow
621 /// calls without the effects of `#[cold]`.
622 #[cold]
623 #[inline(always)]
624 fn get32_by_small_index_cold(&self, code_point: u32) -> T {
625 self.get32_by_small_index(code_point)
626 }
627
628 /// Returns the value that is associated with `code_point` in this [`CodePointTrie`]
629 /// assuming that the small index path should be used.
630 ///
631 /// # Intended Precondition
632 ///
633 /// `code_point` must be at most `CODE_POINT_MAX` AND greter than
634 /// `FAST_TYPE_FAST_INDEXING_MAX` if the trie type is fast or greater
635 /// than `SMALL_TYPE_FAST_INDEXING_MAX` if the trie type is small.
636 /// This is checked when debug assertions are enabled. If this
637 /// precondition is violated, the behavior of this method is
638 /// memory-safe, but the returned value may be bogus (not
639 /// necessarily the designated error value).
640 #[inline(never)]
641 fn get32_by_small_index(&self, code_point: u32) -> T {
642 debug_assert!(code_point <= CODE_POINT_MAX);
643 debug_assert!(
644 code_point
645 > match self.header.trie_type {
646 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
647 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
648 }
649 );
650 self.data
651 .get(self.small_index(code_point) as usize)
652 .unwrap_or(self.error_value)
653 }
654
655 /// Returns the value that is associated with `code_point` in this [`CodePointTrie`].
656 ///
657 /// # Examples
658 ///
659 /// ```
660 /// use icu::collections::codepointtrie::planes;
661 /// let trie = planes::get_planes_trie();
662 ///
663 /// assert_eq!(0, trie.get32(0x41)); // 'A' as u32
664 /// assert_eq!(0, trie.get32(0x13E0)); // 'Ꮰ' as u32
665 /// assert_eq!(1, trie.get32(0x10044)); // '𐁄' as u32
666 /// ```
667 #[inline(always)] // `always` based on normalizer benchmarking
668 pub fn get32(&self, code_point: u32) -> T {
669 if let Some(v) = self.get32_by_fast_index(code_point) {
670 v
671 } else if code_point <= CODE_POINT_MAX {
672 self.get32_by_small_index_cold(code_point)
673 } else {
674 self.error_value
675 }
676 }
677
678 /// Returns the value that is associated with `char` in this [`CodePointTrie`].
679 ///
680 /// # Examples
681 ///
682 /// ```
683 /// use icu::collections::codepointtrie::planes;
684 /// let trie = planes::get_planes_trie();
685 ///
686 /// assert_eq!(0, trie.get('A')); // 'A' as u32
687 /// assert_eq!(0, trie.get('Ꮰ')); // 'Ꮰ' as u32
688 /// assert_eq!(1, trie.get('𐁄')); // '𐁄' as u32
689 /// ```
690 #[inline(always)]
691 pub fn get(&self, c: char) -> T {
692 // LLVM's optimizations have been observed not to be 100%
693 // reliable around collapsing away unnecessary parts of
694 // `get32`, so not just calling `get32` here.
695 let code_point = u32::from(c);
696 if let Some(v) = self.get32_by_fast_index(code_point) {
697 v
698 } else {
699 self.get32_by_small_index_cold(code_point)
700 }
701 }
702
703 /// Returns the value that is associated with `bmp` in this [`CodePointTrie`].
704 #[inline(always)]
705 pub fn get16(&self, bmp: u16) -> T {
706 // LLVM's optimizations have been observed not to be 100%
707 // reliable around collapsing away unnecessary parts of
708 // `get32`, so not just calling `get32` here.
709 let code_point = u32::from(bmp);
710 if let Some(v) = self.get32_by_fast_index(code_point) {
711 v
712 } else {
713 self.get32_by_small_index_cold(code_point)
714 }
715 }
716
717 /// Returns the value that is associated with `latin1` in this [`CodePointTrie`].
718 #[inline(always)]
719 pub fn get8(&self, latin1: u8) -> T {
720 let code_point = u32::from(latin1);
721 debug_assert!(code_point <= SMALL_TYPE_FAST_INDEXING_MAX);
722 // SAFETY: `u8` is always below `SMALL_TYPE_FAST_INDEXING_MAX` and,
723 // therefore, belowe `FAST_TYPE_FAST_INDEXING_MAX`.
724 unsafe { self.get32_assuming_fast_index(code_point) }
725 }
726
727 /// Returns the value that is associated with `ascii` in this [`CodePointTrie`].
728 ///
729 /// # Safety
730 ///
731 /// `ascii` must be less than 128.
732 #[inline(always)]
733 pub unsafe fn get7(&self, ascii: u8) -> T {
734 debug_assert!(ascii < 128);
735 debug_assert!((ascii as usize) < self.data.len());
736 // SAFETY: Length of `self.data` checked in the constructor.
737 T::from_unaligned(*unsafe { self.data.as_ule_slice().get_unchecked(ascii as usize) })
738 }
739
740 /// Returns the value that is associated with a two-byte UTF-8 sequence in this [`CodePointTrie`].
741 ///
742 /// `high_five` is the low five bits of the lead byte of a two-byte UTF-8 sequence.
743 /// `low_six` is the low six bits of the trail byte of a two-byte UTF-8 sequence.
744 ///
745 /// # Safety
746 ///
747 /// `high_five` must not have bit positions other than the lowest 5 set to 1.
748 /// `low_six` must not have bit positions other than the lowest 6 set to 1.
749 ///
750 /// # Panics
751 ///
752 /// With debug assertions enabled, panics if the above safety invariants are
753 /// violated or `high_five` represents non-shortest form.
754 #[inline(always)]
755 pub unsafe fn get_utf8_two_byte(&self, high_five: u32, low_six: u32) -> T {
756 debug_assert!(low_six <= 0b111_111); // Safety invariant.
757 debug_assert!(high_five <= 0b11_111); // Safety invariant.
758 debug_assert!(high_five > 0b1); // Non-shortest form; not safety invariant.
759 // SAFETY: The highest character representable as a two-byte
760 // UTF-8 sequence is U+07FF, eleven binary ones, which is below
761 // both `SMALL_TYPE_FAST_INDEXING_MAX` and `FAST_TYPE_FAST_INDEXING_MAX`.
762 self.get_bit_prefix_suffix_assuming_fast_index(high_five as usize, low_six as usize)
763 }
764
765 /// Returns the value that is associated with a three-byte UTF-8 or WTF-8 sequence in this [`CodePointTrie`].
766 ///
767 /// `high_ten` is the low four bits of the lead byte of three-byte UTF-8 or WTF-8 sequence shifted left by 6 followed by the low six bits of the first trail byte.
768 /// `low_six` is the low six bits of the last trail byte of a three-byte UTF-8 or WTF-8 sequence.
769 ///
770 /// Sequences representing surrogates (WTF-8) are allowed.
771 ///
772 /// # Safety
773 ///
774 /// `low_six` must not have bit positions other than the lowest 6 set to 1.
775 ///
776 /// # Intended Invariant
777 ///
778 /// `high_ten` must not have bit positions other than the lowest 10 set to 1.
779 ///
780 /// # Panics
781 ///
782 /// With debug assertions enabled, panics if the above safety invariant is
783 /// violated or `high_ten` is out of range for three-byte WTF-8 (or UTF-8)
784 /// sequence.
785 #[inline(always)]
786 #[allow(clippy::unusual_byte_groupings)]
787 pub unsafe fn get_utf8_three_byte(&self, high_ten: u32, low_six: u32) -> T {
788 debug_assert!(low_six <= 0b111_111); // Safety invariant.
789 debug_assert!(high_ten <= 0b1111_111_111); // Not actually a _safety_ invariant for this impl.
790 debug_assert!(high_ten > 0b11_111); // Non-shortest form; not safety invariant.
791
792 let fast_max = match self.header.trie_type {
793 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
794 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
795 };
796 // Keep only the prefix bits:
797 let max_bit_prefix = fast_max >> FAST_TYPE_SHIFT;
798 if high_ten <= max_bit_prefix {
799 // SAFETY: The caller is responsible for upholding the safety
800 // invariant for `low_six` and we just checked the safety
801 // invariant of `high_ten`.
802 self.get_bit_prefix_suffix_assuming_fast_index(high_ten as usize, low_six as usize)
803 } else {
804 self.get32_by_small_index_cold((high_ten << 6) | low_six)
805 }
806 }
807
808 /// Lookup trie value by non-Basic Multilingual Plane Scalar Value.
809 ///
810 /// The return value may be bogus (not necessarily `error_value`) is the argument is actually in
811 /// the Basic Multilingual Plane or above the Unicode Scalar Value
812 /// range (panics instead with debug assertions enabled).
813 #[inline(always)]
814 pub fn get32_supplementary(&self, supplementary: u32) -> T {
815 debug_assert!(supplementary > 0xFFFF);
816 debug_assert!(supplementary <= CODE_POINT_MAX);
817 self.get32_by_small_index(supplementary)
818 }
819
820 /// Returns a reference to the ULE of the value that is associated with `code_point` in this [`CodePointTrie`].
821 ///
822 /// # Examples
823 ///
824 /// ```
825 /// use icu::collections::codepointtrie::planes;
826 /// let trie = planes::get_planes_trie();
827 ///
828 /// assert_eq!(Some(&0), trie.get32_ule(0x41)); // 'A' as u32
829 /// assert_eq!(Some(&0), trie.get32_ule(0x13E0)); // 'Ꮰ' as u32
830 /// assert_eq!(Some(&1), trie.get32_ule(0x10044)); // '𐁄' as u32
831 /// ```
832 #[inline(always)] // `always` was based on previous normalizer benchmarking
833 pub fn get32_ule(&self, code_point: u32) -> Option<&T::ULE> {
834 // All code points up to the fast max limit are represented
835 // individually in the `index` array to hold their `data` array position, and
836 // thus only need 2 lookups for a [CodePointTrie::get()](`crate::codepointtrie::CodePointTrie::get`).
837 // Code points above the "fast max" limit require 4 lookups.
838 let fast_max = match self.header.trie_type {
839 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
840 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
841 };
842 let data_pos: u32 = if code_point <= fast_max {
843 Self::fast_index(self, code_point)
844 } else if code_point <= CODE_POINT_MAX {
845 Self::small_index(self, code_point)
846 } else {
847 self.trie_error_val_index()
848 };
849 // Returns the trie value (or trie's error value).
850 self.data.as_ule_slice().get(data_pos as usize)
851 }
852
853 /// Converts the [`CodePointTrie`] into one that returns another type of the same size.
854 ///
855 /// Borrowed data remains borrowed, and owned data remains owned.
856 ///
857 /// If the old and new types are not the same size, use
858 /// [`CodePointTrie::try_alloc_map_value`].
859 ///
860 /// # Panics
861 ///
862 /// Panics if `T` and `P` are different sizes.
863 ///
864 /// More specifically, panics if [`ZeroVec::try_into_converted()`] panics when converting
865 /// `ZeroVec<T>` into `ZeroVec<P>`, which happens if `T::ULE` and `P::ULE` differ in size.
866 ///
867 /// ✨ *Enabled with the `alloc` Cargo feature.*
868 ///
869 /// # Examples
870 ///
871 /// ```no_run
872 /// use icu::collections::codepointtrie::planes;
873 /// use icu::collections::codepointtrie::CodePointTrie;
874 ///
875 /// let planes_trie_u8: CodePointTrie<u8> = planes::get_planes_trie();
876 /// let planes_trie_i8: CodePointTrie<i8> =
877 /// planes_trie_u8.try_into_converted().expect("infallible");
878 ///
879 /// assert_eq!(planes_trie_i8.get32(0x30000), 3);
880 /// ```
881 #[cfg(feature = "alloc")]
882 pub fn try_into_converted<P>(self) -> Result<CodePointTrie<'trie, P>, UleError>
883 where
884 P: TrieValue,
885 {
886 let converted_data = self.data.try_into_converted()?;
887 let error_ule = self.error_value.to_unaligned();
888 let slice = &[error_ule];
889 let error_vec = ZeroVec::<T>::new_borrowed(slice);
890 let error_converted = error_vec.try_into_converted::<P>()?;
891 #[expect(clippy::expect_used)] // we know this cannot fail
892 Ok(CodePointTrie {
893 header: self.header,
894 index: self.index,
895 data: converted_data,
896 error_value: error_converted
897 .get(0)
898 .expect("vector known to have one element"),
899 })
900 }
901
902 /// Maps the [`CodePointTrie`] into one that returns a different type.
903 ///
904 /// This function returns owned data.
905 ///
906 /// If the old and new types are the same size, use the more efficient
907 /// [`CodePointTrie::try_into_converted`].
908 ///
909 /// ✨ *Enabled with the `alloc` Cargo feature.*
910 ///
911 /// # Examples
912 ///
913 /// ```
914 /// use icu::collections::codepointtrie::planes;
915 /// use icu::collections::codepointtrie::CodePointTrie;
916 ///
917 /// let planes_trie_u8: CodePointTrie<u8> = planes::get_planes_trie();
918 /// let planes_trie_u16: CodePointTrie<u16> = planes_trie_u8
919 /// .try_alloc_map_value(TryFrom::try_from)
920 /// .expect("infallible");
921 ///
922 /// assert_eq!(planes_trie_u16.get32(0x30000), 3);
923 /// ```
924 #[cfg(feature = "alloc")]
925 pub fn try_alloc_map_value<P, E>(
926 &self,
927 mut f: impl FnMut(T) -> Result<P, E>,
928 ) -> Result<CodePointTrie<'trie, P>, E>
929 where
930 P: TrieValue,
931 {
932 let error_converted = f(self.error_value)?;
933 let converted_data = self.data.iter().map(f).collect::<Result<ZeroVec<P>, E>>()?;
934 Ok(CodePointTrie {
935 header: self.header,
936 index: self.index.clone(),
937 data: converted_data,
938 error_value: error_converted,
939 })
940 }
941
942 /// Returns a [`CodePointMapRange`] struct which represents a range of code
943 /// points associated with the same trie value. The returned range will be
944 /// the longest stretch of consecutive code points starting at `start` that
945 /// share this value.
946 ///
947 /// This method is designed to use the internal details of
948 /// the structure of [`CodePointTrie`] to be optimally efficient. This will
949 /// outperform a naive approach that just uses [`CodePointTrie::get()`].
950 ///
951 /// This method provides lower-level functionality that can be used in the
952 /// implementation of other methods that are more convenient to the user.
953 /// To obtain an optimal partition of the code point space for
954 /// this trie resulting in the fewest number of ranges, see
955 /// [`CodePointTrie::iter_ranges()`].
956 ///
957 /// # Examples
958 ///
959 /// ```
960 /// use icu::collections::codepointtrie::planes;
961 ///
962 /// let trie = planes::get_planes_trie();
963 ///
964 /// const CODE_POINT_MAX: u32 = 0x10ffff;
965 /// let start = 0x1_0000;
966 /// let exp_end = 0x1_ffff;
967 ///
968 /// let start_val = trie.get32(start);
969 /// assert_eq!(trie.get32(exp_end), start_val);
970 /// assert_ne!(trie.get32(exp_end + 1), start_val);
971 ///
972 /// use icu::collections::codepointtrie::CodePointMapRange;
973 ///
974 /// let cpm_range: CodePointMapRange<u8> = trie.get_range(start).unwrap();
975 ///
976 /// assert_eq!(cpm_range.range.start(), &start);
977 /// assert_eq!(cpm_range.range.end(), &exp_end);
978 /// assert_eq!(cpm_range.value, start_val);
979 ///
980 /// // `start` can be any code point, whether or not it lies on the boundary
981 /// // of a maximally large range that still contains `start`
982 ///
983 /// let submaximal_1_start = start + 0x1234;
984 /// let submaximal_1 = trie.get_range(submaximal_1_start).unwrap();
985 /// assert_eq!(submaximal_1.range.start(), &0x1_1234);
986 /// assert_eq!(submaximal_1.range.end(), &0x1_ffff);
987 /// assert_eq!(submaximal_1.value, start_val);
988 ///
989 /// let submaximal_2_start = start + 0xffff;
990 /// let submaximal_2 = trie.get_range(submaximal_2_start).unwrap();
991 /// assert_eq!(submaximal_2.range.start(), &0x1_ffff);
992 /// assert_eq!(submaximal_2.range.end(), &0x1_ffff);
993 /// assert_eq!(submaximal_2.value, start_val);
994 /// ```
995 pub fn get_range(&self, start: u32) -> Option<CodePointMapRange<T>> {
996 // Exit early if the start code point is out of range, or if it is
997 // in the last range of code points in high_start..=CODE_POINT_MAX
998 // (start- and end-inclusive) that all share the same trie value.
999 if CODE_POINT_MAX < start {
1000 return None;
1001 }
1002 if start >= self.header.high_start {
1003 let di: usize = self.data.len() - (HIGH_VALUE_NEG_DATA_OFFSET as usize);
1004 let value: T = self.data.get(di)?;
1005 return Some(CodePointMapRange {
1006 range: start..=CODE_POINT_MAX,
1007 value,
1008 });
1009 }
1010
1011 let null_value: T = T::try_from_u32(self.header.null_value).ok()?;
1012
1013 let mut prev_i3_block: u32 = u32::MAX; // using u32::MAX (instead of -1 as an i32 in ICU)
1014 let mut prev_block: u32 = u32::MAX; // using u32::MAX (instead of -1 as an i32 in ICU)
1015 let mut c: u32 = start;
1016 let mut trie_value: T = self.error_value();
1017 let mut value: T = self.error_value();
1018 let mut have_value: bool = false;
1019
1020 loop {
1021 let i3_block: u32;
1022 let mut i3: u32;
1023 let i3_block_length: u32;
1024 let data_block_length: u32;
1025
1026 // Initialize values before beginning the iteration in the subsequent
1027 // `loop` block. In particular, use the "i3*" local variables
1028 // (representing the `index` array position's offset + increment
1029 // for a 3rd-level trie lookup) to help initialize the data block
1030 // variable `block` in the loop for the `data` array.
1031 //
1032 // When a lookup code point is <= the trie's *_FAST_INDEXING_MAX that
1033 // corresponds to its `trie_type`, the lookup only takes 2 steps
1034 // (once into the `index`, once into the `data` array); otherwise,
1035 // takes 4 steps (3 iterative lookups into the `index`, once more
1036 // into the `data` array). So for convenience's sake, when we have the
1037 // 2-stage lookup, reuse the "i3*" variable names for the first lookup.
1038 if c <= 0xffff
1039 && (self.header.trie_type == TrieType::Fast || c <= SMALL_TYPE_FAST_INDEXING_MAX)
1040 {
1041 i3_block = 0;
1042 i3 = c >> FAST_TYPE_SHIFT;
1043 i3_block_length = if self.header.trie_type == TrieType::Fast {
1044 BMP_INDEX_LENGTH
1045 } else {
1046 SMALL_INDEX_LENGTH
1047 };
1048 data_block_length = FAST_TYPE_DATA_BLOCK_LENGTH;
1049 } else {
1050 // Use the multi-stage index.
1051 let mut i1: u32 = c >> SHIFT_1;
1052 if self.header.trie_type == TrieType::Fast {
1053 debug_assert!(0xffff < c && c < self.header.high_start);
1054 i1 = i1 + BMP_INDEX_LENGTH - OMITTED_BMP_INDEX_1_LENGTH;
1055 } else {
1056 debug_assert!(
1057 c < self.header.high_start && self.header.high_start > SMALL_LIMIT
1058 );
1059 i1 += SMALL_INDEX_LENGTH;
1060 }
1061 let i2: u16 = self.index.get(i1 as usize)?;
1062 let i3_block_idx: u32 = (i2 as u32) + ((c >> SHIFT_2) & INDEX_2_MASK);
1063 i3_block = if let Some(i3b) = self.index.get(i3_block_idx as usize) {
1064 i3b as u32
1065 } else {
1066 return None;
1067 };
1068 if i3_block == prev_i3_block && (c - start) >= CP_PER_INDEX_2_ENTRY {
1069 // The index-3 block is the same as the previous one, and filled with value.
1070 debug_assert!((c & (CP_PER_INDEX_2_ENTRY - 1)) == 0);
1071 c += CP_PER_INDEX_2_ENTRY;
1072
1073 if c >= self.header.high_start {
1074 break;
1075 } else {
1076 continue;
1077 }
1078 }
1079 prev_i3_block = i3_block;
1080 if i3_block == self.header.index3_null_offset as u32 {
1081 // This is the index-3 null block.
1082 // All of the `data` array blocks pointed to by the values
1083 // in this block of the `index` 3rd-stage subarray will
1084 // contain this trie's null_value. So if we are in the middle
1085 // of a range, end it and return early, otherwise start a new
1086 // range of null values.
1087 if have_value {
1088 if null_value != value {
1089 return Some(CodePointMapRange {
1090 range: start..=(c - 1),
1091 value,
1092 });
1093 }
1094 } else {
1095 trie_value = T::try_from_u32(self.header.null_value).ok()?;
1096 value = null_value;
1097 have_value = true;
1098 }
1099 prev_block = self.header.data_null_offset;
1100 c = (c + CP_PER_INDEX_2_ENTRY) & !(CP_PER_INDEX_2_ENTRY - 1);
1101
1102 if c >= self.header.high_start {
1103 break;
1104 } else {
1105 continue;
1106 }
1107 }
1108 i3 = (c >> SHIFT_3) & INDEX_3_MASK;
1109 i3_block_length = INDEX_3_BLOCK_LENGTH;
1110 data_block_length = SMALL_DATA_BLOCK_LENGTH;
1111 }
1112
1113 // Enumerate data blocks for one index-3 block.
1114 loop {
1115 let mut block: u32;
1116 if (i3_block & 0x8000) == 0 {
1117 block = if let Some(b) = self.index.get((i3_block + i3) as usize) {
1118 b as u32
1119 } else {
1120 return None;
1121 };
1122 } else {
1123 // 18-bit indexes stored in groups of 9 entries per 8 indexes.
1124 let mut group: u32 = (i3_block & 0x7fff) + (i3 & !7) + (i3 >> 3);
1125 let gi: u32 = i3 & 7;
1126 let gi_val: u32 = if let Some(giv) = self.index.get(group as usize) {
1127 giv.into()
1128 } else {
1129 return None;
1130 };
1131 block = (gi_val << (2 + (2 * gi))) & 0x30000;
1132 group += 1;
1133 let ggi_val: u32 = if let Some(ggiv) = self.index.get((group + gi) as usize) {
1134 ggiv as u32
1135 } else {
1136 return None;
1137 };
1138 block |= ggi_val;
1139 }
1140
1141 // If our previous and current return values of the 3rd-stage `index`
1142 // lookup yield the same `data` block offset, and if we already know that
1143 // the entire `data` block / subarray starting at that offset stores
1144 // `value` and nothing else, then we can extend our range by the length
1145 // of a data block and continue.
1146 // Otherwise, we have to iterate over the values stored in the
1147 // new data block to see if they differ from `value`.
1148 if block == prev_block && (c - start) >= data_block_length {
1149 // The block is the same as the previous one, and filled with value.
1150 debug_assert!((c & (data_block_length - 1)) == 0);
1151 c += data_block_length;
1152 } else {
1153 let data_mask: u32 = data_block_length - 1;
1154 prev_block = block;
1155 if block == self.header.data_null_offset {
1156 // This is the data null block.
1157 // If we are in the middle of a range, end it and
1158 // return early, otherwise start a new range of null
1159 // values.
1160 if have_value {
1161 if null_value != value {
1162 return Some(CodePointMapRange {
1163 range: start..=(c - 1),
1164 value,
1165 });
1166 }
1167 } else {
1168 trie_value = T::try_from_u32(self.header.null_value).ok()?;
1169 value = null_value;
1170 have_value = true;
1171 }
1172 c = (c + data_block_length) & !data_mask;
1173 } else {
1174 let mut di: u32 = block + (c & data_mask);
1175 let mut trie_value_2: T = self.data.get(di as usize)?;
1176 if have_value {
1177 if trie_value_2 != trie_value {
1178 if maybe_filter_value(
1179 trie_value_2,
1180 T::try_from_u32(self.header.null_value).ok()?,
1181 null_value,
1182 ) != value
1183 {
1184 return Some(CodePointMapRange {
1185 range: start..=(c - 1),
1186 value,
1187 });
1188 }
1189 // `trie_value` stores the previous value that was retrieved
1190 // from the trie.
1191 // `value` stores the value associated for the range (return
1192 // value) that we are currently building, which is computed
1193 // as a transformation by applying maybe_filter_value()
1194 // to the trie value.
1195 // The current trie value `trie_value_2` within this data block
1196 // differs here from the previous value in `trie_value`.
1197 // But both map to `value` after applying `maybe_filter_value`.
1198 // It is not clear whether the previous or the current trie value
1199 // (or neither) is more likely to match potential subsequent trie
1200 // values that would extend the range by mapping to `value`.
1201 // On the assumption of locality -- often times consecutive
1202 // characters map to the same trie values -- remembering the new
1203 // one might make it faster to extend this range further
1204 // (by increasing the chance that the next `trie_value_2 !=
1205 // trie_value` test will be false).
1206 trie_value = trie_value_2; // may or may not help
1207 }
1208 } else {
1209 trie_value = trie_value_2;
1210 value = maybe_filter_value(
1211 trie_value_2,
1212 T::try_from_u32(self.header.null_value).ok()?,
1213 null_value,
1214 );
1215 have_value = true;
1216 }
1217
1218 c += 1;
1219 while (c & data_mask) != 0 {
1220 di += 1;
1221 trie_value_2 = self.data.get(di as usize)?;
1222 if trie_value_2 != trie_value {
1223 if maybe_filter_value(
1224 trie_value_2,
1225 T::try_from_u32(self.header.null_value).ok()?,
1226 null_value,
1227 ) != value
1228 {
1229 return Some(CodePointMapRange {
1230 range: start..=(c - 1),
1231 value,
1232 });
1233 }
1234 // `trie_value` stores the previous value that was retrieved
1235 // from the trie.
1236 // `value` stores the value associated for the range (return
1237 // value) that we are currently building, which is computed
1238 // as a transformation by applying maybe_filter_value()
1239 // to the trie value.
1240 // The current trie value `trie_value_2` within this data block
1241 // differs here from the previous value in `trie_value`.
1242 // But both map to `value` after applying `maybe_filter_value`.
1243 // It is not clear whether the previous or the current trie value
1244 // (or neither) is more likely to match potential subsequent trie
1245 // values that would extend the range by mapping to `value`.
1246 // On the assumption of locality -- often times consecutive
1247 // characters map to the same trie values -- remembering the new
1248 // one might make it faster to extend this range further
1249 // (by increasing the chance that the next `trie_value_2 !=
1250 // trie_value` test will be false).
1251 trie_value = trie_value_2; // may or may not help
1252 }
1253
1254 c += 1;
1255 }
1256 }
1257 }
1258
1259 i3 += 1;
1260 if i3 >= i3_block_length {
1261 break;
1262 }
1263 }
1264
1265 if c >= self.header.high_start {
1266 break;
1267 }
1268 }
1269
1270 debug_assert!(have_value);
1271
1272 // Now that c >= high_start, compare `value` to `high_value` to see
1273 // if we can merge our current range with the high_value range
1274 // high_start..=CODE_POINT_MAX (start- and end-inclusive), otherwise
1275 // stop at high_start - 1.
1276 let di: u32 = self.data.len() as u32 - HIGH_VALUE_NEG_DATA_OFFSET;
1277 let high_value: T = self.data.get(di as usize)?;
1278 if maybe_filter_value(
1279 high_value,
1280 T::try_from_u32(self.header.null_value).ok()?,
1281 null_value,
1282 ) != value
1283 {
1284 c -= 1;
1285 } else {
1286 c = CODE_POINT_MAX;
1287 }
1288 Some(CodePointMapRange {
1289 range: start..=c,
1290 value,
1291 })
1292 }
1293
1294 /// Yields an [`Iterator`] returning ranges of consecutive code points that
1295 /// share the same value in the [`CodePointTrie`], as given by
1296 /// [`CodePointTrie::get_range()`].
1297 ///
1298 /// # Examples
1299 ///
1300 /// ```
1301 /// use core::ops::RangeInclusive;
1302 /// use icu::collections::codepointtrie::planes;
1303 /// use icu::collections::codepointtrie::CodePointMapRange;
1304 ///
1305 /// let planes_trie = planes::get_planes_trie();
1306 ///
1307 /// let mut ranges = planes_trie.iter_ranges();
1308 ///
1309 /// for plane in 0..=16 {
1310 /// let exp_start = plane * 0x1_0000;
1311 /// let exp_end = exp_start + 0xffff;
1312 /// assert_eq!(
1313 /// ranges.next(),
1314 /// Some(CodePointMapRange {
1315 /// range: exp_start..=exp_end,
1316 /// value: plane as u8
1317 /// })
1318 /// );
1319 /// }
1320 ///
1321 /// // Hitting the end of the iterator returns `None`, as will subsequent
1322 /// // calls to .next().
1323 /// assert_eq!(ranges.next(), None);
1324 /// assert_eq!(ranges.next(), None);
1325 /// ```
1326 pub fn iter_ranges(&self) -> CodePointMapRangeIterator<'_, T> {
1327 let init_range = Some(CodePointMapRange {
1328 range: u32::MAX..=u32::MAX,
1329 value: self.error_value(),
1330 });
1331 CodePointMapRangeIterator::<T> {
1332 cpt: self,
1333 cpm_range: init_range,
1334 }
1335 }
1336
1337 /// Yields an [`Iterator`] returning the ranges of the code points whose values
1338 /// match `value` in the [`CodePointTrie`].
1339 ///
1340 /// # Examples
1341 ///
1342 /// ```
1343 /// use icu::collections::codepointtrie::planes;
1344 ///
1345 /// let trie = planes::get_planes_trie();
1346 ///
1347 /// let plane_val = 2;
1348 /// let mut sip_range_iter = trie.iter_ranges_for_value(plane_val as u8);
1349 ///
1350 /// let start = plane_val * 0x1_0000;
1351 /// let end = start + 0xffff;
1352 ///
1353 /// let sip_range = sip_range_iter.next()
1354 /// .expect("Plane 2 (SIP) should exist in planes data");
1355 /// assert_eq!(start..=end, sip_range);
1356 ///
1357 /// assert!(sip_range_iter.next().is_none());
1358 pub fn iter_ranges_for_value(
1359 &self,
1360 value: T,
1361 ) -> impl Iterator<Item = RangeInclusive<u32>> + '_ {
1362 self.iter_ranges()
1363 .filter(move |cpm_range| cpm_range.value == value)
1364 .map(|cpm_range| cpm_range.range)
1365 }
1366
1367 /// Yields an [`Iterator`] returning the ranges of the code points after passing
1368 /// the value through a mapping function.
1369 ///
1370 /// This is preferable to calling `.get_ranges().map()` since it will coalesce
1371 /// adjacent ranges into one.
1372 ///
1373 /// # Examples
1374 ///
1375 /// ```
1376 /// use icu::collections::codepointtrie::planes;
1377 ///
1378 /// let trie = planes::get_planes_trie();
1379 ///
1380 /// let plane_val = 2;
1381 /// let mut sip_range_iter = trie.iter_ranges_mapped(|value| value != plane_val as u8).filter(|range| range.value);
1382 ///
1383 /// let end = plane_val * 0x1_0000 - 1;
1384 ///
1385 /// let sip_range = sip_range_iter.next()
1386 /// .expect("Complemented planes data should have at least one entry");
1387 /// assert_eq!(0..=end, sip_range.range);
1388 pub fn iter_ranges_mapped<'a, U: Eq + 'a>(
1389 &'a self,
1390 mut map: impl FnMut(T) -> U + Copy + 'a,
1391 ) -> impl Iterator<Item = CodePointMapRange<U>> + 'a {
1392 crate::iterator_utils::RangeListIteratorCoalescer::new(self.iter_ranges().map(
1393 move |range| CodePointMapRange {
1394 range: range.range,
1395 value: map(range.value),
1396 },
1397 ))
1398 }
1399
1400 /// Returns a [`CodePointInversionList`] for the code points that have the given
1401 /// [`TrieValue`] in the trie.
1402 ///
1403 /// ✨ *Enabled with the `alloc` Cargo feature.*
1404 ///
1405 /// # Examples
1406 ///
1407 /// ```
1408 /// use icu::collections::codepointtrie::planes;
1409 ///
1410 /// let trie = planes::get_planes_trie();
1411 ///
1412 /// let plane_val = 2;
1413 /// let sip = trie.get_set_for_value(plane_val as u8);
1414 ///
1415 /// let start = plane_val * 0x1_0000;
1416 /// let end = start + 0xffff;
1417 ///
1418 /// assert!(!sip.contains32(start - 1));
1419 /// assert!(sip.contains32(start));
1420 /// assert!(sip.contains32(end));
1421 /// assert!(!sip.contains32(end + 1));
1422 /// ```
1423 #[cfg(feature = "alloc")]
1424 pub fn get_set_for_value(&self, value: T) -> CodePointInversionList<'static> {
1425 let value_ranges = self.iter_ranges_for_value(value);
1426 CodePointInversionList::from_iter(value_ranges)
1427 }
1428
1429 /// Returns the value used as an error value for this trie
1430 #[inline]
1431 pub fn error_value(&self) -> T {
1432 self.error_value
1433 }
1434}
1435
1436#[cfg(feature = "databake")]
1437impl<T: TrieValue + databake::Bake> databake::Bake for CodePointTrie<'_, T> {
1438 fn bake(&self, env: &databake::CrateEnv) -> databake::TokenStream {
1439 let header = self.header.bake(env);
1440 let index = self.index.bake(env);
1441 let data = self.data.bake(env);
1442 let error_value = self.error_value.bake(env);
1443 databake::quote! { unsafe { icu_collections::codepointtrie::CodePointTrie::from_parts_unstable_unchecked_v1(#header, #index, #data, #error_value) } }
1444 }
1445}
1446
1447#[cfg(feature = "databake")]
1448impl<T: TrieValue + databake::Bake> databake::BakeSize for CodePointTrie<'_, T> {
1449 fn borrows_size(&self) -> usize {
1450 self.header.borrows_size() + self.index.borrows_size() + self.data.borrows_size()
1451 }
1452}
1453
1454impl<T: TrieValue + Into<u32>> CodePointTrie<'_, T> {
1455 /// Returns the value that is associated with `code_point` for this [`CodePointTrie`]
1456 /// as a `u32`.
1457 ///
1458 /// # Examples
1459 ///
1460 /// ```
1461 /// use icu::collections::codepointtrie::planes;
1462 /// let trie = planes::get_planes_trie();
1463 ///
1464 /// let cp = '𑖎' as u32;
1465 /// assert_eq!(cp, 0x1158E);
1466 ///
1467 /// let plane_num: u8 = trie.get32(cp);
1468 /// assert_eq!(trie.get32_u32(cp), plane_num as u32);
1469 /// ```
1470 // Note: This API method maintains consistency with the corresponding
1471 // original ICU APIs.
1472 pub fn get32_u32(&self, code_point: u32) -> u32 {
1473 self.get32(code_point).into()
1474 }
1475}
1476
1477impl<T: TrieValue> Clone for CodePointTrie<'_, T>
1478where
1479 <T as AsULE>::ULE: Clone,
1480{
1481 fn clone(&self) -> Self {
1482 CodePointTrie {
1483 header: self.header,
1484 index: self.index.clone(),
1485 data: self.data.clone(),
1486 error_value: self.error_value,
1487 }
1488 }
1489}
1490
1491/// Represents a range of consecutive code points sharing the same value in a
1492/// code point map.
1493///
1494/// The start and end of the interval is represented as a
1495/// `RangeInclusive<u32>`, and the value is represented as `T`.
1496#[derive(PartialEq, Eq, Debug, Clone)]
1497#[allow(clippy::exhaustive_structs)] // based on a stable serialized form
1498pub struct CodePointMapRange<T> {
1499 /// Range of code points from start to end (inclusive).
1500 pub range: RangeInclusive<u32>,
1501 /// Trie value associated with this range.
1502 pub value: T,
1503}
1504
1505/// A custom [`Iterator`] type specifically for a code point trie that returns
1506/// [`CodePointMapRange`]s.
1507#[derive(Debug)]
1508pub struct CodePointMapRangeIterator<'a, T: TrieValue> {
1509 cpt: &'a CodePointTrie<'a, T>,
1510 // Initialize `range` to Some(CodePointMapRange{ start: u32::MAX, end: u32::MAX, value: 0}).
1511 // When `range` is Some(...) and has a start value different from u32::MAX, then we have
1512 // returned at least one code point range due to a call to `next()`.
1513 // When `range` == `None`, it means that we have hit the end of iteration. It would occur
1514 // after a call to `next()` returns a None <=> we attempted to call `get_range()`
1515 // with a start code point that is > CODE_POINT_MAX.
1516 cpm_range: Option<CodePointMapRange<T>>,
1517}
1518
1519impl<T: TrieValue> Iterator for CodePointMapRangeIterator<'_, T> {
1520 type Item = CodePointMapRange<T>;
1521
1522 fn next(&mut self) -> Option<Self::Item> {
1523 self.cpm_range = match &self.cpm_range {
1524 Some(cpmr) => {
1525 if *cpmr.range.start() == u32::MAX {
1526 self.cpt.get_range(0)
1527 } else {
1528 self.cpt.get_range(cpmr.range.end() + 1)
1529 }
1530 }
1531 None => None,
1532 };
1533 // Note: Clone is cheap. We can't Copy because RangeInclusive does not impl Copy.
1534 self.cpm_range.clone()
1535 }
1536}
1537
1538/// For sealing `TypedCodePointTrie`
1539///
1540/// # Safety Usable Invariant
1541///
1542/// All implementations of `TypedCodePointTrie` are reviewable in this module.
1543trait Seal {}
1544
1545impl<'trie, T: TrieValue> Seal for CodePointTrie<'trie, T> {}
1546
1547/// Trait for writing trait bounds for monomorphizing over either
1548/// `FastCodePointTrie` or `SmallCodePointTrie`.
1549#[allow(private_bounds)] // Permit sealing
1550pub trait TypedCodePointTrie<'trie, T: TrieValue>: Seal {
1551 /// The `TrieType` associated with this `TypedCodePointTrie`
1552 ///
1553 /// # Safety Usable Invariant
1554 ///
1555 /// This constant matches `self.as_untyped_ref().header.trie_type`.
1556 const TRIE_TYPE: TrieType;
1557
1558 /// Lookup trie value as `u32` by Unicode Scalar Value without branching on trie type.
1559 #[inline(always)]
1560 fn get32_u32(&self, code_point: u32) -> u32 {
1561 self.get32(code_point).to_u32()
1562 }
1563
1564 /// Lookup trie value by Basic Multilingual Plane Code Point without branching on trie type.
1565 #[inline(always)]
1566 fn get16(&self, bmp: u16) -> T {
1567 // LLVM's optimizations have been observed not to be 100%
1568 // reliable around collapsing away unnecessary parts of
1569 // `get32`, so not just calling `get32` here.
1570 let code_point = u32::from(bmp);
1571 if let Some(v) = self.get32_by_fast_index(code_point) {
1572 v
1573 } else {
1574 self.as_untyped_ref().get32_by_small_index_cold(code_point)
1575 }
1576 }
1577
1578 /// Lookup trie value by Latin1 Code Point without branching on trie type.
1579 #[inline(always)]
1580 fn get8(&self, latin1: u8) -> T {
1581 self.as_untyped_ref().get8(latin1)
1582 }
1583
1584 /// Lookup trie value by ASCII Code Point without branching on trie type.
1585 ///
1586 /// # Safety
1587 ///
1588 /// `ascii` must be less than 128.
1589 #[inline(always)]
1590 unsafe fn get7(&self, ascii: u8) -> T {
1591 self.as_untyped_ref().get7(ascii)
1592 }
1593
1594 /// Lookup trie value by non-Basic Multilingual Plane Scalar Value without branching on trie type.
1595 #[inline(always)]
1596 fn get32_supplementary(&self, supplementary: u32) -> T {
1597 self.as_untyped_ref().get32_supplementary(supplementary)
1598 }
1599
1600 /// Lookup trie value by Unicode Scalar Value without branching on trie type.
1601 #[inline(always)]
1602 fn get(&self, c: char) -> T {
1603 // LLVM's optimizations have been observed not to be 100%
1604 // reliable around collapsing away unnecessary parts of
1605 // `get32`, so not just calling `get32` here.
1606 let code_point = u32::from(c);
1607 if let Some(v) = self.get32_by_fast_index(code_point) {
1608 v
1609 } else {
1610 self.as_untyped_ref().get32_by_small_index_cold(code_point)
1611 }
1612 }
1613
1614 /// Lookup trie value by Unicode Code Point without branching on trie type.
1615 #[inline(always)]
1616 fn get32(&self, code_point: u32) -> T {
1617 if let Some(v) = self.get32_by_fast_index(code_point) {
1618 v
1619 } else if code_point <= CODE_POINT_MAX {
1620 self.as_untyped_ref().get32_by_small_index_cold(code_point)
1621 } else {
1622 self.as_untyped_ref().error_value
1623 }
1624 }
1625
1626 /// Returns the value that is associated with `code_point` in this [`CodePointTrie`]
1627 /// if `code_point` uses fast-path lookup or `None` if `code_point`
1628 /// should use small-path lookup or is above the supported range.
1629 #[inline(always)] // "always" to make the `Option` collapse away
1630 fn get32_by_fast_index(&self, code_point: u32) -> Option<T> {
1631 debug_assert_eq!(Self::TRIE_TYPE, self.as_untyped_ref().header.trie_type);
1632 let fast_max = match Self::TRIE_TYPE {
1633 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
1634 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
1635 };
1636 if code_point <= fast_max {
1637 // SAFETY: We just checked the invariant of
1638 // `get32_assuming_fast_index`,
1639 // which is
1640 // "If `self.header.trie_type == TrieType::Small`, `code_point` must be at most
1641 // `SMALL_TYPE_FAST_INDEXING_MAX`. If `self.header.trie_type ==
1642 // TrieType::Fast`, `code_point` must be at most `FAST_TYPE_FAST_INDEXING_MAX`."
1643 // ... assuming that `Self::TRIE_TYPE` always matches
1644 // `self.as_untyped_ref().header.trie_type`, i.e. we're relying on
1645 // `CodePointTrie::to_typed` and `CodePointTrie::as_typed_ref` being correct
1646 // and the exclusive ways of obtaining `Self`.
1647 Some(unsafe { self.as_untyped_ref().get32_assuming_fast_index(code_point) })
1648 } else {
1649 // The caller needs to call `get32_by_small_index` or determine
1650 // that the argument is above the permitted range.
1651 None
1652 }
1653 }
1654
1655 /// Returns the value that is associated with a two-byte UTF-8 sequence.
1656 ///
1657 /// `high_five` is the low five bits of the lead byte of a two-byte UTF-8 sequence.
1658 /// `low_six` is the low six bits of the trail byte of a two-byte UTF-8 sequence.
1659 ///
1660 /// # Safety
1661 ///
1662 /// `high_five` must not have bit positions other than the lowest 5 set to 1.
1663 /// `low_six` must not have bit positions other than the lowest 6 set to 1.
1664 ///
1665 /// # Panics
1666 ///
1667 /// With debug assertions enabled, panics if the above safety invariants are
1668 /// violated or `high_five` represents non-shortest form.
1669 #[inline(always)]
1670 unsafe fn get_utf8_two_byte(&self, high_five: u32, low_six: u32) -> T {
1671 self.as_untyped_ref().get_utf8_two_byte(high_five, low_six)
1672 }
1673
1674 /// Returns the value that is associated with a three-byte UTF-8 or WTF-8 sequence.
1675 ///
1676 /// `high_ten` is the low four bits of the lead byte of three-byte UTF-8 or WTF-8 sequence shifted left by 6 followed by the low six bits of the first trail byte.
1677 /// `low_six` is the low six bits of the last trail byte of a three-byte UTF-8 or WTF-8 sequence.
1678 ///
1679 /// Sequences representing surrogates (WTF-8) are allowed.
1680 ///
1681 /// # Safety
1682 ///
1683 /// `high_ten` must not have bit positions other than the lowest 10 set to 1.
1684 /// `low_six` must not have bit positions other than the lowest 6 set to 1.
1685 ///
1686 /// # Panics
1687 ///
1688 /// With debug assertions enabled, panics if the above safety invariants are
1689 /// violated or `high_ten` is out of range for three-byte WTF-8 (or UTF-8)
1690 /// sequence.
1691 #[inline(always)]
1692 #[allow(clippy::unusual_byte_groupings)]
1693 unsafe fn get_utf8_three_byte(&self, high_ten: u32, low_six: u32) -> T {
1694 debug_assert!(low_six <= 0b111_111); // Safety invariant.
1695 debug_assert!(high_ten <= 0b1111_111_111); // Not actually a _safety_ invariant for this impl.
1696 debug_assert!(high_ten > 0b11_111); // Non-shortest form; not safety invariant.
1697
1698 debug_assert_eq!(Self::TRIE_TYPE, self.as_untyped_ref().header.trie_type);
1699 let fast_max = match Self::TRIE_TYPE {
1700 TrieType::Fast => FAST_TYPE_FAST_INDEXING_MAX,
1701 TrieType::Small => SMALL_TYPE_FAST_INDEXING_MAX,
1702 };
1703
1704 // Keep only the prefix bits:
1705 let max_bit_prefix = fast_max >> FAST_TYPE_SHIFT;
1706 if high_ten <= max_bit_prefix {
1707 // SAFETY: The caller is responsible for upholding the safety
1708 // invariant for `low_six` and we just checked the safety
1709 // invariant of `high_ten`.
1710 self.as_untyped_ref()
1711 .get_bit_prefix_suffix_assuming_fast_index(high_ten as usize, low_six as usize)
1712 } else {
1713 self.as_untyped_ref()
1714 .get32_by_small_index_cold((high_ten << 6) | low_six)
1715 }
1716 }
1717
1718 /// Returns a reference to the wrapped `CodePointTrie`.
1719 fn as_untyped_ref(&self) -> &CodePointTrie<'trie, T>;
1720
1721 /// Extracts the wrapped `CodePointTrie`.
1722 fn to_untyped(self) -> CodePointTrie<'trie, T>;
1723}
1724
1725/// Type-safe wrapper for a fast trie guaranteeing
1726/// the the getters don't branch on the trie type
1727/// and for guarenteeing that `get16` is branchless
1728/// in release builds.
1729#[derive(Debug, Eq, PartialEq, Yokeable, ZeroFrom, Clone)]
1730#[repr(transparent)]
1731pub struct FastCodePointTrie<'trie, T: TrieValue> {
1732 inner: CodePointTrie<'trie, T>,
1733}
1734
1735impl<'trie, T: TrieValue> FastCodePointTrie<'trie, T> {
1736 #[doc(hidden)] // databake internal
1737 /// # Safety
1738 ///
1739 /// `header.trie_type`, `index`, and `data` must
1740 /// satisfy the invariants for the fields of the
1741 /// same names on `CodePointTrie`.
1742 pub const unsafe fn from_parts_unstable_unchecked_v1(
1743 header: CodePointTrieHeader,
1744 index: ZeroVec<'trie, u16>,
1745 data: ZeroVec<'trie, T>,
1746 error_value: T,
1747 ) -> Self {
1748 // Field invariants upheld: The caller is responsible.
1749 // In practice, this means that datagen in the databake
1750 // mode upholds these invariants when constructing the
1751 // `CodePointTrie` that is then baked.
1752 let untyped = CodePointTrie::<'trie, T>::from_parts_unstable_unchecked_v1(
1753 header,
1754 index,
1755 data,
1756 error_value,
1757 );
1758 Self { inner: untyped }
1759 }
1760}
1761
1762impl<'trie, T: TrieValue> TypedCodePointTrie<'trie, T> for FastCodePointTrie<'trie, T> {
1763 const TRIE_TYPE: TrieType = TrieType::Fast;
1764
1765 /// Returns a reference to the wrapped `CodePointTrie`.
1766 #[inline(always)]
1767 fn as_untyped_ref(&self) -> &CodePointTrie<'trie, T> {
1768 &self.inner
1769 }
1770
1771 /// Extracts the wrapped `CodePointTrie`.
1772 #[inline(always)]
1773 fn to_untyped(self) -> CodePointTrie<'trie, T> {
1774 self.inner
1775 }
1776
1777 /// Lookup trie value by Basic Multilingual Plane Code Point without branching on trie type.
1778 #[inline(always)]
1779 fn get16(&self, bmp: u16) -> T {
1780 debug_assert!(u32::from(u16::MAX) <= FAST_TYPE_FAST_INDEXING_MAX);
1781 debug_assert_eq!(Self::TRIE_TYPE, TrieType::Fast);
1782 debug_assert_eq!(self.as_untyped_ref().header.trie_type, TrieType::Fast);
1783 let code_point = u32::from(bmp);
1784 // SAFETY: With `TrieType::Fast`, the `u16` range satisfies
1785 // the invariant of `get32_assuming_fast_index`, which is
1786 // "If `self.header.trie_type == TrieType::Small`, `code_point` must be at most
1787 // `SMALL_TYPE_FAST_INDEXING_MAX`. If `self.header.trie_type ==
1788 // TrieType::Fast`, `code_point` must be at most `FAST_TYPE_FAST_INDEXING_MAX`."
1789 //
1790 // We're relying on `CodePointTrie::to_typed` and `CodePointTrie::as_typed_ref`
1791 // being correct and the exclusive ways of obtaining `Self`.
1792 unsafe { self.as_untyped_ref().get32_assuming_fast_index(code_point) }
1793 }
1794
1795 /// Returns the value that is associated with a three-byte UTF-8 or WTF-8 sequence.
1796 ///
1797 /// `high_ten` is the low four bits of the lead byte of three-byte UTF-8 or WTF-8 sequence shifted left by 6 followed by the low six bits of the first trail byte.
1798 /// `low_six` is the low six bits of the last trail byte of a three-byte UTF-8 or WTF-8 sequence.
1799 ///
1800 /// Sequences representing surrogates (WTF-8) are allowed.
1801 ///
1802 /// # Safety
1803 ///
1804 /// `high_ten` must not have bit positions other than the lowest 10 set to 1.
1805 /// `low_six` must not have bit positions other than the lowest 6 set to 1.
1806 ///
1807 /// # Panics
1808 ///
1809 /// With debug assertions enabled, panics if the above safety invariants are
1810 /// violated or `high_ten` is out of range for three-byte WTF-8 (or UTF-8)
1811 /// sequence.
1812 #[inline(always)]
1813 #[allow(clippy::unusual_byte_groupings)]
1814 unsafe fn get_utf8_three_byte(&self, high_ten: u32, low_six: u32) -> T {
1815 debug_assert!(low_six <= 0b111_111); // Safety invariant.
1816 debug_assert!(high_ten <= 0b1111_111_111); // Safety invariant.
1817 debug_assert!(high_ten > 0b11_111); // Non-shortest form; not safety invariant.
1818 debug_assert_eq!(Self::TRIE_TYPE, TrieType::Fast);
1819 debug_assert_eq!(self.as_untyped_ref().header.trie_type, TrieType::Fast);
1820 // SAFETY: The highest character representable as a three-byte
1821 // UTF-8 sequence is U+FFFF, which is `FAST_TYPE_FAST_INDEXING_MAX`.
1822 self.inner
1823 .get_bit_prefix_suffix_assuming_fast_index(high_ten as usize, low_six as usize)
1824 }
1825}
1826
1827impl<'trie, T: TrieValue> Seal for FastCodePointTrie<'trie, T> {}
1828
1829impl<'trie, T: TrieValue> TryFrom<&'trie CodePointTrie<'trie, T>>
1830 for &'trie FastCodePointTrie<'trie, T>
1831{
1832 type Error = TypedCodePointTrieError;
1833
1834 fn try_from(
1835 reference: &'trie CodePointTrie<'trie, T>,
1836 ) -> Result<&'trie FastCodePointTrie<'trie, T>, TypedCodePointTrieError> {
1837 match reference.as_typed_ref() {
1838 Typed::Fast(trie) => Ok(trie),
1839 Typed::Small(_) => Err(TypedCodePointTrieError),
1840 }
1841 }
1842}
1843
1844impl<'trie, T: TrieValue> TryFrom<CodePointTrie<'trie, T>> for FastCodePointTrie<'trie, T> {
1845 type Error = TypedCodePointTrieError;
1846
1847 fn try_from(
1848 value: CodePointTrie<'trie, T>,
1849 ) -> Result<FastCodePointTrie<'trie, T>, TypedCodePointTrieError> {
1850 match value.to_typed() {
1851 Typed::Fast(trie) => Ok(trie),
1852 Typed::Small(_) => Err(TypedCodePointTrieError),
1853 }
1854 }
1855}
1856
1857#[cfg(feature = "databake")]
1858impl<T: TrieValue + databake::Bake> databake::Bake for FastCodePointTrie<'_, T> {
1859 fn bake(&self, env: &databake::CrateEnv) -> databake::TokenStream {
1860 let header = self.inner.header.bake(env);
1861 let index = self.inner.index.bake(env);
1862 let data = self.inner.data.bake(env);
1863 let error_value = self.inner.error_value.bake(env);
1864 databake::quote! { unsafe { icu_collections::codepointtrie::FastCodePointTrie::from_parts_unstable_unchecked_v1(#header, #index, #data, #error_value) } }
1865 }
1866}
1867
1868#[cfg(feature = "databake")]
1869impl<T: TrieValue + databake::Bake> databake::BakeSize for FastCodePointTrie<'_, T> {
1870 fn borrows_size(&self) -> usize {
1871 self.inner.borrows_size()
1872 }
1873}
1874
1875/// Type-safe wrapper for a small trie guaranteeing
1876/// the the getters don't branch on the trie type.
1877#[derive(Debug, Eq, PartialEq, Yokeable, ZeroFrom, Clone)]
1878#[repr(transparent)]
1879pub struct SmallCodePointTrie<'trie, T: TrieValue> {
1880 inner: CodePointTrie<'trie, T>,
1881}
1882
1883impl<'trie, T: TrieValue> SmallCodePointTrie<'trie, T> {
1884 #[doc(hidden)] // databake internal
1885 /// # Safety
1886 ///
1887 /// `header.trie_type`, `index`, and `data` must
1888 /// satisfy the invariants for the fields of the
1889 /// same names on `CodePointTrie`.
1890 pub const unsafe fn from_parts_unstable_unchecked_v1(
1891 header: CodePointTrieHeader,
1892 index: ZeroVec<'trie, u16>,
1893 data: ZeroVec<'trie, T>,
1894 error_value: T,
1895 ) -> Self {
1896 // Field invariants upheld: The caller is responsible.
1897 // In practice, this means that datagen in the databake
1898 // mode upholds these invariants when constructing the
1899 // `CodePointTrie` that is then baked.
1900 let untyped = CodePointTrie::<'trie, T>::from_parts_unstable_unchecked_v1(
1901 header,
1902 index,
1903 data,
1904 error_value,
1905 );
1906 Self { inner: untyped }
1907 }
1908}
1909
1910impl<'trie, T: TrieValue> TypedCodePointTrie<'trie, T> for SmallCodePointTrie<'trie, T> {
1911 const TRIE_TYPE: TrieType = TrieType::Small;
1912
1913 /// Returns a reference to the wrapped `CodePointTrie`.
1914 #[inline(always)]
1915 fn as_untyped_ref(&self) -> &CodePointTrie<'trie, T> {
1916 &self.inner
1917 }
1918
1919 /// Extracts the wrapped `CodePointTrie`.
1920 #[inline(always)]
1921 fn to_untyped(self) -> CodePointTrie<'trie, T> {
1922 self.inner
1923 }
1924}
1925
1926impl<'trie, T: TrieValue> Seal for SmallCodePointTrie<'trie, T> {}
1927
1928impl<'trie, T: TrieValue> TryFrom<&'trie CodePointTrie<'trie, T>>
1929 for &'trie SmallCodePointTrie<'trie, T>
1930{
1931 type Error = TypedCodePointTrieError;
1932
1933 fn try_from(
1934 reference: &'trie CodePointTrie<'trie, T>,
1935 ) -> Result<&'trie SmallCodePointTrie<'trie, T>, TypedCodePointTrieError> {
1936 match reference.as_typed_ref() {
1937 Typed::Fast(_) => Err(TypedCodePointTrieError),
1938 Typed::Small(trie) => Ok(trie),
1939 }
1940 }
1941}
1942
1943impl<'trie, T: TrieValue> TryFrom<CodePointTrie<'trie, T>> for SmallCodePointTrie<'trie, T> {
1944 type Error = TypedCodePointTrieError;
1945
1946 fn try_from(
1947 value: CodePointTrie<'trie, T>,
1948 ) -> Result<SmallCodePointTrie<'trie, T>, TypedCodePointTrieError> {
1949 match value.to_typed() {
1950 Typed::Fast(_) => Err(TypedCodePointTrieError),
1951 Typed::Small(trie) => Ok(trie),
1952 }
1953 }
1954}
1955
1956#[cfg(feature = "databake")]
1957impl<T: TrieValue + databake::Bake> databake::Bake for SmallCodePointTrie<'_, T> {
1958 fn bake(&self, env: &databake::CrateEnv) -> databake::TokenStream {
1959 let header = self.inner.header.bake(env);
1960 let index = self.inner.index.bake(env);
1961 let data = self.inner.data.bake(env);
1962 let error_value = self.inner.error_value.bake(env);
1963 databake::quote! { unsafe { icu_collections::codepointtrie::SmallCodePointTrie::from_parts_unstable_unchecked_v1(#header, #index, #data, #error_value) } }
1964 }
1965}
1966
1967#[cfg(feature = "databake")]
1968impl<T: TrieValue + databake::Bake> databake::BakeSize for SmallCodePointTrie<'_, T> {
1969 fn borrows_size(&self) -> usize {
1970 self.inner.borrows_size()
1971 }
1972}
1973
1974/// Error indicating that the `TrieType` of an untyped trie
1975/// does not match the requested typed trie type.
1976#[derive(Debug)]
1977#[non_exhaustive]
1978pub struct TypedCodePointTrieError;
1979
1980/// Holder for either fast or small trie with the trie
1981/// type encoded into the Rust type.
1982#[allow(clippy::exhaustive_enums)]
1983#[derive(Debug)]
1984pub enum Typed<F, S> {
1985 /// The trie type is fast.
1986 Fast(F),
1987 /// The trie type is small.
1988 Small(S),
1989}
1990
1991/// Trait for writing trait bounds for monomorphizing over either
1992/// `CodePointTrie`, `FastCodePointTrie`, or `SmallCodePointTrie`.
1993///
1994/// Method naming intentionally differs from the method naming on
1995/// those types in order to disambiguate.
1996#[allow(private_bounds)] // Permit sealing
1997pub trait AbstractCodePointTrie<'trie, T: TrieValue>: Seal {
1998 /// Look up trie value by an ASCII character.
1999 ///
2000 /// # Safety
2001 ///
2002 /// `ascii` must be less than 128.
2003 unsafe fn ascii(&self, ascii: u8) -> T;
2004
2005 /// Look up trie value by a two-byte UTF-8 sequence.
2006 ///
2007 /// `high_five` is the low five bits of the lead byte of a two-byte UTF-8 sequence.
2008 /// `low_six` is the low six bits of the trail byte of a two-byte UTF-8 sequence.
2009 ///
2010 /// # Safety
2011 ///
2012 /// `high_five` must not have bit positions other than the lowest 5 set to 1.
2013 /// `low_six` must not have bit positions other than the lowest 6 set to 1.
2014 unsafe fn utf8_two_byte(&self, high_five: u32, low_six: u32) -> T;
2015
2016 /// Look up trie value by a three-byte UTF-8 or WTF-8 sequence.
2017 ///
2018 /// `high_ten` is the low four bits of the lead byte of three-byte UTF-8 or WTF-8 sequence shifted left by 6 followed by the low six bits of the first trail byte.
2019 /// `low_six` is the low six bits of the last trail byte of a three-byte UTF-8 or WTF-8 sequence.
2020 ///
2021 /// Sequences representing surrogates (WTF-8) are allowed.
2022 ///
2023 /// # Safety
2024 ///
2025 /// `high_ten` must not have bit positions other than the lowest 10 set to 1.
2026 /// `low_six` must not have bit positions other than the lowest 6 set to 1.
2027 unsafe fn utf8_three_byte(&self, high_ten: u32, low_six: u32) -> T;
2028
2029 /// Look up trie value by a Latin1 character.
2030 fn latin1(&self, latin1: u8) -> T;
2031
2032 /// Look up trie value by a Basic Multilingual Plane character.
2033 ///
2034 /// Surrogate values are allowed.
2035 fn bmp(&self, bmp: u16) -> T;
2036
2037 /// Look up trie value by a non-Basic Multilingual Plane character.
2038 ///
2039 /// The behavior is memory-safe nonsense if the argument is not
2040 /// actually a non-Basic Multilingual Plane character.
2041 fn supplementary(&self, supplementary: u32) -> T;
2042
2043 /// Look up trie value by a Unicode Scalar Value.
2044 fn scalar(&self, scalar: char) -> T;
2045
2046 /// Look up trie value by Unicode Code Point.
2047 ///
2048 /// Surrogate values are allowed. Out of range input
2049 /// results in the error value.
2050 fn code_point(&self, code_point: u32) -> T;
2051}
2052
2053impl<'trie, T: TrieValue> AbstractCodePointTrie<'trie, T> for FastCodePointTrie<'trie, T> {
2054 #[inline(always)]
2055 unsafe fn ascii(&self, ascii: u8) -> T {
2056 self.get7(ascii)
2057 }
2058
2059 #[inline(always)]
2060 unsafe fn utf8_two_byte(&self, high_five: u32, low_six: u32) -> T {
2061 self.get_utf8_two_byte(high_five, low_six)
2062 }
2063
2064 #[inline(always)]
2065 unsafe fn utf8_three_byte(&self, high_ten: u32, low_six: u32) -> T {
2066 self.get_utf8_three_byte(high_ten, low_six)
2067 }
2068
2069 #[inline(always)]
2070 fn latin1(&self, latin1: u8) -> T {
2071 self.get8(latin1)
2072 }
2073
2074 #[inline(always)]
2075 fn bmp(&self, bmp: u16) -> T {
2076 self.get16(bmp)
2077 }
2078
2079 #[inline(always)]
2080 fn supplementary(&self, supplementary: u32) -> T {
2081 self.get32_supplementary(supplementary)
2082 }
2083
2084 #[inline(always)]
2085 fn scalar(&self, scalar: char) -> T {
2086 self.get(scalar)
2087 }
2088
2089 #[inline(always)]
2090 fn code_point(&self, code_point: u32) -> T {
2091 self.get32(code_point)
2092 }
2093}
2094
2095impl<'trie, T: TrieValue> AbstractCodePointTrie<'trie, T> for SmallCodePointTrie<'trie, T> {
2096 #[inline(always)]
2097 unsafe fn ascii(&self, ascii: u8) -> T {
2098 self.get7(ascii)
2099 }
2100
2101 #[inline(always)]
2102 unsafe fn utf8_two_byte(&self, high_five: u32, low_six: u32) -> T {
2103 self.get_utf8_two_byte(high_five, low_six)
2104 }
2105
2106 #[inline(always)]
2107 unsafe fn utf8_three_byte(&self, high_ten: u32, low_six: u32) -> T {
2108 self.get_utf8_three_byte(high_ten, low_six)
2109 }
2110
2111 #[inline(always)]
2112 fn latin1(&self, latin1: u8) -> T {
2113 self.get8(latin1)
2114 }
2115
2116 #[inline(always)]
2117 fn bmp(&self, bmp: u16) -> T {
2118 self.get16(bmp)
2119 }
2120
2121 #[inline(always)]
2122 fn supplementary(&self, supplementary: u32) -> T {
2123 self.get32_supplementary(supplementary)
2124 }
2125
2126 #[inline(always)]
2127 fn scalar(&self, scalar: char) -> T {
2128 self.get(scalar)
2129 }
2130
2131 #[inline(always)]
2132 fn code_point(&self, code_point: u32) -> T {
2133 self.get32(code_point)
2134 }
2135}
2136
2137impl<'trie, T: TrieValue> AbstractCodePointTrie<'trie, T> for CodePointTrie<'trie, T> {
2138 #[inline(always)]
2139 unsafe fn ascii(&self, ascii: u8) -> T {
2140 self.get7(ascii)
2141 }
2142
2143 #[inline(always)]
2144 unsafe fn utf8_two_byte(&self, high_five: u32, low_six: u32) -> T {
2145 self.get_utf8_two_byte(high_five, low_six)
2146 }
2147
2148 #[inline(always)]
2149 unsafe fn utf8_three_byte(&self, high_ten: u32, low_six: u32) -> T {
2150 self.get_utf8_three_byte(high_ten, low_six)
2151 }
2152
2153 #[inline(always)]
2154 fn latin1(&self, latin1: u8) -> T {
2155 self.get8(latin1)
2156 }
2157
2158 #[inline(always)]
2159 fn bmp(&self, bmp: u16) -> T {
2160 self.get16(bmp)
2161 }
2162
2163 #[inline(always)]
2164 fn supplementary(&self, supplementary: u32) -> T {
2165 self.get32_supplementary(supplementary)
2166 }
2167
2168 #[inline(always)]
2169 fn scalar(&self, scalar: char) -> T {
2170 self.get(scalar)
2171 }
2172
2173 #[inline(always)]
2174 fn code_point(&self, code_point: u32) -> T {
2175 self.get32(code_point)
2176 }
2177}
2178
2179#[cfg(test)]
2180mod tests {
2181 use super::*;
2182 use crate::codepointtrie::planes;
2183 use alloc::vec::Vec;
2184
2185 #[test]
2186 #[cfg(feature = "serde")]
2187 fn test_serde_with_postcard_roundtrip() -> Result<(), postcard::Error> {
2188 let trie = planes::get_planes_trie();
2189 let trie_serialized: Vec<u8> = postcard::to_allocvec(&trie).unwrap();
2190
2191 // Assert an expected (golden data) version of the serialized trie.
2192 const EXP_TRIE_SERIALIZED: &[u8] = &[
2193 128, 128, 64, 128, 2, 2, 0, 0, 1, 160, 18, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2194 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2195 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2196 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2197 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136,
2198 2, 144, 2, 144, 2, 144, 2, 176, 2, 176, 2, 176, 2, 176, 2, 208, 2, 208, 2, 208, 2, 208,
2199 2, 240, 2, 240, 2, 240, 2, 240, 2, 16, 3, 16, 3, 16, 3, 16, 3, 48, 3, 48, 3, 48, 3, 48,
2200 3, 80, 3, 80, 3, 80, 3, 80, 3, 112, 3, 112, 3, 112, 3, 112, 3, 144, 3, 144, 3, 144, 3,
2201 144, 3, 176, 3, 176, 3, 176, 3, 176, 3, 208, 3, 208, 3, 208, 3, 208, 3, 240, 3, 240, 3,
2202 240, 3, 240, 3, 16, 4, 16, 4, 16, 4, 16, 4, 48, 4, 48, 4, 48, 4, 48, 4, 80, 4, 80, 4,
2203 80, 4, 80, 4, 112, 4, 112, 4, 112, 4, 112, 4, 0, 0, 16, 0, 32, 0, 48, 0, 64, 0, 80, 0,
2204 96, 0, 112, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32,
2205 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48,
2206 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 128, 0, 128, 0, 128, 0, 128,
2207 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128,
2208 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128,
2209 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144,
2210 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144,
2211 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144,
2212 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160,
2213 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160,
2214 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160,
2215 0, 160, 0, 160, 0, 160, 0, 160, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176,
2216 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176,
2217 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176,
2218 0, 176, 0, 176, 0, 176, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192,
2219 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192,
2220 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192,
2221 0, 192, 0, 192, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208,
2222 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208,
2223 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208,
2224 0, 208, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224,
2225 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224,
2226 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224,
2227 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240,
2228 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240,
2229 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 0,
2230 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
2231 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
2232 1, 0, 1, 0, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1,
2233 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16,
2234 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 32, 1, 32, 1, 32, 1,
2235 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32,
2236 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1,
2237 32, 1, 32, 1, 32, 1, 32, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48,
2238 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1,
2239 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 64, 1, 64,
2240 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1,
2241 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64,
2242 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1,
2243 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80,
2244 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1,
2245 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96,
2246 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1,
2247 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 128, 0, 136, 0, 136, 0, 136, 0, 136,
2248 0, 136, 0, 136, 0, 136, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0,
2249 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2,
2250 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0,
2251 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0,
2252 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0,
2253 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0,
2254 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0,
2255 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0,
2256 200, 0, 200, 0, 200, 0, 200, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0,
2257 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0,
2258 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0,
2259 232, 0, 232, 0, 232, 0, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8,
2260 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1,
2261 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40,
2262 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1,
2263 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40,
2264 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1,
2265 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72,
2266 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 104, 1, 104, 1, 104, 1, 104, 1,
2267 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1,
2268 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1,
2269 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1,
2270 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1,
2271 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1,
2272 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1,
2273 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1,
2274 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1,
2275 168, 1, 168, 1, 168, 1, 168, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1,
2276 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1,
2277 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1,
2278 200, 1, 200, 1, 200, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1,
2279 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1,
2280 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1,
2281 232, 1, 232, 1, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2,
2282 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8,
2283 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40,
2284 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2,
2285 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 72,
2286 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2,
2287 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72,
2288 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2,
2289 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2,
2290 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2,
2291 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 244, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2292 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2293 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2294 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2295 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
2296 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2297 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
2298 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6,
2299 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8,
2300 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10,
2301 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11,
2302 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
2303 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14,
2304 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15,
2305 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 0,
2306 ];
2307 assert_eq!(trie_serialized, EXP_TRIE_SERIALIZED);
2308
2309 let trie_deserialized = postcard::from_bytes::<CodePointTrie<u8>>(&trie_serialized)?;
2310
2311 assert_eq!(&trie.index, &trie_deserialized.index);
2312 assert_eq!(&trie.data, &trie_deserialized.data);
2313
2314 assert!(!trie_deserialized.index.is_owned());
2315 assert!(!trie_deserialized.data.is_owned());
2316
2317 Ok(())
2318 }
2319
2320 #[test]
2321 fn test_typed() {
2322 let untyped = planes::get_planes_trie();
2323 assert_eq!(untyped.get('\u{10000}'), 1);
2324 let small_ref = <&SmallCodePointTrie<_>>::try_from(&untyped).unwrap();
2325 assert_eq!(small_ref.get('\u{10000}'), 1);
2326 let _ = <&FastCodePointTrie<_>>::try_from(&untyped).is_err();
2327 let small = <SmallCodePointTrie<_>>::try_from(untyped).unwrap();
2328 assert_eq!(small.get('\u{10000}'), 1);
2329 }
2330
2331 #[test]
2332 #[cfg(feature = "serde")]
2333 fn test_serde_with_postcard_roundtrip_small() -> Result<(), postcard::Error> {
2334 let untyped = planes::get_planes_trie();
2335 let trie = <SmallCodePointTrie<_>>::try_from(untyped.clone()).unwrap();
2336
2337 let trie_serialized: Vec<u8> = postcard::to_allocvec(&trie).unwrap();
2338
2339 // Assert an expected (golden data) version of the serialized trie.
2340 const EXP_TRIE_SERIALIZED: &[u8] = &[
2341 128, 128, 64, 128, 2, 2, 0, 0, 1, 160, 18, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2342 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2343 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2344 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2345 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136,
2346 2, 144, 2, 144, 2, 144, 2, 176, 2, 176, 2, 176, 2, 176, 2, 208, 2, 208, 2, 208, 2, 208,
2347 2, 240, 2, 240, 2, 240, 2, 240, 2, 16, 3, 16, 3, 16, 3, 16, 3, 48, 3, 48, 3, 48, 3, 48,
2348 3, 80, 3, 80, 3, 80, 3, 80, 3, 112, 3, 112, 3, 112, 3, 112, 3, 144, 3, 144, 3, 144, 3,
2349 144, 3, 176, 3, 176, 3, 176, 3, 176, 3, 208, 3, 208, 3, 208, 3, 208, 3, 240, 3, 240, 3,
2350 240, 3, 240, 3, 16, 4, 16, 4, 16, 4, 16, 4, 48, 4, 48, 4, 48, 4, 48, 4, 80, 4, 80, 4,
2351 80, 4, 80, 4, 112, 4, 112, 4, 112, 4, 112, 4, 0, 0, 16, 0, 32, 0, 48, 0, 64, 0, 80, 0,
2352 96, 0, 112, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32,
2353 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48,
2354 0, 0, 0, 16, 0, 32, 0, 48, 0, 0, 0, 16, 0, 32, 0, 48, 0, 128, 0, 128, 0, 128, 0, 128,
2355 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128,
2356 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128,
2357 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144,
2358 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144,
2359 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 144,
2360 0, 144, 0, 144, 0, 144, 0, 144, 0, 144, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160,
2361 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160,
2362 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160, 0, 160,
2363 0, 160, 0, 160, 0, 160, 0, 160, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176,
2364 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176,
2365 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176, 0, 176,
2366 0, 176, 0, 176, 0, 176, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192,
2367 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192,
2368 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192, 0, 192,
2369 0, 192, 0, 192, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208,
2370 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208,
2371 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208, 0, 208,
2372 0, 208, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224,
2373 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224,
2374 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224, 0, 224,
2375 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240,
2376 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240,
2377 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 240, 0, 0,
2378 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
2379 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
2380 1, 0, 1, 0, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1,
2381 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16,
2382 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 32, 1, 32, 1, 32, 1,
2383 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32,
2384 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1,
2385 32, 1, 32, 1, 32, 1, 32, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48,
2386 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1,
2387 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 64, 1, 64,
2388 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1,
2389 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64,
2390 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1,
2391 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80,
2392 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1,
2393 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96,
2394 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1,
2395 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 96, 1, 128, 0, 136, 0, 136, 0, 136, 0, 136,
2396 0, 136, 0, 136, 0, 136, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0,
2397 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2,
2398 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0,
2399 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0,
2400 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 168, 0,
2401 168, 0, 168, 0, 168, 0, 168, 0, 168, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0,
2402 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0,
2403 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0, 200, 0,
2404 200, 0, 200, 0, 200, 0, 200, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0,
2405 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0,
2406 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0, 232, 0,
2407 232, 0, 232, 0, 232, 0, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8,
2408 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1,
2409 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40,
2410 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1,
2411 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40,
2412 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1,
2413 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72,
2414 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 104, 1, 104, 1, 104, 1, 104, 1,
2415 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1,
2416 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1,
2417 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 104, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1,
2418 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1,
2419 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1,
2420 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1,
2421 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1,
2422 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1,
2423 168, 1, 168, 1, 168, 1, 168, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1,
2424 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1,
2425 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1, 200, 1,
2426 200, 1, 200, 1, 200, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1,
2427 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1,
2428 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1,
2429 232, 1, 232, 1, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2,
2430 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 8,
2431 2, 8, 2, 8, 2, 8, 2, 8, 2, 8, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40,
2432 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2,
2433 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 40, 2, 72,
2434 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2,
2435 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72,
2436 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 72, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2,
2437 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2,
2438 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 104, 2,
2439 104, 2, 104, 2, 104, 2, 104, 2, 104, 2, 244, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2440 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2441 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2442 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2443 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
2444 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2445 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
2446 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6,
2447 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8,
2448 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10,
2449 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11,
2450 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
2451 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14,
2452 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15,
2453 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 0,
2454 ];
2455 assert_eq!(trie_serialized, EXP_TRIE_SERIALIZED);
2456
2457 let trie_deserialized = postcard::from_bytes::<SmallCodePointTrie<u8>>(&trie_serialized)?;
2458
2459 let trie_deserialized_untyped = trie_deserialized.as_untyped_ref();
2460 assert_eq!(&untyped.index, &trie_deserialized_untyped.index);
2461 assert_eq!(&untyped.data, &trie_deserialized_untyped.data);
2462
2463 assert!(!trie_deserialized_untyped.index.is_owned());
2464 assert!(!trie_deserialized_untyped.data.is_owned());
2465
2466 Ok(())
2467 }
2468
2469 #[test]
2470 fn test_get_range() {
2471 let planes_trie = planes::get_planes_trie();
2472
2473 let first_range: Option<CodePointMapRange<u8>> = planes_trie.get_range(0x0);
2474 assert_eq!(
2475 first_range,
2476 Some(CodePointMapRange {
2477 range: 0x0..=0xffff,
2478 value: 0
2479 })
2480 );
2481
2482 let second_range: Option<CodePointMapRange<u8>> = planes_trie.get_range(0x1_0000);
2483 assert_eq!(
2484 second_range,
2485 Some(CodePointMapRange {
2486 range: 0x10000..=0x1ffff,
2487 value: 1
2488 })
2489 );
2490
2491 let penultimate_range: Option<CodePointMapRange<u8>> = planes_trie.get_range(0xf_0000);
2492 assert_eq!(
2493 penultimate_range,
2494 Some(CodePointMapRange {
2495 range: 0xf_0000..=0xf_ffff,
2496 value: 15
2497 })
2498 );
2499
2500 let last_range: Option<CodePointMapRange<u8>> = planes_trie.get_range(0x10_0000);
2501 assert_eq!(
2502 last_range,
2503 Some(CodePointMapRange {
2504 range: 0x10_0000..=0x10_ffff,
2505 value: 16
2506 })
2507 );
2508 }
2509
2510 #[test]
2511 #[allow(unused_unsafe)] // `unsafe` below is both necessary and unnecessary
2512 fn databake() {
2513 databake::test_bake!(
2514 CodePointTrie<'static, u32>,
2515 const,
2516 unsafe {
2517 crate::codepointtrie::CodePointTrie::from_parts_unstable_unchecked_v1(
2518 crate::codepointtrie::CodePointTrieHeader {
2519 high_start: 1u32,
2520 shifted12_high_start: 2u16,
2521 index3_null_offset: 3u16,
2522 data_null_offset: 4u32,
2523 null_value: 5u32,
2524 trie_type: crate::codepointtrie::TrieType::Small,
2525 },
2526 zerovec::ZeroVec::new(),
2527 zerovec::ZeroVec::new(),
2528 0u32,
2529 )
2530 },
2531 icu_collections,
2532 [zerovec],
2533 );
2534 }
2535
2536 #[test]
2537 #[allow(unused_unsafe)] // `unsafe` below is both necessary and unnecessary
2538 fn databake_small() {
2539 databake::test_bake!(
2540 SmallCodePointTrie<'static, u32>,
2541 const,
2542 unsafe {
2543 crate::codepointtrie::SmallCodePointTrie::from_parts_unstable_unchecked_v1(
2544 crate::codepointtrie::CodePointTrieHeader {
2545 high_start: 1u32,
2546 shifted12_high_start: 2u16,
2547 index3_null_offset: 3u16,
2548 data_null_offset: 4u32,
2549 null_value: 5u32,
2550 trie_type: crate::codepointtrie::TrieType::Small,
2551 },
2552 zerovec::ZeroVec::new(),
2553 zerovec::ZeroVec::new(),
2554 0u32,
2555 )
2556 },
2557 icu_collections,
2558 [zerovec],
2559 );
2560 }
2561
2562 #[test]
2563 #[allow(unused_unsafe)] // `unsafe` below is both necessary and unnecessary
2564 fn databake_fast() {
2565 databake::test_bake!(
2566 FastCodePointTrie<'static, u32>,
2567 const,
2568 unsafe {
2569 crate::codepointtrie::FastCodePointTrie::from_parts_unstable_unchecked_v1(
2570 crate::codepointtrie::CodePointTrieHeader {
2571 high_start: 1u32,
2572 shifted12_high_start: 2u16,
2573 index3_null_offset: 3u16,
2574 data_null_offset: 4u32,
2575 null_value: 5u32,
2576 trie_type: crate::codepointtrie::TrieType::Fast,
2577 },
2578 zerovec::ZeroVec::new(),
2579 zerovec::ZeroVec::new(),
2580 0u32,
2581 )
2582 },
2583 icu_collections,
2584 [zerovec],
2585 );
2586 }
2587}