1#![allow(
4 non_camel_case_types,
5 non_snake_case,
6 non_upper_case_globals,
7 clippy::inline_always,
8 clippy::wrong_self_convention,
9 clippy::enum_glob_use
10)]
11
12#[expect(clippy::identity_op)]
13#[rustfmt::skip]
14mod emoji_table;
15#[cfg(not(feature = "icu"))]
16#[rustfmt::skip]
17mod ucd_table;
18
19use crate::Script;
20
21pub(crate) type Codepoint = u32;
22
23#[derive(Clone, Copy, Debug, Eq, PartialEq)]
25pub enum Decomposed {
26 Singleton(u32),
28 Pair(u32, u32),
30}
31
32#[inline]
34pub fn compose(a: u32, b: u32) -> Option<u32> {
35 compose_impl(a, b)
36}
37
38#[inline]
40pub fn decompose(codepoint: u32) -> Option<Decomposed> {
41 decompose_impl(codepoint).map(|(a, b)| {
42 if b == 0 {
43 Decomposed::Singleton(a)
44 } else {
45 Decomposed::Pair(a, b)
46 }
47 })
48}
49
50#[inline]
52pub fn combining_class(codepoint: u32) -> u8 {
53 combining_class_for(codepoint)
54}
55
56#[inline]
60pub fn script(codepoint: u32) -> Script {
61 if codepoint > 0x10_FFFF {
62 Script::UNKNOWN
63 } else {
64 script_for(codepoint)
65 }
66}
67
68pub(crate) mod space_fallback {
72 pub type SpaceCode = u8;
73 pub const NOT_SPACE: u8 = 0;
74 pub const SPACE_EM: u8 = 1;
75 pub const SPACE_EM_2: u8 = 2;
76 pub const SPACE_EM_3: u8 = 3;
77 pub const SPACE_EM_4: u8 = 4;
78 pub const SPACE_EM_5: u8 = 5;
79 pub const SPACE_EM_6: u8 = 6;
80 pub const SPACE_EM_16: u8 = 16;
81 pub const SPACE_4_EM_18: u8 = 17; pub const SPACE: u8 = 18;
83 pub const SPACE_FIGURE: u8 = 19;
84 pub const SPACE_PUNCTUATION: u8 = 20;
85 pub const SPACE_NARROW: u8 = 21;
86}
87
88#[derive(Copy, Clone, PartialEq, Eq, Debug)]
91pub(crate) struct GeneralCategory(pub u8);
92
93#[allow(unused)]
94impl GeneralCategory {
95 pub const CONTROL: Self = Self(0);
97 pub const FORMAT: Self = Self(1);
99 pub const UNASSIGNED: Self = Self(2);
101 pub const PRIVATE_USE: Self = Self(3);
103 pub const SURROGATE: Self = Self(4);
105 pub const LOWERCASE_LETTER: Self = Self(5);
107 pub const MODIFIER_LETTER: Self = Self(6);
109 pub const OTHER_LETTER: Self = Self(7);
111 pub const TITLECASE_LETTER: Self = Self(8);
113 pub const UPPERCASE_LETTER: Self = Self(9);
115 pub const SPACING_MARK: Self = Self(10);
117 pub const ENCLOSING_MARK: Self = Self(11);
119 pub const NON_SPACING_MARK: Self = Self(12);
121 pub const DECIMAL_NUMBER: Self = Self(13);
123 pub const LETTER_NUMBER: Self = Self(14);
125 pub const OTHER_NUMBER: Self = Self(15);
127 pub const CONNECT_PUNCTUATION: Self = Self(16);
129 pub const DASH_PUNCTUATION: Self = Self(17);
131 pub const CLOSE_PUNCTUATION: Self = Self(18);
133 pub const FINAL_PUNCTUATION: Self = Self(19);
135 pub const INITIAL_PUNCTUATION: Self = Self(20);
137 pub const OTHER_PUNCTUATION: Self = Self(21);
139 pub const OPEN_PUNCTUATION: Self = Self(22);
141 pub const CURRENCY_SYMBOL: Self = Self(23);
143 pub const MODIFIER_SYMBOL: Self = Self(24);
145 pub const MATH_SYMBOL: Self = Self(25);
147 pub const OTHER_SYMBOL: Self = Self(26);
149 pub const LINE_SEPARATOR: Self = Self(27);
151 pub const PARAGRAPH_SEPARATOR: Self = Self(28);
153 pub const SPACE_SEPARATOR: Self = Self(29);
155}
156
157impl GeneralCategory {
158 pub fn is_mark(&self) -> bool {
159 matches!(
160 *self,
161 Self::SPACING_MARK | Self::ENCLOSING_MARK | Self::NON_SPACING_MARK
162 )
163 }
164
165 pub fn is_letter(&self) -> bool {
166 matches!(
167 *self,
168 Self::LOWERCASE_LETTER
169 | Self::MODIFIER_LETTER
170 | Self::OTHER_LETTER
171 | Self::TITLECASE_LETTER
172 | Self::UPPERCASE_LETTER
173 )
174 }
175
176 #[inline(always)]
177 pub(crate) const fn flag(self) -> u32 {
178 1 << self.0
179 }
180
181 #[inline(always)]
182 pub(crate) const fn flag_unsafe(self) -> u32 {
183 if self.0 < 32 {
184 1 << self.0
185 } else {
186 0
187 }
188 }
189
190 #[inline(always)]
191 pub(crate) const fn in_range_inclusive(self, start: Self, end: Self) -> bool {
192 debug_assert!(start.0 < end.0);
193 let range_mask = (1 << (end.0 + 1)) - (1 << start.0);
194 self.flag_unsafe() & range_mask != 0
195 }
196}
197
198#[allow(dead_code)]
199pub(crate) mod combining_class {
200 pub const NotReordered: u8 = 0;
201 pub const Overlay: u8 = 1;
202 pub const Nukta: u8 = 7;
203 pub const KanaVoicing: u8 = 8;
204 pub const Virama: u8 = 9;
205
206 pub const CCC10: u8 = 10;
208 pub const CCC11: u8 = 11;
209 pub const CCC12: u8 = 12;
210 pub const CCC13: u8 = 13;
211 pub const CCC14: u8 = 14;
212 pub const CCC15: u8 = 15;
213 pub const CCC16: u8 = 16;
214 pub const CCC17: u8 = 17;
215 pub const CCC18: u8 = 18;
216 pub const CCC19: u8 = 19;
217 pub const CCC20: u8 = 20;
218 pub const CCC21: u8 = 21;
219 pub const CCC22: u8 = 22;
220 pub const CCC23: u8 = 23;
221 pub const CCC24: u8 = 24;
222 pub const CCC25: u8 = 25;
223 pub const CCC26: u8 = 26;
224
225 pub const CCC27: u8 = 27;
227 pub const CCC28: u8 = 28;
228 pub const CCC29: u8 = 29;
229 pub const CCC30: u8 = 30;
230 pub const CCC31: u8 = 31;
231 pub const CCC32: u8 = 32;
232 pub const CCC33: u8 = 33;
233 pub const CCC34: u8 = 34;
234 pub const CCC35: u8 = 35;
235
236 pub const CCC36: u8 = 36;
238
239 pub const CCC84: u8 = 84;
241 pub const CCC91: u8 = 91;
242
243 pub const CCC103: u8 = 103;
245 pub const CCC107: u8 = 107;
246
247 pub const CCC118: u8 = 118;
249 pub const CCC122: u8 = 122;
250
251 pub const CCC129: u8 = 129;
253 pub const CCC130: u8 = 130;
254 pub const CCC132: u8 = 132;
255
256 pub const AttachedBelowLeft: u8 = 200;
257 pub const AttachedBelow: u8 = 202;
258 pub const AttachedAbove: u8 = 214;
259 pub const AttachedAboveRight: u8 = 216;
260 pub const BelowLeft: u8 = 218;
261 pub const Below: u8 = 220;
262 pub const BelowRight: u8 = 222;
263 pub const Left: u8 = 224;
264 pub const Right: u8 = 226;
265 pub const AboveLeft: u8 = 228;
266 pub const Above: u8 = 230;
267 pub const AboveRight: u8 = 232;
268 pub const DoubleBelow: u8 = 233;
269 pub const DoubleAbove: u8 = 234;
270
271 pub const IotaSubscript: u8 = 240;
272
273 pub const Invalid: u8 = 255;
274}
275
276#[allow(dead_code)]
277pub(crate) mod modified_combining_class {
278 pub const CCC10: u8 = 22; pub const CCC11: u8 = 15; pub const CCC12: u8 = 16; pub const CCC13: u8 = 17; pub const CCC14: u8 = 23; pub const CCC15: u8 = 18; pub const CCC16: u8 = 19; pub const CCC17: u8 = 20; pub const CCC18: u8 = 21; pub const CCC19: u8 = 14; pub const CCC20: u8 = 24; pub const CCC21: u8 = 12; pub const CCC22: u8 = 25; pub const CCC23: u8 = 13; pub const CCC24: u8 = 10; pub const CCC25: u8 = 11; pub const CCC26: u8 = 26; pub const CCC27: u8 = 28; pub const CCC28: u8 = 29; pub const CCC29: u8 = 30; pub const CCC30: u8 = 31; pub const CCC31: u8 = 32; pub const CCC32: u8 = 33; pub const CCC33: u8 = 27; pub const CCC34: u8 = 34; pub const CCC35: u8 = 35; pub const CCC36: u8 = 36; pub const CCC84: u8 = 4; pub const CCC91: u8 = 5; pub const CCC103: u8 = 3; pub const CCC107: u8 = 107; pub const CCC118: u8 = 118; pub const CCC122: u8 = 122; pub const CCC129: u8 = 129; pub const CCC130: u8 = 132; pub const CCC132: u8 = 131; }
355
356#[rustfmt::skip]
357static MODIFIED_COMBINING_CLASS: &[u8; 256] = &[
358 combining_class::NotReordered,
359 combining_class::Overlay,
360 2, 3, 4, 5, 6,
361 combining_class::Nukta,
362 combining_class::KanaVoicing,
363 combining_class::Virama,
364
365 modified_combining_class::CCC10,
367 modified_combining_class::CCC11,
368 modified_combining_class::CCC12,
369 modified_combining_class::CCC13,
370 modified_combining_class::CCC14,
371 modified_combining_class::CCC15,
372 modified_combining_class::CCC16,
373 modified_combining_class::CCC17,
374 modified_combining_class::CCC18,
375 modified_combining_class::CCC19,
376 modified_combining_class::CCC20,
377 modified_combining_class::CCC21,
378 modified_combining_class::CCC22,
379 modified_combining_class::CCC23,
380 modified_combining_class::CCC24,
381 modified_combining_class::CCC25,
382 modified_combining_class::CCC26,
383
384 modified_combining_class::CCC27,
386 modified_combining_class::CCC28,
387 modified_combining_class::CCC29,
388 modified_combining_class::CCC30,
389 modified_combining_class::CCC31,
390 modified_combining_class::CCC32,
391 modified_combining_class::CCC33,
392 modified_combining_class::CCC34,
393 modified_combining_class::CCC35,
394
395 modified_combining_class::CCC36,
397
398 37, 38, 39,
399 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
400 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
401 80, 81, 82, 83,
402
403 modified_combining_class::CCC84,
405 85, 86, 87, 88, 89, 90,
406 modified_combining_class::CCC91,
407 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102,
408
409 modified_combining_class::CCC103,
411 104, 105, 106,
412 modified_combining_class::CCC107,
413 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
414
415 modified_combining_class::CCC118,
417 119, 120, 121,
418 modified_combining_class::CCC122,
419 123, 124, 125, 126, 127, 128,
420
421 modified_combining_class::CCC129,
423 modified_combining_class::CCC130,
424 131,
425 modified_combining_class::CCC132,
426 133, 134, 135, 136, 137, 138, 139,
427
428
429 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
430 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
431 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
432 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
433 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
434 190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
435
436 combining_class::AttachedBelowLeft,
437 201,
438 combining_class::AttachedBelow,
439 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213,
440 combining_class::AttachedAbove,
441 215,
442 combining_class::AttachedAboveRight,
443 217,
444 combining_class::BelowLeft,
445 219,
446 combining_class::Below,
447 221,
448 combining_class::BelowRight,
449 223,
450 combining_class::Left,
451 225,
452 combining_class::Right,
453 227,
454 combining_class::AboveLeft,
455 229,
456 combining_class::Above,
457 231,
458 combining_class::AboveRight,
459 combining_class::DoubleBelow,
460 combining_class::DoubleAbove,
461 235, 236, 237, 238, 239,
462 combining_class::IotaSubscript,
463 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254,
464 combining_class::Invalid,
465];
466
467pub(crate) trait CharExt {
468 fn script(self) -> Script;
469 fn general_category(self) -> GeneralCategory;
470 fn space_fallback(self) -> space_fallback::SpaceCode;
471 fn combining_class(self) -> u8;
472 fn modified_combining_class(self) -> u8;
473 fn mirroring(self) -> Option<Codepoint>;
474 fn is_emoji_extended_pictographic(self) -> bool;
475 fn is_default_ignorable(self) -> bool;
476 fn is_variation_selector(self) -> bool;
477 fn vertical(self) -> Option<Codepoint>;
478}
479
480impl CharExt for Codepoint {
481 fn script(self) -> Script {
482 script_for(self)
483 }
484
485 fn general_category(self) -> GeneralCategory {
486 general_category_for(self)
487 }
488
489 fn combining_class(self) -> u8 {
490 combining_class_for(self)
491 }
492
493 fn mirroring(self) -> Option<Codepoint> {
494 mirroring_for(self)
495 }
496
497 fn space_fallback(self) -> space_fallback::SpaceCode {
498 use space_fallback::*;
499
500 match self {
502 0x0020 => SPACE, 0x00A0 => SPACE, 0x2000 => SPACE_EM_2, 0x2001 => SPACE_EM, 0x2002 => SPACE_EM_2, 0x2003 => SPACE_EM, 0x2004 => SPACE_EM_3, 0x2005 => SPACE_EM_4, 0x2006 => SPACE_EM_6, 0x2007 => SPACE_FIGURE, 0x2008 => SPACE_PUNCTUATION, 0x2009 => SPACE_EM_5, 0x200A => SPACE_EM_16, 0x202F => SPACE_NARROW, 0x205F => SPACE_4_EM_18, 0x3000 => SPACE_EM, _ => NOT_SPACE, }
520 }
521
522 fn modified_combining_class(self) -> u8 {
523 let u = self;
524
525 if u == 0x1A60 {
527 return 254;
528 }
529
530 if u == 0x0FC6 {
532 return 254;
533 }
534
535 if u == 0x0F39 {
537 return 127;
538 }
539
540 let k = u.combining_class();
541
542 MODIFIED_COMBINING_CLASS[k as usize]
543 }
544
545 fn is_emoji_extended_pictographic(self) -> bool {
546 emoji_table::is_Extended_Pictographic(self)
547 }
548
549 fn is_default_ignorable(self) -> bool {
587 let ch = self;
588 let plane = ch >> 16;
589 if plane == 0 {
590 let page = ch >> 8;
592 match page {
593 0x00 => ch == 0x00AD,
594 0x03 => ch == 0x034F,
595 0x06 => ch == 0x061C,
596 0x17 => (0x17B4..=0x17B5).contains(&ch),
597 0x18 => (0x180B..=0x180F).contains(&ch),
598 0x20 => {
599 (0x200B..=0x200F).contains(&ch)
600 || (0x202A..=0x202E).contains(&ch)
601 || (0x2060..=0x206F).contains(&ch)
602 }
603 0xFE => (0xFE00..=0xFE0F).contains(&ch) || ch == 0xFEFF,
604 0xFF => (0xFFF0..=0xFFF8).contains(&ch),
605 _ => false,
606 }
607 } else {
608 match plane {
610 0x01 => (0x1D173..=0x1D17A).contains(&ch),
611 0x0E => (0xE0000..=0xE0FFF).contains(&ch),
612 _ => false,
613 }
614 }
615 }
616
617 fn is_variation_selector(self) -> bool {
618 (0x0FE00..=0x0FE0F).contains(&self) || (0xE0100..=0xE01EF).contains(&self) }
623
624 fn vertical(self) -> Option<Codepoint> {
625 Some(match self >> 8 {
626 0x20 => match self {
627 0x2013 => 0xfe32, 0x2014 => 0xfe31, 0x2025 => 0xfe30, 0x2026 => 0xfe19, _ => return None,
632 },
633 0x30 => match self {
634 0x3001 => 0xfe11, 0x3002 => 0xfe12, 0x3008 => 0xfe3f, 0x3009 => 0xfe40, 0x300a => 0xfe3d, 0x300b => 0xfe3e, 0x300c => 0xfe41, 0x300d => 0xfe42, 0x300e => 0xfe43, 0x300f => 0xfe44, 0x3010 => 0xfe3b, 0x3011 => 0xfe3c, 0x3014 => 0xfe39, 0x3015 => 0xfe3a, 0x3016 => 0xfe17, 0x3017 => 0xfe18, _ => return None,
651 },
652 0xfe => match self {
653 0xfe4f => 0xfe34, _ => return None,
655 },
656 0xff => match self {
657 0xff01 => 0xfe15, 0xff08 => 0xfe35, 0xff09 => 0xfe36, 0xff0c => 0xfe10, 0xff1a => 0xfe13, 0xff1b => 0xfe14, 0xff1f => 0xfe16, 0xff3b => 0xfe47, 0xff3d => 0xfe48, 0xff3f => 0xfe33, 0xff5b => 0xfe37, 0xff5d => 0xfe38, _ => return None,
670 },
671 _ => return None,
672 })
673 }
674}
675
676#[cfg(feature = "icu")]
677pub(crate) use icu::*;
678
679#[cfg(not(feature = "icu"))]
680pub(crate) use builtin::*;
681
682#[cfg(feature = "icu")]
683mod icu {
684 use super::{Codepoint, GeneralCategory, Script};
685 use icu_normalizer::properties::{
686 CanonicalCompositionBorrowed, CanonicalDecompositionBorrowed, Decomposed,
687 };
688 use icu_properties::{
689 props::{
690 BidiMirroringGlyph, CanonicalCombiningClass as IcuCanonicalCombiningClass,
691 GeneralCategory as IcuGeneralCategory, Script as IcuScript,
692 },
693 CodePointMapData, PropertyNamesShort,
694 };
695
696 pub(crate) fn script_for(c: u32) -> Script {
697 let Some(icu_script) = PropertyNamesShort::new()
698 .get_locale_script(CodePointMapData::<IcuScript>::new().get32(c))
699 else {
700 return Script::UNKNOWN;
701 };
702 Script::from_bytes(&icu_script.into_raw())
706 }
707
708 pub(crate) fn general_category_for(c: u32) -> GeneralCategory {
709 GeneralCategory::from_icu(CodePointMapData::<IcuGeneralCategory>::new().get32(c))
710 }
711
712 pub(crate) fn combining_class_for(c: u32) -> u8 {
713 CodePointMapData::<IcuCanonicalCombiningClass>::new()
714 .get32(c)
715 .to_icu4c_value()
716 }
717
718 pub(crate) fn mirroring_for(c: u32) -> Option<Codepoint> {
719 CodePointMapData::<BidiMirroringGlyph>::new()
720 .get32(c)
721 .mirroring_glyph
722 .map(Codepoint::from)
723 }
724
725 pub(crate) fn compose_impl(a: Codepoint, b: Codepoint) -> Option<Codepoint> {
726 let a = char::from_u32(a)?;
727 let b = char::from_u32(b)?;
728 CanonicalCompositionBorrowed::new()
729 .compose(a, b)
730 .map(Codepoint::from)
731 }
732
733 pub(crate) fn decompose_impl(ab: Codepoint) -> Option<(Codepoint, Codepoint)> {
734 let ch = char::from_u32(ab)?;
735 match CanonicalDecompositionBorrowed::new().decompose(ch) {
736 Decomposed::Default => None,
737 Decomposed::Singleton(a) => Some((Codepoint::from(a), 0)),
738 Decomposed::Expansion(a, b) => Some((Codepoint::from(a), Codepoint::from(b))),
739 }
740 }
741
742 impl GeneralCategory {
743 fn from_icu(category: IcuGeneralCategory) -> Self {
744 use IcuGeneralCategory::*;
745 match category {
746 Control => Self::CONTROL,
747 Format => Self::FORMAT,
748 Unassigned => Self::UNASSIGNED,
749 PrivateUse => Self::PRIVATE_USE,
750 Surrogate => Self::SURROGATE,
751 LowercaseLetter => Self::LOWERCASE_LETTER,
752 ModifierLetter => Self::MODIFIER_LETTER,
753 OtherLetter => Self::OTHER_LETTER,
754 TitlecaseLetter => Self::TITLECASE_LETTER,
755 UppercaseLetter => Self::UPPERCASE_LETTER,
756 SpacingMark => Self::SPACING_MARK,
757 EnclosingMark => Self::ENCLOSING_MARK,
758 NonspacingMark => Self::NON_SPACING_MARK,
759 DecimalNumber => Self::DECIMAL_NUMBER,
760 LetterNumber => Self::LETTER_NUMBER,
761 OtherNumber => Self::OTHER_NUMBER,
762 ConnectorPunctuation => Self::CONNECT_PUNCTUATION,
763 DashPunctuation => Self::DASH_PUNCTUATION,
764 ClosePunctuation => Self::CLOSE_PUNCTUATION,
765 FinalPunctuation => Self::FINAL_PUNCTUATION,
766 InitialPunctuation => Self::INITIAL_PUNCTUATION,
767 OtherPunctuation => Self::OTHER_PUNCTUATION,
768 OpenPunctuation => Self::OPEN_PUNCTUATION,
769 CurrencySymbol => Self::CURRENCY_SYMBOL,
770 ModifierSymbol => Self::MODIFIER_SYMBOL,
771 MathSymbol => Self::MATH_SYMBOL,
772 OtherSymbol => Self::OTHER_SYMBOL,
773 LineSeparator => Self::LINE_SEPARATOR,
774 ParagraphSeparator => Self::PARAGRAPH_SEPARATOR,
775 SpaceSeparator => Self::SPACE_SEPARATOR,
776 }
777 }
778 }
779}
780
781#[cfg(not(feature = "icu"))]
782#[expect(clippy::manual_range_contains)]
783mod builtin {
784 use super::{super::algs::*, ucd_table::ucd::*, Codepoint, GeneralCategory, Script};
785
786 pub(crate) fn script_for(c: u32) -> Script {
787 ucd_sc_map[ucd_sc(c as usize) as usize]
788 }
789
790 pub(crate) fn general_category_for(c: u32) -> GeneralCategory {
791 GeneralCategory(ucd_gc(c as usize))
792 }
793
794 pub(crate) fn combining_class_for(c: u32) -> u8 {
795 ucd_ccc(c as usize)
796 }
797
798 pub(crate) fn mirroring_for(c: u32) -> Option<Codepoint> {
799 let delta = ucd_bmg(c as usize);
800 if delta == 0 {
801 None
802 } else {
803 Some(((c as i32).wrapping_add(delta)) as u32)
804 }
805 }
806
807 pub(crate) fn compose_impl(a: Codepoint, b: Codepoint) -> Option<Codepoint> {
808 if let Some(ab) = compose_hangul(a, b) {
810 return Some(ab);
811 }
812
813 let u = if (a & 0xFFFF_F800) == 0x0000 && (b & 0xFFFF_FF80) == 0x0300 {
814 let k = HB_CODEPOINT_ENCODE3_11_7_14(a, b, 0);
818 let v = ucd_dm2_u32_map
819 .binary_search_by(|probe| {
820 let key = probe & HB_CODEPOINT_ENCODE3_11_7_14(0x001F_FFFF, 0x001F_FFFF, 0);
821 key.cmp(&k)
822 })
823 .ok()
824 .map(|index| ucd_dm2_u32_map[index]);
825
826 HB_CODEPOINT_DECODE3_11_7_14_3(v?)
827 } else {
828 let k = HB_CODEPOINT_ENCODE3(a, b, 0);
831 let v = ucd_dm2_u64_map
832 .binary_search_by(|probe| {
833 let key = probe & HB_CODEPOINT_ENCODE3(0x001F_FFFF, 0x001F_FFFF, 0);
834 key.cmp(&k)
835 })
836 .ok()
837 .map(|index| ucd_dm2_u64_map[index]);
838
839 HB_CODEPOINT_DECODE3_3(v?)
840 };
841
842 if u == 0 {
843 None
844 } else {
845 Some(u)
846 }
847 }
848
849 pub(crate) fn decompose_impl(ab: Codepoint) -> Option<(Codepoint, Codepoint)> {
850 if let Some((a, b)) = decompose_hangul(ab) {
851 return Some((a, b));
852 }
853
854 let mut i = ucd_dm(ab as usize) as usize;
855
856 if i == 0 {
858 return None;
859 }
860 i -= 1;
861
862 if i < ucd_dm1_p0_map.len() + ucd_dm1_p2_map.len() {
863 let a = if i < ucd_dm1_p0_map.len() {
864 ucd_dm1_p0_map[i] as u32
865 } else {
866 let j = i - ucd_dm1_p0_map.len();
867 0x20000 | ucd_dm1_p2_map[j] as u32
868 };
869 return Some((a, 0));
870 }
871
872 i -= ucd_dm1_p0_map.len() + ucd_dm1_p2_map.len();
873
874 if i < ucd_dm2_u32_map.len() {
875 let v = ucd_dm2_u32_map[i];
876 let a = HB_CODEPOINT_DECODE3_11_7_14_1(v);
877 let b = HB_CODEPOINT_DECODE3_11_7_14_2(v);
878 return Some((a, b));
879 }
880
881 i -= ucd_dm2_u32_map.len();
882
883 let v = ucd_dm2_u64_map[i];
884 let a = HB_CODEPOINT_DECODE3_1(v);
885 let b = HB_CODEPOINT_DECODE3_2(v);
886 Some((a, b))
887 }
888
889 const S_BASE: u32 = 0xAC00;
890 const L_BASE: u32 = 0x1100;
891 const V_BASE: u32 = 0x1161;
892 const T_BASE: u32 = 0x11A7;
893 const L_COUNT: u32 = 19;
894 const V_COUNT: u32 = 21;
895 const T_COUNT: u32 = 28;
896 const N_COUNT: u32 = V_COUNT * T_COUNT;
897 const S_COUNT: u32 = L_COUNT * N_COUNT;
898
899 fn compose_hangul(a: Codepoint, b: Codepoint) -> Option<Codepoint> {
900 let l = a;
901 let v = b;
902 if L_BASE <= l && l < (L_BASE + L_COUNT) && V_BASE <= v && v < (V_BASE + V_COUNT) {
903 let r = S_BASE + (l - L_BASE) * N_COUNT + (v - V_BASE) * T_COUNT;
904 Some(r)
905 } else if S_BASE <= l
906 && l <= (S_BASE + S_COUNT - T_COUNT)
907 && T_BASE < v
908 && v < (T_BASE + T_COUNT)
909 && (l - S_BASE) % T_COUNT == 0
910 {
911 let r = l + (v - T_BASE);
912 Some(r)
913 } else {
914 None
915 }
916 }
917
918 fn decompose_hangul(ab: Codepoint) -> Option<(Codepoint, Codepoint)> {
919 let si = ab.wrapping_sub(S_BASE);
920 if si >= S_COUNT {
921 return None;
922 }
923 let (a, b) = if si % T_COUNT != 0 {
924 (S_BASE + (si / T_COUNT) * T_COUNT, T_BASE + (si % T_COUNT))
926 } else {
927 (L_BASE + (si / N_COUNT), V_BASE + (si % N_COUNT) / T_COUNT)
929 };
930 Some((a, b))
931 }
932}
933
934#[cfg(test)]
935mod tests {
936 #[test]
937 fn telugu_length_marks_reorder_before_virama() {
938 use super::CharExt;
939 assert_eq!(0x0C55u32.modified_combining_class(), 4);
940 assert_eq!(0x0C56u32.modified_combining_class(), 5);
941 assert_eq!(0x0C4Du32.modified_combining_class(), 9);
942 assert_eq!(super::combining_class(0x0C55), 84);
943 assert_eq!(super::combining_class(0x0C56), 91);
944 }
945
946 #[cfg(not(feature = "icu"))]
947 #[test]
948 fn unicode_18_scripts() {
949 use super::GeneralCategory as Gc;
950 use crate::Script;
951
952 for (range, script, category) in [
953 (
954 0x125A8..=0x1264B,
955 Script::PROTO_CUNEIFORM,
956 Gc::LETTER_NUMBER,
957 ),
958 (0x18E00..=0x19191, Script::JURCHEN, Gc::OTHER_LETTER),
959 (0x191A0..=0x191D2, Script::JURCHEN, Gc::OTHER_LETTER),
960 (0x3D000..=0x3FC3F, Script::SEAL, Gc::OTHER_LETTER),
961 ] {
962 for cp in range {
963 assert_eq!(super::script_for(cp), script, "U+{cp:04X}");
964 assert_eq!(super::general_category_for(cp), category, "U+{cp:04X}");
965 assert_eq!(super::combining_class_for(cp), 0, "U+{cp:04X}");
966 }
967 }
968 }
969
970 #[test]
971 fn general_category_ranges() {
972 use super::GeneralCategory as Gc;
973 assert!(!Gc::CONTROL.in_range_inclusive(Gc::FORMAT, Gc::NON_SPACING_MARK));
974 assert!(Gc::LOWERCASE_LETTER.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER));
975 assert!(Gc::UPPERCASE_LETTER.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER));
976 assert!(!Gc::SPACING_MARK.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER));
977 }
978
979 #[cfg(feature = "icu")]
980 #[test]
981 fn script_mapping() {
982 use icu_properties::props::Script as IcuScript;
983 use icu_properties::{CodePointMapData, PropertyNamesShort};
984
985 let mut mismatches = Vec::new();
986
987 for cp in 0..=char::MAX as u32 {
989 let script = super::icu::script_for(cp);
990 let short_name = PropertyNamesShort::<IcuScript>::new()
991 .get_locale_script(CodePointMapData::new().get32(cp))
992 .unwrap();
993 let tag = crate::Tag::from_be_bytes(short_name.into_raw());
994 let expected = crate::Script::from_iso15924_tag(tag).unwrap();
995 if expected != script {
996 let expected_tag = expected.tag().to_be_bytes();
997 let actual_tag = script.tag().to_be_bytes();
998 mismatches.push(format!(
999 "Script::from_iso15924_tag({short_name}) -> {}, mapped tag {}",
1000 core::str::from_utf8(&expected_tag).unwrap_or("????"),
1001 core::str::from_utf8(&actual_tag).unwrap_or("????")
1002 ));
1003 }
1004 }
1005
1006 assert!(
1007 mismatches.is_empty(),
1008 "script mapping mismatches:\n{}",
1009 mismatches.join("\n")
1010 );
1011 }
1012}