Skip to main content

harfrust/unicode/
mod.rs

1//! Unicode properties and canonical composition used by the shaper.
2
3#![allow(
4    non_camel_case_types,
5    non_snake_case,
6    non_upper_case_globals,
7    clippy::inline_always,
8    clippy::wrong_self_convention,
9    clippy::enum_glob_use
10)]
11
12#[expect(clippy::identity_op)]
13#[rustfmt::skip]
14mod emoji_table;
15#[cfg(not(feature = "icu"))]
16#[rustfmt::skip]
17mod ucd_table;
18
19use crate::Script;
20
21pub(crate) type Codepoint = u32;
22
23/// One step of canonical decomposition.
24#[derive(Clone, Copy, Debug, Eq, PartialEq)]
25pub enum Decomposed {
26    /// A single resulting code point.
27    Singleton(u32),
28    /// Two resulting code points.
29    Pair(u32, u32),
30}
31
32/// Compose two code points, if they have a canonical composition.
33#[inline]
34pub fn compose(a: u32, b: u32) -> Option<u32> {
35    compose_impl(a, b)
36}
37
38/// Return one step of canonical decomposition, if one exists.
39#[inline]
40pub fn decompose(codepoint: u32) -> Option<Decomposed> {
41    decompose_impl(codepoint).map(|(a, b)| {
42        if b == 0 {
43            Decomposed::Singleton(a)
44        } else {
45            Decomposed::Pair(a, b)
46        }
47    })
48}
49
50/// The canonical combining class of a code point.
51#[inline]
52pub fn combining_class(codepoint: u32) -> u8 {
53    combining_class_for(codepoint)
54}
55
56/// The Unicode script of a code point, using the same data as shaping.
57///
58/// Values outside the Unicode range have the unknown script.
59#[inline]
60pub fn script(codepoint: u32) -> Script {
61    if codepoint > 0x10_FFFF {
62        Script::UNKNOWN
63    } else {
64        script_for(codepoint)
65    }
66}
67
68// Space estimates based on:
69// https://unicode.org/charts/PDF/U2000.pdf
70// https://docs.microsoft.com/en-us/typography/develop/character-design-standards/whitespace
71pub(crate) mod space_fallback {
72    pub type SpaceCode = u8;
73    pub const NOT_SPACE: u8 = 0;
74    pub const SPACE_EM: u8 = 1;
75    pub const SPACE_EM_2: u8 = 2;
76    pub const SPACE_EM_3: u8 = 3;
77    pub const SPACE_EM_4: u8 = 4;
78    pub const SPACE_EM_5: u8 = 5;
79    pub const SPACE_EM_6: u8 = 6;
80    pub const SPACE_EM_16: u8 = 16;
81    pub const SPACE_4_EM_18: u8 = 17; // 4/18th of an EM!
82    pub const SPACE: u8 = 18;
83    pub const SPACE_FIGURE: u8 = 19;
84    pub const SPACE_PUNCTUATION: u8 = 20;
85    pub const SPACE_NARROW: u8 = 21;
86}
87
88/// Data type for the "General_Category" (gc) property from the Unicode
89/// Character Database.
90#[derive(Copy, Clone, PartialEq, Eq, Debug)]
91pub(crate) struct GeneralCategory(pub u8);
92
93#[allow(unused)]
94impl GeneralCategory {
95    /// Control (`Cc`).
96    pub const CONTROL: Self = Self(0);
97    /// Format (`Cf`).
98    pub const FORMAT: Self = Self(1);
99    /// Unassigned (`Cn`).
100    pub const UNASSIGNED: Self = Self(2);
101    /// Private Use (`Co`).
102    pub const PRIVATE_USE: Self = Self(3);
103    /// Surrogate (`Cs`).
104    pub const SURROGATE: Self = Self(4);
105    /// Lowercase Letter (`Ll`).
106    pub const LOWERCASE_LETTER: Self = Self(5);
107    /// Modifier Letter (`Lm`).
108    pub const MODIFIER_LETTER: Self = Self(6);
109    /// Other Letter (`Lo`).
110    pub const OTHER_LETTER: Self = Self(7);
111    /// Titlecase Letter (`Lt`).
112    pub const TITLECASE_LETTER: Self = Self(8);
113    /// Uppercase Letter (`Lu`).
114    pub const UPPERCASE_LETTER: Self = Self(9);
115    /// Spacing Mark (`Mc`).
116    pub const SPACING_MARK: Self = Self(10);
117    /// Enclosing Mark (`Me`).
118    pub const ENCLOSING_MARK: Self = Self(11);
119    /// Nonspacing Mark (`Mn`).
120    pub const NON_SPACING_MARK: Self = Self(12);
121    /// Decimal Number (`Nd`).
122    pub const DECIMAL_NUMBER: Self = Self(13);
123    /// Letter Number (`Nl`).
124    pub const LETTER_NUMBER: Self = Self(14);
125    /// Other Number (`No`).
126    pub const OTHER_NUMBER: Self = Self(15);
127    /// Connector Punctuation (`Pc`).
128    pub const CONNECT_PUNCTUATION: Self = Self(16);
129    /// Dash Punctuation (`Pd`).
130    pub const DASH_PUNCTUATION: Self = Self(17);
131    /// Close Punctuation (`Pe`).
132    pub const CLOSE_PUNCTUATION: Self = Self(18);
133    /// Final Punctuation (`Pf`).
134    pub const FINAL_PUNCTUATION: Self = Self(19);
135    /// Initial Punctuation (`Pi`).
136    pub const INITIAL_PUNCTUATION: Self = Self(20);
137    /// Other Punctuation (`Po`).
138    pub const OTHER_PUNCTUATION: Self = Self(21);
139    /// Open Punctuation (`Ps`).
140    pub const OPEN_PUNCTUATION: Self = Self(22);
141    /// Currency Symbol (`Sc`).
142    pub const CURRENCY_SYMBOL: Self = Self(23);
143    /// Modifier Symbol (`Sk`).
144    pub const MODIFIER_SYMBOL: Self = Self(24);
145    /// Math Symbol (`Sm`).
146    pub const MATH_SYMBOL: Self = Self(25);
147    /// Other Symbol (`So`).
148    pub const OTHER_SYMBOL: Self = Self(26);
149    /// Line Separator (`Zl`).
150    pub const LINE_SEPARATOR: Self = Self(27);
151    /// Paragraph Separator (`Zp`).
152    pub const PARAGRAPH_SEPARATOR: Self = Self(28);
153    /// Space Separator (`Zs`).
154    pub const SPACE_SEPARATOR: Self = Self(29);
155}
156
157impl GeneralCategory {
158    pub fn is_mark(&self) -> bool {
159        matches!(
160            *self,
161            Self::SPACING_MARK | Self::ENCLOSING_MARK | Self::NON_SPACING_MARK
162        )
163    }
164
165    pub fn is_letter(&self) -> bool {
166        matches!(
167            *self,
168            Self::LOWERCASE_LETTER
169                | Self::MODIFIER_LETTER
170                | Self::OTHER_LETTER
171                | Self::TITLECASE_LETTER
172                | Self::UPPERCASE_LETTER
173        )
174    }
175
176    #[inline(always)]
177    pub(crate) const fn flag(self) -> u32 {
178        1 << self.0
179    }
180
181    #[inline(always)]
182    pub(crate) const fn flag_unsafe(self) -> u32 {
183        if self.0 < 32 {
184            1 << self.0
185        } else {
186            0
187        }
188    }
189
190    #[inline(always)]
191    pub(crate) const fn in_range_inclusive(self, start: Self, end: Self) -> bool {
192        debug_assert!(start.0 < end.0);
193        let range_mask = (1 << (end.0 + 1)) - (1 << start.0);
194        self.flag_unsafe() & range_mask != 0
195    }
196}
197
198#[allow(dead_code)]
199pub(crate) mod combining_class {
200    pub const NotReordered: u8 = 0;
201    pub const Overlay: u8 = 1;
202    pub const Nukta: u8 = 7;
203    pub const KanaVoicing: u8 = 8;
204    pub const Virama: u8 = 9;
205
206    /* Hebrew */
207    pub const CCC10: u8 = 10;
208    pub const CCC11: u8 = 11;
209    pub const CCC12: u8 = 12;
210    pub const CCC13: u8 = 13;
211    pub const CCC14: u8 = 14;
212    pub const CCC15: u8 = 15;
213    pub const CCC16: u8 = 16;
214    pub const CCC17: u8 = 17;
215    pub const CCC18: u8 = 18;
216    pub const CCC19: u8 = 19;
217    pub const CCC20: u8 = 20;
218    pub const CCC21: u8 = 21;
219    pub const CCC22: u8 = 22;
220    pub const CCC23: u8 = 23;
221    pub const CCC24: u8 = 24;
222    pub const CCC25: u8 = 25;
223    pub const CCC26: u8 = 26;
224
225    /* Arabic */
226    pub const CCC27: u8 = 27;
227    pub const CCC28: u8 = 28;
228    pub const CCC29: u8 = 29;
229    pub const CCC30: u8 = 30;
230    pub const CCC31: u8 = 31;
231    pub const CCC32: u8 = 32;
232    pub const CCC33: u8 = 33;
233    pub const CCC34: u8 = 34;
234    pub const CCC35: u8 = 35;
235
236    /* Syriac */
237    pub const CCC36: u8 = 36;
238
239    /* Telugu */
240    pub const CCC84: u8 = 84;
241    pub const CCC91: u8 = 91;
242
243    /* Thai */
244    pub const CCC103: u8 = 103;
245    pub const CCC107: u8 = 107;
246
247    /* Lao */
248    pub const CCC118: u8 = 118;
249    pub const CCC122: u8 = 122;
250
251    /* Tibetan */
252    pub const CCC129: u8 = 129;
253    pub const CCC130: u8 = 130;
254    pub const CCC132: u8 = 132;
255
256    pub const AttachedBelowLeft: u8 = 200;
257    pub const AttachedBelow: u8 = 202;
258    pub const AttachedAbove: u8 = 214;
259    pub const AttachedAboveRight: u8 = 216;
260    pub const BelowLeft: u8 = 218;
261    pub const Below: u8 = 220;
262    pub const BelowRight: u8 = 222;
263    pub const Left: u8 = 224;
264    pub const Right: u8 = 226;
265    pub const AboveLeft: u8 = 228;
266    pub const Above: u8 = 230;
267    pub const AboveRight: u8 = 232;
268    pub const DoubleBelow: u8 = 233;
269    pub const DoubleAbove: u8 = 234;
270
271    pub const IotaSubscript: u8 = 240;
272
273    pub const Invalid: u8 = 255;
274}
275
276#[allow(dead_code)]
277pub(crate) mod modified_combining_class {
278    // Hebrew
279    //
280    // We permute the "fixed-position" classes 10-26 into the order
281    // described in the SBL Hebrew manual:
282    //
283    // https://www.sbl-site.org/Fonts/SBLHebrewUserManual1.5x.pdf
284    //
285    // (as recommended by:
286    //  https://forum.fontlab.com/archive-old-microsoft-volt-group/vista-and-diacritic-ordering/msg22823/)
287    //
288    // More details here:
289    // https://bugzilla.mozilla.org/show_bug.cgi?id=662055
290    pub const CCC10: u8 = 22; // sheva
291    pub const CCC11: u8 = 15; // hataf segol
292    pub const CCC12: u8 = 16; // hataf patah
293    pub const CCC13: u8 = 17; // hataf qamats
294    pub const CCC14: u8 = 23; // hiriq
295    pub const CCC15: u8 = 18; // tsere
296    pub const CCC16: u8 = 19; // segol
297    pub const CCC17: u8 = 20; // patah
298    pub const CCC18: u8 = 21; // qamats & qamats qatan
299    pub const CCC19: u8 = 14; // holam & holam haser for vav
300    pub const CCC20: u8 = 24; // qubuts
301    pub const CCC21: u8 = 12; // dagesh
302    pub const CCC22: u8 = 25; // meteg
303    pub const CCC23: u8 = 13; // rafe
304    pub const CCC24: u8 = 10; // shin dot
305    pub const CCC25: u8 = 11; // sin dot
306    pub const CCC26: u8 = 26; // point varika
307
308    // Arabic
309    //
310    // Modify to move Shadda (ccc=33) before other marks.  See:
311    // https://unicode.org/faq/normalization.html#8
312    // https://unicode.org/faq/normalization.html#9
313    pub const CCC27: u8 = 28; // fathatan
314    pub const CCC28: u8 = 29; // dammatan
315    pub const CCC29: u8 = 30; // kasratan
316    pub const CCC30: u8 = 31; // fatha
317    pub const CCC31: u8 = 32; // damma
318    pub const CCC32: u8 = 33; // kasra
319    pub const CCC33: u8 = 27; // shadda
320    pub const CCC34: u8 = 34; // sukun
321    pub const CCC35: u8 = 35; // superscript alef
322
323    // Syriac
324    pub const CCC36: u8 = 36; // superscript alaph
325
326    // Telugu
327    //
328    // Modify Telugu length marks (ccc=84, ccc=91).
329    // These are the only matras in the main Indic scripts range that have
330    // a non-zero ccc.  That makes them reorder with the Halant that is
331    // ccc=9. Assign 4 and 5, which are otherwise unassigned.
332    pub const CCC84: u8 = 4; // length mark
333    pub const CCC91: u8 = 5; // ai length mark
334
335    // Thai
336    //
337    // Modify U+0E38 and U+0E39 (ccc=103) to be reordered before U+0E3A (ccc=9).
338    // Assign 3, which is unassigned otherwise.
339    // Uniscribe does this reordering too.
340    pub const CCC103: u8 = 3; // sara u / sara uu
341    pub const CCC107: u8 = 107; // mai *
342
343    // Lao
344    pub const CCC118: u8 = 118; // sign u / sign uu
345    pub const CCC122: u8 = 122; // mai *
346
347    // Tibetan
348    //
349    // In case of multiple vowel-signs, use u first (but after achung)
350    // this allows Dzongkha multi-vowel shortcuts to render correctly
351    pub const CCC129: u8 = 129; // sign aa
352    pub const CCC130: u8 = 132; // sign i
353    pub const CCC132: u8 = 131; // sign u
354}
355
356#[rustfmt::skip]
357static MODIFIED_COMBINING_CLASS: &[u8; 256] = &[
358    combining_class::NotReordered,
359    combining_class::Overlay,
360    2, 3, 4, 5, 6,
361    combining_class::Nukta,
362    combining_class::KanaVoicing,
363    combining_class::Virama,
364
365    // Hebrew
366    modified_combining_class::CCC10,
367    modified_combining_class::CCC11,
368    modified_combining_class::CCC12,
369    modified_combining_class::CCC13,
370    modified_combining_class::CCC14,
371    modified_combining_class::CCC15,
372    modified_combining_class::CCC16,
373    modified_combining_class::CCC17,
374    modified_combining_class::CCC18,
375    modified_combining_class::CCC19,
376    modified_combining_class::CCC20,
377    modified_combining_class::CCC21,
378    modified_combining_class::CCC22,
379    modified_combining_class::CCC23,
380    modified_combining_class::CCC24,
381    modified_combining_class::CCC25,
382    modified_combining_class::CCC26,
383
384    // Arabic
385    modified_combining_class::CCC27,
386    modified_combining_class::CCC28,
387    modified_combining_class::CCC29,
388    modified_combining_class::CCC30,
389    modified_combining_class::CCC31,
390    modified_combining_class::CCC32,
391    modified_combining_class::CCC33,
392    modified_combining_class::CCC34,
393    modified_combining_class::CCC35,
394
395    // Syriac
396    modified_combining_class::CCC36,
397
398    37, 38, 39,
399    40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
400    60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
401    80, 81, 82, 83,
402
403    // Telugu
404    modified_combining_class::CCC84,
405    85, 86, 87, 88, 89, 90,
406    modified_combining_class::CCC91,
407    92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102,
408
409    // Thai
410    modified_combining_class::CCC103,
411    104, 105, 106,
412    modified_combining_class::CCC107,
413    108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
414
415    // Lao
416    modified_combining_class::CCC118,
417    119, 120, 121,
418    modified_combining_class::CCC122,
419    123, 124, 125, 126, 127, 128,
420
421    // Tibetan
422    modified_combining_class::CCC129,
423    modified_combining_class::CCC130,
424    131,
425    modified_combining_class::CCC132,
426    133, 134, 135, 136, 137, 138, 139,
427
428
429    140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
430    150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
431    160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
432    170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
433    180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
434    190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
435
436    combining_class::AttachedBelowLeft,
437    201,
438    combining_class::AttachedBelow,
439    203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213,
440    combining_class::AttachedAbove,
441    215,
442    combining_class::AttachedAboveRight,
443    217,
444    combining_class::BelowLeft,
445    219,
446    combining_class::Below,
447    221,
448    combining_class::BelowRight,
449    223,
450    combining_class::Left,
451    225,
452    combining_class::Right,
453    227,
454    combining_class::AboveLeft,
455    229,
456    combining_class::Above,
457    231,
458    combining_class::AboveRight,
459    combining_class::DoubleBelow,
460    combining_class::DoubleAbove,
461    235, 236, 237, 238, 239,
462    combining_class::IotaSubscript,
463    241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254,
464    combining_class::Invalid,
465];
466
467pub(crate) trait CharExt {
468    fn script(self) -> Script;
469    fn general_category(self) -> GeneralCategory;
470    fn space_fallback(self) -> space_fallback::SpaceCode;
471    fn combining_class(self) -> u8;
472    fn modified_combining_class(self) -> u8;
473    fn mirroring(self) -> Option<Codepoint>;
474    fn is_emoji_extended_pictographic(self) -> bool;
475    fn is_default_ignorable(self) -> bool;
476    fn is_variation_selector(self) -> bool;
477    fn vertical(self) -> Option<Codepoint>;
478}
479
480impl CharExt for Codepoint {
481    fn script(self) -> Script {
482        script_for(self)
483    }
484
485    fn general_category(self) -> GeneralCategory {
486        general_category_for(self)
487    }
488
489    fn combining_class(self) -> u8 {
490        combining_class_for(self)
491    }
492
493    fn mirroring(self) -> Option<Codepoint> {
494        mirroring_for(self)
495    }
496
497    fn space_fallback(self) -> space_fallback::SpaceCode {
498        use space_fallback::*;
499
500        // All GC=Zs chars that can use a fallback.
501        match self {
502            0x0020 => SPACE,             // SPACE
503            0x00A0 => SPACE,             // NO-BREAK SPACE
504            0x2000 => SPACE_EM_2,        // EN QUAD
505            0x2001 => SPACE_EM,          // EM QUAD
506            0x2002 => SPACE_EM_2,        // EN SPACE
507            0x2003 => SPACE_EM,          // EM SPACE
508            0x2004 => SPACE_EM_3,        // THREE-PER-EM SPACE
509            0x2005 => SPACE_EM_4,        // FOUR-PER-EM SPACE
510            0x2006 => SPACE_EM_6,        // SIX-PER-EM SPACE
511            0x2007 => SPACE_FIGURE,      // FIGURE SPACE
512            0x2008 => SPACE_PUNCTUATION, // PUNCTUATION SPACE
513            0x2009 => SPACE_EM_5,        // THIN SPACE
514            0x200A => SPACE_EM_16,       // HAIR SPACE
515            0x202F => SPACE_NARROW,      // NARROW NO-BREAK SPACE
516            0x205F => SPACE_4_EM_18,     // MEDIUM MATHEMATICAL SPACE
517            0x3000 => SPACE_EM,          // IDEOGRAPHIC SPACE
518            _ => NOT_SPACE,              // OGHAM SPACE MARK
519        }
520    }
521
522    fn modified_combining_class(self) -> u8 {
523        let u = self;
524
525        // Reorder SAKOT to ensure it comes after any tone marks.
526        if u == 0x1A60 {
527            return 254;
528        }
529
530        // Reorder PADMA to ensure it comes after any vowel marks.
531        if u == 0x0FC6 {
532            return 254;
533        }
534
535        // Reorder TSA -PHRU to reorder before U+0F74
536        if u == 0x0F39 {
537            return 127;
538        }
539
540        let k = u.combining_class();
541
542        MODIFIED_COMBINING_CLASS[k as usize]
543    }
544
545    fn is_emoji_extended_pictographic(self) -> bool {
546        emoji_table::is_Extended_Pictographic(self)
547    }
548
549    /// Default_Ignorable codepoints:
550    ///
551    /// Note: While U+115F, U+1160, U+3164 and U+FFA0 are Default_Ignorable,
552    /// we do NOT want to hide them, as the way Uniscribe has implemented them
553    /// is with regular spacing glyphs, and that's the way fonts are made to work.
554    /// As such, we make exceptions for those four.
555    /// Also ignoring U+1BCA0..1BCA3. https://github.com/harfbuzz/harfbuzz/issues/503
556    ///
557    /// Unicode 14.0:
558    /// $ grep '; Default_Ignorable_Code_Point ' DerivedCoreProperties.txt | sed 's/;.*#/#/'
559    /// 00AD          # Cf       SOFT HYPHEN
560    /// 034F          # Mn       COMBINING GRAPHEME JOINER
561    /// 061C          # Cf       ARABIC LETTER MARK
562    /// 115F..1160    # Lo   [2] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG FILLER
563    /// 17B4..17B5    # Mn   [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA
564    /// 180B..180D    # Mn   [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE
565    /// 180E          # Cf       MONGOLIAN VOWEL SEPARATOR
566    /// 180F          # Mn       MONGOLIAN FREE VARIATION SELECTOR FOUR
567    /// 200B..200F    # Cf   [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK
568    /// 202A..202E    # Cf   [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE
569    /// 2060..2064    # Cf   [5] WORD JOINER..INVISIBLE PLUS
570    /// 2065          # Cn       <reserved-2065>
571    /// 2066..206F    # Cf  [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES
572    /// 3164          # Lo       HANGUL FILLER
573    /// FE00..FE0F    # Mn  [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16
574    /// FEFF          # Cf       ZERO WIDTH NO-BREAK SPACE
575    /// FFA0          # Lo       HALFWIDTH HANGUL FILLER
576    /// FFF0..FFF8    # Cn   [9] <reserved-FFF0>..<reserved-FFF8>
577    /// 1BCA0..1BCA3  # Cf   [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP
578    /// 1D173..1D17A  # Cf   [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE
579    /// E0000         # Cn       <reserved-E0000>
580    /// E0001         # Cf       LANGUAGE TAG
581    /// E0002..E001F  # Cn  [30] <reserved-E0002>..<reserved-E001F>
582    /// E0020..E007F  # Cf  [96] TAG SPACE..CANCEL TAG
583    /// E0080..E00FF  # Cn [128] <reserved-E0080>..<reserved-E00FF>
584    /// E0100..E01EF  # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
585    /// E01F0..E0FFF  # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
586    fn is_default_ignorable(self) -> bool {
587        let ch = self;
588        let plane = ch >> 16;
589        if plane == 0 {
590            // BMP
591            let page = ch >> 8;
592            match page {
593                0x00 => ch == 0x00AD,
594                0x03 => ch == 0x034F,
595                0x06 => ch == 0x061C,
596                0x17 => (0x17B4..=0x17B5).contains(&ch),
597                0x18 => (0x180B..=0x180F).contains(&ch),
598                0x20 => {
599                    (0x200B..=0x200F).contains(&ch)
600                        || (0x202A..=0x202E).contains(&ch)
601                        || (0x2060..=0x206F).contains(&ch)
602                }
603                0xFE => (0xFE00..=0xFE0F).contains(&ch) || ch == 0xFEFF,
604                0xFF => (0xFFF0..=0xFFF8).contains(&ch),
605                _ => false,
606            }
607        } else {
608            // Other planes
609            match plane {
610                0x01 => (0x1D173..=0x1D17A).contains(&ch),
611                0x0E => (0xE0000..=0xE0FFF).contains(&ch),
612                _ => false,
613            }
614        }
615    }
616
617    fn is_variation_selector(self) -> bool {
618        // U+180B..180D, U+180F MONGOLIAN FREE VARIATION SELECTORs are handled in the
619        //Arabic shaper. No need to match them here.
620        (0x0FE00..=0x0FE0F).contains(&self) || // VARIATION SELECTOR - 1..16
621        (0xE0100..=0xE01EF).contains(&self) // VARIATION SELECTOR - 17..256
622    }
623
624    fn vertical(self) -> Option<Codepoint> {
625        Some(match self >> 8 {
626            0x20 => match self {
627                0x2013 => 0xfe32, // EN DASH
628                0x2014 => 0xfe31, // EM DASH
629                0x2025 => 0xfe30, // TWO DOT LEADER
630                0x2026 => 0xfe19, // HORIZONTAL ELLIPSIS
631                _ => return None,
632            },
633            0x30 => match self {
634                0x3001 => 0xfe11, // IDEOGRAPHIC COMMA
635                0x3002 => 0xfe12, // IDEOGRAPHIC FULL STOP
636                0x3008 => 0xfe3f, // LEFT ANGLE BRACKET
637                0x3009 => 0xfe40, // RIGHT ANGLE BRACKET
638                0x300a => 0xfe3d, // LEFT DOUBLE ANGLE BRACKET
639                0x300b => 0xfe3e, // RIGHT DOUBLE ANGLE BRACKET
640                0x300c => 0xfe41, // LEFT CORNER BRACKET
641                0x300d => 0xfe42, // RIGHT CORNER BRACKET
642                0x300e => 0xfe43, // LEFT WHITE CORNER BRACKET
643                0x300f => 0xfe44, // RIGHT WHITE CORNER BRACKET
644                0x3010 => 0xfe3b, // LEFT BLACK LENTICULAR BRACKET
645                0x3011 => 0xfe3c, // RIGHT BLACK LENTICULAR BRACKET
646                0x3014 => 0xfe39, // LEFT TORTOISE SHELL BRACKET
647                0x3015 => 0xfe3a, // RIGHT TORTOISE SHELL BRACKET
648                0x3016 => 0xfe17, // LEFT WHITE LENTICULAR BRACKET
649                0x3017 => 0xfe18, // RIGHT WHITE LENTICULAR BRACKET
650                _ => return None,
651            },
652            0xfe => match self {
653                0xfe4f => 0xfe34, // WAVY LOW LINE
654                _ => return None,
655            },
656            0xff => match self {
657                0xff01 => 0xfe15, // FULLWIDTH EXCLAMATION MARK
658                0xff08 => 0xfe35, // FULLWIDTH LEFT PARENTHESIS
659                0xff09 => 0xfe36, // FULLWIDTH RIGHT PARENTHESIS
660                0xff0c => 0xfe10, // FULLWIDTH COMMA
661                0xff1a => 0xfe13, // FULLWIDTH COLON
662                0xff1b => 0xfe14, // FULLWIDTH SEMICOLON
663                0xff1f => 0xfe16, // FULLWIDTH QUESTION MARK
664                0xff3b => 0xfe47, // FULLWIDTH LEFT SQUARE BRACKET
665                0xff3d => 0xfe48, // FULLWIDTH RIGHT SQUARE BRACKET
666                0xff3f => 0xfe33, // FULLWIDTH LOW LINE
667                0xff5b => 0xfe37, // FULLWIDTH LEFT CURLY BRACKET
668                0xff5d => 0xfe38, // FULLWIDTH RIGHT CURLY BRACKET
669                _ => return None,
670            },
671            _ => return None,
672        })
673    }
674}
675
676#[cfg(feature = "icu")]
677pub(crate) use icu::*;
678
679#[cfg(not(feature = "icu"))]
680pub(crate) use builtin::*;
681
682#[cfg(feature = "icu")]
683mod icu {
684    use super::{Codepoint, GeneralCategory, Script};
685    use icu_normalizer::properties::{
686        CanonicalCompositionBorrowed, CanonicalDecompositionBorrowed, Decomposed,
687    };
688    use icu_properties::{
689        props::{
690            BidiMirroringGlyph, CanonicalCombiningClass as IcuCanonicalCombiningClass,
691            GeneralCategory as IcuGeneralCategory, Script as IcuScript,
692        },
693        CodePointMapData, PropertyNamesShort,
694    };
695
696    pub(crate) fn script_for(c: u32) -> Script {
697        let Some(icu_script) = PropertyNamesShort::new()
698            .get_locale_script(CodePointMapData::<IcuScript>::new().get32(c))
699        else {
700            return Script::UNKNOWN;
701        };
702        // As this is the script of some Unicode code point, correctly cased, we don't need to apply
703        // any of the checks and conversion from Script::from_iso15924. Furthermore, both the
704        // `from_bytes` and the `into_raw` calls are free, as the representation of both types is [u8; 4].
705        Script::from_bytes(&icu_script.into_raw())
706    }
707
708    pub(crate) fn general_category_for(c: u32) -> GeneralCategory {
709        GeneralCategory::from_icu(CodePointMapData::<IcuGeneralCategory>::new().get32(c))
710    }
711
712    pub(crate) fn combining_class_for(c: u32) -> u8 {
713        CodePointMapData::<IcuCanonicalCombiningClass>::new()
714            .get32(c)
715            .to_icu4c_value()
716    }
717
718    pub(crate) fn mirroring_for(c: u32) -> Option<Codepoint> {
719        CodePointMapData::<BidiMirroringGlyph>::new()
720            .get32(c)
721            .mirroring_glyph
722            .map(Codepoint::from)
723    }
724
725    pub(crate) fn compose_impl(a: Codepoint, b: Codepoint) -> Option<Codepoint> {
726        let a = char::from_u32(a)?;
727        let b = char::from_u32(b)?;
728        CanonicalCompositionBorrowed::new()
729            .compose(a, b)
730            .map(Codepoint::from)
731    }
732
733    pub(crate) fn decompose_impl(ab: Codepoint) -> Option<(Codepoint, Codepoint)> {
734        let ch = char::from_u32(ab)?;
735        match CanonicalDecompositionBorrowed::new().decompose(ch) {
736            Decomposed::Default => None,
737            Decomposed::Singleton(a) => Some((Codepoint::from(a), 0)),
738            Decomposed::Expansion(a, b) => Some((Codepoint::from(a), Codepoint::from(b))),
739        }
740    }
741
742    impl GeneralCategory {
743        fn from_icu(category: IcuGeneralCategory) -> Self {
744            use IcuGeneralCategory::*;
745            match category {
746                Control => Self::CONTROL,
747                Format => Self::FORMAT,
748                Unassigned => Self::UNASSIGNED,
749                PrivateUse => Self::PRIVATE_USE,
750                Surrogate => Self::SURROGATE,
751                LowercaseLetter => Self::LOWERCASE_LETTER,
752                ModifierLetter => Self::MODIFIER_LETTER,
753                OtherLetter => Self::OTHER_LETTER,
754                TitlecaseLetter => Self::TITLECASE_LETTER,
755                UppercaseLetter => Self::UPPERCASE_LETTER,
756                SpacingMark => Self::SPACING_MARK,
757                EnclosingMark => Self::ENCLOSING_MARK,
758                NonspacingMark => Self::NON_SPACING_MARK,
759                DecimalNumber => Self::DECIMAL_NUMBER,
760                LetterNumber => Self::LETTER_NUMBER,
761                OtherNumber => Self::OTHER_NUMBER,
762                ConnectorPunctuation => Self::CONNECT_PUNCTUATION,
763                DashPunctuation => Self::DASH_PUNCTUATION,
764                ClosePunctuation => Self::CLOSE_PUNCTUATION,
765                FinalPunctuation => Self::FINAL_PUNCTUATION,
766                InitialPunctuation => Self::INITIAL_PUNCTUATION,
767                OtherPunctuation => Self::OTHER_PUNCTUATION,
768                OpenPunctuation => Self::OPEN_PUNCTUATION,
769                CurrencySymbol => Self::CURRENCY_SYMBOL,
770                ModifierSymbol => Self::MODIFIER_SYMBOL,
771                MathSymbol => Self::MATH_SYMBOL,
772                OtherSymbol => Self::OTHER_SYMBOL,
773                LineSeparator => Self::LINE_SEPARATOR,
774                ParagraphSeparator => Self::PARAGRAPH_SEPARATOR,
775                SpaceSeparator => Self::SPACE_SEPARATOR,
776            }
777        }
778    }
779}
780
781#[cfg(not(feature = "icu"))]
782#[expect(clippy::manual_range_contains)]
783mod builtin {
784    use super::{super::algs::*, ucd_table::ucd::*, Codepoint, GeneralCategory, Script};
785
786    pub(crate) fn script_for(c: u32) -> Script {
787        ucd_sc_map[ucd_sc(c as usize) as usize]
788    }
789
790    pub(crate) fn general_category_for(c: u32) -> GeneralCategory {
791        GeneralCategory(ucd_gc(c as usize))
792    }
793
794    pub(crate) fn combining_class_for(c: u32) -> u8 {
795        ucd_ccc(c as usize)
796    }
797
798    pub(crate) fn mirroring_for(c: u32) -> Option<Codepoint> {
799        let delta = ucd_bmg(c as usize);
800        if delta == 0 {
801            None
802        } else {
803            Some(((c as i32).wrapping_add(delta)) as u32)
804        }
805    }
806
807    pub(crate) fn compose_impl(a: Codepoint, b: Codepoint) -> Option<Codepoint> {
808        // Hangul is handled algorithmically.
809        if let Some(ab) = compose_hangul(a, b) {
810            return Some(ab);
811        }
812
813        let u = if (a & 0xFFFF_F800) == 0x0000 && (b & 0xFFFF_FF80) == 0x0300 {
814            /* If "a" is small enough and "b" is in the U+0300 range,
815             * the composition data is encoded in a 32bit array sorted
816             * by "a,b" pair. */
817            let k = HB_CODEPOINT_ENCODE3_11_7_14(a, b, 0);
818            let v = ucd_dm2_u32_map
819                .binary_search_by(|probe| {
820                    let key = probe & HB_CODEPOINT_ENCODE3_11_7_14(0x001F_FFFF, 0x001F_FFFF, 0);
821                    key.cmp(&k)
822                })
823                .ok()
824                .map(|index| ucd_dm2_u32_map[index]);
825
826            HB_CODEPOINT_DECODE3_11_7_14_3(v?)
827        } else {
828            /* Otherwise it is stored in a 64bit array sorted by
829             * "a,b" pair. */
830            let k = HB_CODEPOINT_ENCODE3(a, b, 0);
831            let v = ucd_dm2_u64_map
832                .binary_search_by(|probe| {
833                    let key = probe & HB_CODEPOINT_ENCODE3(0x001F_FFFF, 0x001F_FFFF, 0);
834                    key.cmp(&k)
835                })
836                .ok()
837                .map(|index| ucd_dm2_u64_map[index]);
838
839            HB_CODEPOINT_DECODE3_3(v?)
840        };
841
842        if u == 0 {
843            None
844        } else {
845            Some(u)
846        }
847    }
848
849    pub(crate) fn decompose_impl(ab: Codepoint) -> Option<(Codepoint, Codepoint)> {
850        if let Some((a, b)) = decompose_hangul(ab) {
851            return Some((a, b));
852        }
853
854        let mut i = ucd_dm(ab as usize) as usize;
855
856        // If no data, there's no decomposition.
857        if i == 0 {
858            return None;
859        }
860        i -= 1;
861
862        if i < ucd_dm1_p0_map.len() + ucd_dm1_p2_map.len() {
863            let a = if i < ucd_dm1_p0_map.len() {
864                ucd_dm1_p0_map[i] as u32
865            } else {
866                let j = i - ucd_dm1_p0_map.len();
867                0x20000 | ucd_dm1_p2_map[j] as u32
868            };
869            return Some((a, 0));
870        }
871
872        i -= ucd_dm1_p0_map.len() + ucd_dm1_p2_map.len();
873
874        if i < ucd_dm2_u32_map.len() {
875            let v = ucd_dm2_u32_map[i];
876            let a = HB_CODEPOINT_DECODE3_11_7_14_1(v);
877            let b = HB_CODEPOINT_DECODE3_11_7_14_2(v);
878            return Some((a, b));
879        }
880
881        i -= ucd_dm2_u32_map.len();
882
883        let v = ucd_dm2_u64_map[i];
884        let a = HB_CODEPOINT_DECODE3_1(v);
885        let b = HB_CODEPOINT_DECODE3_2(v);
886        Some((a, b))
887    }
888
889    const S_BASE: u32 = 0xAC00;
890    const L_BASE: u32 = 0x1100;
891    const V_BASE: u32 = 0x1161;
892    const T_BASE: u32 = 0x11A7;
893    const L_COUNT: u32 = 19;
894    const V_COUNT: u32 = 21;
895    const T_COUNT: u32 = 28;
896    const N_COUNT: u32 = V_COUNT * T_COUNT;
897    const S_COUNT: u32 = L_COUNT * N_COUNT;
898
899    fn compose_hangul(a: Codepoint, b: Codepoint) -> Option<Codepoint> {
900        let l = a;
901        let v = b;
902        if L_BASE <= l && l < (L_BASE + L_COUNT) && V_BASE <= v && v < (V_BASE + V_COUNT) {
903            let r = S_BASE + (l - L_BASE) * N_COUNT + (v - V_BASE) * T_COUNT;
904            Some(r)
905        } else if S_BASE <= l
906            && l <= (S_BASE + S_COUNT - T_COUNT)
907            && T_BASE < v
908            && v < (T_BASE + T_COUNT)
909            && (l - S_BASE) % T_COUNT == 0
910        {
911            let r = l + (v - T_BASE);
912            Some(r)
913        } else {
914            None
915        }
916    }
917
918    fn decompose_hangul(ab: Codepoint) -> Option<(Codepoint, Codepoint)> {
919        let si = ab.wrapping_sub(S_BASE);
920        if si >= S_COUNT {
921            return None;
922        }
923        let (a, b) = if si % T_COUNT != 0 {
924            // LV,T
925            (S_BASE + (si / T_COUNT) * T_COUNT, T_BASE + (si % T_COUNT))
926        } else {
927            // L,V
928            (L_BASE + (si / N_COUNT), V_BASE + (si % N_COUNT) / T_COUNT)
929        };
930        Some((a, b))
931    }
932}
933
934#[cfg(test)]
935mod tests {
936    #[test]
937    fn telugu_length_marks_reorder_before_virama() {
938        use super::CharExt;
939        assert_eq!(0x0C55u32.modified_combining_class(), 4);
940        assert_eq!(0x0C56u32.modified_combining_class(), 5);
941        assert_eq!(0x0C4Du32.modified_combining_class(), 9);
942        assert_eq!(super::combining_class(0x0C55), 84);
943        assert_eq!(super::combining_class(0x0C56), 91);
944    }
945
946    #[cfg(not(feature = "icu"))]
947    #[test]
948    fn unicode_18_scripts() {
949        use super::GeneralCategory as Gc;
950        use crate::Script;
951
952        for (range, script, category) in [
953            (
954                0x125A8..=0x1264B,
955                Script::PROTO_CUNEIFORM,
956                Gc::LETTER_NUMBER,
957            ),
958            (0x18E00..=0x19191, Script::JURCHEN, Gc::OTHER_LETTER),
959            (0x191A0..=0x191D2, Script::JURCHEN, Gc::OTHER_LETTER),
960            (0x3D000..=0x3FC3F, Script::SEAL, Gc::OTHER_LETTER),
961        ] {
962            for cp in range {
963                assert_eq!(super::script_for(cp), script, "U+{cp:04X}");
964                assert_eq!(super::general_category_for(cp), category, "U+{cp:04X}");
965                assert_eq!(super::combining_class_for(cp), 0, "U+{cp:04X}");
966            }
967        }
968    }
969
970    #[test]
971    fn general_category_ranges() {
972        use super::GeneralCategory as Gc;
973        assert!(!Gc::CONTROL.in_range_inclusive(Gc::FORMAT, Gc::NON_SPACING_MARK));
974        assert!(Gc::LOWERCASE_LETTER.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER));
975        assert!(Gc::UPPERCASE_LETTER.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER));
976        assert!(!Gc::SPACING_MARK.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER));
977    }
978
979    #[cfg(feature = "icu")]
980    #[test]
981    fn script_mapping() {
982        use icu_properties::props::Script as IcuScript;
983        use icu_properties::{CodePointMapData, PropertyNamesShort};
984
985        let mut mismatches = Vec::new();
986
987        // All ICU scripts that are returned for some code point convert to the correct crate::Script
988        for cp in 0..=char::MAX as u32 {
989            let script = super::icu::script_for(cp);
990            let short_name = PropertyNamesShort::<IcuScript>::new()
991                .get_locale_script(CodePointMapData::new().get32(cp))
992                .unwrap();
993            let tag = crate::Tag::from_be_bytes(short_name.into_raw());
994            let expected = crate::Script::from_iso15924_tag(tag).unwrap();
995            if expected != script {
996                let expected_tag = expected.tag().to_be_bytes();
997                let actual_tag = script.tag().to_be_bytes();
998                mismatches.push(format!(
999                    "Script::from_iso15924_tag({short_name}) -> {}, mapped tag {}",
1000                    core::str::from_utf8(&expected_tag).unwrap_or("????"),
1001                    core::str::from_utf8(&actual_tag).unwrap_or("????")
1002                ));
1003            }
1004        }
1005
1006        assert!(
1007            mismatches.is_empty(),
1008            "script mapping mismatches:\n{}",
1009            mismatches.join("\n")
1010        );
1011    }
1012}