Skip to main content

harfrust/
normalize.rs

1use super::plan::ShapePlan;
2use crate::buffer::*;
3use crate::ot::shaper::{ComposeFn, DecomposeFn, MAX_COMBINING_MARKS};
4use crate::unicode::{space_fallback, CharExt, Codepoint};
5use crate::NominalGlyphs;
6use crate::ShaperFont;
7use read_fonts::types::GlyphId;
8
9impl GlyphInfo {
10    declare_buffer_var!(
11        u32,
12        1,
13        0,
14        NORMALIZER_GLYPH_INDEX_VAR,
15        normalizer_glyph_index,
16        set_normalizer_glyph_index
17    );
18}
19
20pub struct NormalizeContext<'a, 'x, 'c, 'd> {
21    pub plan: &'a ShapePlan,
22    pub buffer: &'x mut Buffer,
23    pub font: &'x ShaperFont<'c, 'd>,
24    pub decompose: DecomposeFn,
25    pub compose: ComposeFn,
26}
27
28impl NormalizeContext<'_, '_, '_, '_> {
29    fn nominal_glyph(&mut self, codepoint: u32) -> Option<GlyphId> {
30        self.font.nominal_glyph(codepoint)
31    }
32
33    fn variation_glyph(&mut self, codepoint: u32, selector: u32) -> Option<GlyphId> {
34        self.font.variation_glyph(codepoint, selector)
35    }
36
37    fn set_current_glyph(&mut self) -> Option<()> {
38        let info = self.buffer.cur_mut(0)?;
39        if let Some(glyph_id) = self.font.nominal_glyph(info.glyph_id) {
40            info.set_normalizer_glyph_index(u32::from(glyph_id));
41        }
42        Some(())
43    }
44}
45
46#[derive(Clone, Copy, PartialEq, Eq)]
47pub enum NormalizationMode {
48    None,
49    Decomposed,
50    /// Never composes base to base.
51    ComposedDiacritics,
52    /// Fully decomposes and recomposes.
53    ComposedDiacriticsNoShortCircuit,
54    Auto,
55}
56
57// HIGHLEVEL DESIGN:
58//
59// This file exports one main function: normalize().
60//
61// This function closely reflects the Unicode Normalization Algorithm,
62// yet it's different.
63//
64// Each shaper specifies whether it prefers decomposed (NFD) or composed (NFC).
65// The logic however tries to use whatever the font can support.
66//
67// In general what happens is that: each grapheme is decomposed in a chain
68// of 1:2 decompositions, marks reordered, and then recomposed if desired,
69// so far it's like Unicode Normalization.  However, the decomposition and
70// recomposition only happens if the font supports the resulting characters.
71//
72// The goals are:
73//
74//   - Try to render all canonically equivalent strings similarly.  To really
75//     achieve this we have to always do the full decomposition and then
76//     selectively recompose from there.  It's kinda too expensive though, so
77//     we skip some cases.  For example, if composed is desired, we simply
78//     don't touch 1-character clusters that are supported by the font, even
79//     though their NFC may be different.
80//
81//   - When a font has a precomposed character for a sequence but the 'ccmp'
82//     feature in the font is not adequate, use the precomposed character
83//     which typically has better mark positioning.
84//
85//   - When a font does not support a combining mark, but supports it precomposed
86//     with previous base, use that.  This needs the itemizer to have this
87//     knowledge too.  We need to provide assistance to the itemizer.
88//
89//   - When a font does not support a character but supports its canonical
90//     decomposition, well, use the decomposition.
91//
92//   - The shapers can customize the compose and decompose functions to
93//     offload some of their requirements to the normalizer.  For example, the
94//     Indic shaper may want to disallow recomposing of two matras.
95
96fn decompose_unicode(_: &NormalizeContext, ab: Codepoint) -> Option<(Codepoint, Codepoint)> {
97    crate::unicode::decompose_impl(ab)
98}
99
100fn compose_unicode(_: &NormalizeContext, a: Codepoint, b: Codepoint) -> Option<Codepoint> {
101    crate::unicode::compose(a, b)
102}
103
104fn output_char(buffer: &mut Buffer, unichar: u32, glyph: u32) -> Option<()> {
105    // This is very confusing indeed.
106    buffer.cur_mut(0)?.set_normalizer_glyph_index(glyph);
107    buffer.output_glyph(unichar);
108    // TODO: should be _hb_glyph_info_set_unicode_props (&buffer->prev(), buffer);
109    let mut flags = buffer.scratch_flags;
110    buffer.prev_mut()?.init_unicode_props(&mut flags);
111    buffer.scratch_flags = flags;
112    Some(())
113}
114
115fn next_char(buffer: &mut Buffer, glyph: u32) -> Option<()> {
116    buffer.cur_mut(0)?.set_normalizer_glyph_index(glyph);
117    buffer.next_glyph();
118    Some(())
119}
120
121fn skip_char(buffer: &mut Buffer) {
122    buffer.skip_glyph();
123}
124
125/// Returns 0 if didn't decompose, number of resulting characters otherwise.
126fn decompose(ctx: &mut NormalizeContext, shortest: bool, ab: Codepoint) -> u32 {
127    let Some((a, b)) = (ctx.decompose)(ctx, ab) else {
128        return 0;
129    };
130
131    let a_glyph = ctx.nominal_glyph(a);
132    let b_glyph = if b != 0 {
133        match ctx.nominal_glyph(b) {
134            Some(glyph_id) => Some(glyph_id),
135            None => return 0,
136        }
137    } else {
138        None
139    };
140
141    if let Some(a_glyph) = a_glyph {
142        if shortest {
143            // Output a and b
144            if output_char(ctx.buffer, a, u32::from(a_glyph)).is_none() {
145                ctx.buffer.index_failed();
146            }
147            if let Some(b_glyph) = b_glyph {
148                if output_char(ctx.buffer, b, u32::from(b_glyph)).is_none() {
149                    ctx.buffer.index_failed();
150                }
151                return 2;
152            }
153            return 1;
154        }
155    }
156
157    let ret = decompose(ctx, shortest, a);
158    if ret != 0 {
159        if let Some(b_glyph) = b_glyph {
160            if output_char(ctx.buffer, b, u32::from(b_glyph)).is_none() {
161                ctx.buffer.index_failed();
162            }
163            return ret + 1;
164        }
165        return ret;
166    }
167
168    if let Some(a_glyph) = a_glyph {
169        if output_char(ctx.buffer, a, u32::from(a_glyph)).is_none() {
170            ctx.buffer.index_failed();
171        }
172        if let Some(b_glyph) = b_glyph {
173            if output_char(ctx.buffer, b, u32::from(b_glyph)).is_none() {
174                ctx.buffer.index_failed();
175            }
176            return 2;
177        }
178        return 1;
179    }
180
181    0
182}
183
184fn decompose_current_character(ctx: &mut NormalizeContext, shortest: bool) -> Option<()> {
185    let u = ctx.buffer.cur(0)?.as_codepoint();
186    let glyph = ctx.nominal_glyph(u);
187
188    if let Some(glyph) = glyph {
189        if shortest {
190            if next_char(ctx.buffer, u32::from(glyph)).is_none() {
191                ctx.buffer.index_failed();
192            }
193            return Some(());
194        }
195    }
196
197    if decompose(ctx, shortest, u) > 0 {
198        skip_char(ctx.buffer);
199        return Some(());
200    }
201
202    if let Some(glyph) = glyph {
203        if next_char(ctx.buffer, u32::from(glyph)).is_none() {
204            ctx.buffer.index_failed();
205        }
206        return Some(());
207    }
208
209    if ctx.buffer.cur(0)?.is_unicode_space() {
210        let space_type = u.space_fallback();
211        if space_type != space_fallback::NOT_SPACE {
212            let space_glyph = ctx.nominal_glyph(0x0020).or(ctx.buffer.invisible);
213
214            if let Some(space_glyph) = space_glyph {
215                ctx.buffer
216                    .cur_mut(0)?
217                    .set_unicode_space_fallback_type(space_type);
218                if next_char(ctx.buffer, u32::from(space_glyph)).is_none() {
219                    ctx.buffer.index_failed();
220                }
221                ctx.buffer.scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_SPACE_FALLBACK;
222                return Some(());
223            }
224        }
225    }
226
227    // U+2011 is the only sensible character that is a no-break version of another character
228    // and not a space.  The space ones are handled already.  Handle this lone one.
229    if u == 0x2011 {
230        if let Some(other_glyph) = ctx.nominal_glyph(0x2010) {
231            if next_char(ctx.buffer, u32::from(other_glyph)).is_none() {
232                ctx.buffer.index_failed();
233            }
234            return Some(());
235        }
236    }
237
238    // Insert a .notdef glyph if decomposition failed.
239    if next_char(ctx.buffer, 0).is_none() {
240        ctx.buffer.index_failed();
241    }
242    Some(())
243}
244
245fn handle_variation_selector_cluster(
246    ctx: &mut NormalizeContext,
247    end: usize,
248    _: bool,
249) -> Option<()> {
250    // Currently if there's a variation-selector we give-up on normalization, it's just too hard.
251    while ctx.buffer.idx < end - 1 && ctx.buffer.successful {
252        if ctx.buffer.cur(1)?.as_codepoint().is_variation_selector() {
253            let base = ctx.buffer.cur(0)?.as_codepoint();
254            let selector = ctx.buffer.cur(1)?.as_codepoint();
255            if let Some(glyph_id) = ctx.variation_glyph(base, selector) {
256                ctx.buffer
257                    .cur_mut(0)?
258                    .set_normalizer_glyph_index(u32::from(glyph_id));
259                let unicode = ctx.buffer.cur(0)?.glyph_id;
260                ctx.buffer.replace_glyphs(2, 1, &[unicode]);
261            } else {
262                // Just pass on the two characters separately, let GSUB do its magic.
263                if ctx.set_current_glyph().is_none() {
264                    ctx.buffer.index_failed();
265                }
266                ctx.buffer.next_glyph();
267
268                ctx.buffer.scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_VARIATION_SELECTOR_FALLBACK;
269
270                ctx.buffer.cur_mut(0)?.set_variation_selector(true);
271
272                if ctx.buffer.not_found_variation_selector.is_some() {
273                    ctx.buffer.cur_mut(0)?.clear_default_ignorable();
274                }
275
276                if ctx.set_current_glyph().is_none() {
277                    ctx.buffer.index_failed();
278                }
279                ctx.buffer.next_glyph();
280            }
281
282            // Skip any further variation selectors.
283            while ctx.buffer.idx < end && ctx.buffer.cur(0)?.as_codepoint().is_variation_selector()
284            {
285                if ctx.set_current_glyph().is_none() {
286                    ctx.buffer.index_failed();
287                }
288                ctx.buffer.next_glyph();
289            }
290        } else {
291            if ctx.set_current_glyph().is_none() {
292                ctx.buffer.index_failed();
293            }
294            ctx.buffer.next_glyph();
295        }
296    }
297
298    if ctx.buffer.idx < end {
299        if ctx.set_current_glyph().is_none() {
300            ctx.buffer.index_failed();
301        }
302        ctx.buffer.next_glyph();
303    }
304    Some(())
305}
306
307fn decompose_multi_char_cluster(ctx: &mut NormalizeContext, end: usize, short_circuit: bool) {
308    let mut i = ctx.buffer.idx;
309    while i < end && ctx.buffer.successful {
310        if ctx
311            .buffer
312            .info
313            .get(i)
314            .is_some_and(|info| info.as_codepoint().is_variation_selector())
315        {
316            if handle_variation_selector_cluster(ctx, end, short_circuit).is_none() {
317                ctx.buffer.index_failed();
318            }
319            return;
320        }
321        i += 1;
322    }
323
324    while ctx.buffer.idx < end && ctx.buffer.successful {
325        if decompose_current_character(ctx, short_circuit).is_none() {
326            ctx.buffer.index_failed();
327        }
328    }
329}
330
331fn compare_combining_class(pa: &GlyphInfo, pb: &GlyphInfo) -> bool {
332    let a = pa.modified_combining_class();
333    let b = pb.modified_combining_class();
334    a > b
335}
336
337pub fn normalize<'x>(
338    plan: &ShapePlan,
339    buffer: &'x mut Buffer,
340    font: &'x ShaperFont<'_, '_>,
341) -> Option<()> {
342    if buffer.is_empty() {
343        return Some(());
344    }
345
346    buffer.assert_unicode_vars();
347
348    let mut mode = plan.shaper.normalization_preference;
349    if mode == NormalizationMode::Auto {
350        if plan.has_gpos_mark {
351            // https://github.com/harfbuzz/harfbuzz/issues/653#issuecomment-423905920
352            // mode = Some(NormalizationMode::Decomposed);
353            mode = NormalizationMode::ComposedDiacritics;
354        } else {
355            mode = NormalizationMode::ComposedDiacritics;
356        }
357    }
358
359    let mut ctx = NormalizeContext {
360        plan,
361        buffer,
362        font,
363        decompose: plan.shaper.decompose.unwrap_or(decompose_unicode),
364        compose: plan.shaper.compose.unwrap_or(compose_unicode),
365    };
366
367    let always_short_circuit = mode == NormalizationMode::None;
368    let might_short_circuit = always_short_circuit
369        || (mode != NormalizationMode::Decomposed
370            && mode != NormalizationMode::ComposedDiacriticsNoShortCircuit);
371
372    // We do a fairly straightforward yet custom normalization process in three
373    // separate rounds: decompose, reorder, recompose (if desired).  Currently
374    // this makes two buffer swaps.  We can make it faster by moving the last
375    // two rounds into the inner loop for the first round, but it's more readable
376    // this way.
377
378    // First round, decompose
379    let mut all_simple = true;
380    if message!(ctx, "start decompose") {
381        ctx.buffer.clear_output();
382        let count = ctx.buffer.len;
383        ctx.buffer.idx = 0;
384        loop {
385            let mut end = ctx.buffer.idx + 1;
386            while end < count
387                && ctx
388                    .buffer
389                    .info
390                    .get(end)
391                    .is_some_and(|info| !info.is_unicode_mark())
392            {
393                end += 1;
394            }
395
396            if end < count {
397                // Leave one base for the marks to cluster with.
398                end -= 1;
399            }
400
401            // From idx to end are simple clusters.
402            if might_short_circuit {
403                let idx = ctx.buffer.idx;
404                // Any glyphs not mapped here are handled one at a time below.
405                let done = ctx.buffer.info.get_mut(idx..end).map_or(0, |infos| {
406                    ctx.font.nominal_glyphs(NominalGlyphs::new(infos))
407                });
408                ctx.buffer.next_glyphs(done);
409            }
410
411            while ctx.buffer.idx < end && ctx.buffer.successful {
412                if decompose_current_character(&mut ctx, might_short_circuit).is_none() {
413                    ctx.buffer.index_failed();
414                }
415            }
416
417            if ctx.buffer.idx == count || !ctx.buffer.successful {
418                break;
419            }
420
421            all_simple = false;
422
423            // Find all the marks now.
424            end = ctx.buffer.idx + 1;
425            while end < count
426                && ctx
427                    .buffer
428                    .info
429                    .get(end)
430                    .is_some_and(|info| info.is_unicode_mark())
431            {
432                end += 1;
433            }
434
435            // idx to end is one non-simple cluster.
436            decompose_multi_char_cluster(&mut ctx, end, always_short_circuit);
437
438            if ctx.buffer.idx >= count || !ctx.buffer.successful {
439                break;
440            }
441        }
442
443        ctx.buffer.sync();
444        message!(ctx, "end decompose");
445    }
446
447    // Second round, reorder (inplace)
448    if !all_simple && buffer_message!(ctx.buffer, ctx.font, "start reorder") {
449        let count = ctx.buffer.len;
450        let mut i = 0;
451        while i < count {
452            if ctx
453                .buffer
454                .info
455                .get(i)
456                .is_none_or(|info| info.modified_combining_class() == 0)
457            {
458                i += 1;
459                continue;
460            }
461
462            let mut end = i + 1;
463            while end < count
464                && ctx
465                    .buffer
466                    .info
467                    .get(end)
468                    .is_some_and(|info| info.modified_combining_class() != 0)
469            {
470                end += 1;
471            }
472
473            // We are going to do a O(n^2).  Only do this if the sequence is short.
474            if end - i <= MAX_COMBINING_MARKS {
475                ctx.buffer.sort(i, end, compare_combining_class);
476
477                if let Some(reorder_marks) = ctx.plan.shaper.reorder_marks {
478                    reorder_marks(ctx.plan, ctx.buffer, i, end);
479                }
480            }
481
482            i = end + 1;
483        }
484        message!(ctx, "end reorder");
485    }
486    if ctx.buffer.scratch_flags & HB_BUFFER_SCRATCH_FLAG_HAS_CGJ != 0 {
487        // For all CGJ, check if it prevented any reordering at all.
488        // If it did NOT, then make it skippable.
489        // https://github.com/harfbuzz/harfbuzz/issues/554
490        let infos = ctx.buffer.info_slice_mut();
491        for i in 1..infos.len().saturating_sub(1) {
492            let Some([last, info, next]) = infos.get_mut(i - 1..=i + 1) else {
493                continue;
494            };
495            if info.glyph_id == 0x034F
496            /* CGJ */
497            {
498                let last = last.modified_combining_class();
499                let next = next.modified_combining_class();
500                if next == 0 || last <= next {
501                    info.unhide();
502                }
503            }
504        }
505    }
506
507    // Third round, recompose
508    if !all_simple
509        && ctx.buffer.successful
510        && (mode == NormalizationMode::ComposedDiacritics
511            || mode == NormalizationMode::ComposedDiacriticsNoShortCircuit)
512        && message!(ctx, "start compose")
513    {
514        // As noted in the comment earlier, we don't try to combine
515        // ccc=0 chars with their previous Starter.
516
517        let count = ctx.buffer.len;
518        let mut starter = 0;
519        ctx.buffer.clear_output();
520        ctx.buffer.next_glyph();
521        while ctx.buffer.idx < count && ctx.buffer.successful {
522            // We don't try to compose a non-mark character with it's preceding starter.
523            // This is both an optimization to avoid trying to compose every two neighboring
524            // glyphs in most scripts AND a desired feature for Hangul.  Apparently Hangul
525            // fonts are not designed to mix-and-match pre-composed syllables and Jamo.
526            let cur = ctx.buffer.cur(0)?;
527            if cur.is_unicode_mark() &&
528                // If there's anything between the starter and this char, they should have CCC
529                // smaller than this character's.
530                (starter == ctx.buffer.out_len - 1
531                    || ctx.buffer.prev()?.modified_combining_class() < cur.modified_combining_class())
532            {
533                let composed = ctx.buffer.out_info().get(starter).and_then(|starter_info| {
534                    (ctx.compose)(&ctx, starter_info.as_codepoint(), cur.as_codepoint())
535                });
536                if let Some(composed) = composed {
537                    if let Some(glyph_id) = ctx.nominal_glyph(composed) {
538                        // Copy to out-ctx.buffer.
539                        ctx.buffer.next_glyph();
540                        if !ctx.buffer.successful {
541                            return Some(());
542                        }
543
544                        // Merge and remove the second composable.
545                        ctx.buffer.merge_out_clusters(starter, ctx.buffer.out_len);
546                        ctx.buffer.out_len -= 1;
547
548                        // Modify starter and carry on.
549                        let mut flags = ctx.buffer.scratch_flags;
550                        if let Some(info) = ctx.buffer.out_info_mut().get_mut(starter) {
551                            info.glyph_id = composed;
552                            info.set_normalizer_glyph_index(u32::from(glyph_id));
553                            info.init_unicode_props(&mut flags);
554                        }
555                        ctx.buffer.scratch_flags = flags;
556
557                        continue;
558                    }
559                }
560            }
561
562            // Blocked, or doesn't compose.
563            ctx.buffer.next_glyph();
564
565            if ctx.buffer.prev()?.modified_combining_class() == 0 {
566                starter = ctx.buffer.out_len - 1;
567            }
568        }
569
570        ctx.buffer.sync();
571        message!(ctx, "end compose");
572    }
573    Some(())
574}