icu_normalizer/
latin1.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5//! Methods for normalizing Latin1 input into a UTF-16 sink.
6//!
7//! NFC is not available, since Latin1 input is already known to be
8//! in NFC.
9
10use write16::Write16;
11
12/// Entries start from U+00A0 NO-BREAK SPACE. If the character is
13/// always its own normalization, the value in the table is 0.
14/// If the character has a compatibility decompositons, the value
15/// in the table is the index into `COMPATIBILITY_DECOMPOSITIONS`
16/// shifted left by two and the length of the subslice of
17/// `COMPATIBILITY_DECOMPOSITIONS` in the low 2 bits. This means
18/// that the high half is zero. Otherwise, the high 8 bits are the
19/// first character of the canonical decomposition and the low 8
20/// bits are the offset that needs to be added to U+0300 to get the
21/// second character of the canonical decomposition.
22static TABLE: [u16; 96] = [
23    0x01,   // nbsp
24    0,      // ¡
25    0,      // ¢
26    0,      // £
27    0,      // ¤
28    0,      // ¥
29    0,      // ¦
30    0,      // §
31    0x02,   // ¨
32    0,      // ©
33    0x09,   // ª
34    0,      // «
35    0,      // ¬
36    0,      // shy
37    0,      // ®
38    0x0E,   // ¯
39    0,      // °
40    0,      // ±
41    0x41,   // ²
42    0x45,   // ³
43    0x16,   // ´
44    0x1D,   // µ
45    0,      // ¶
46    0,      // ·
47    0x22,   // ¸
48    0x2D,   // ¹
49    0x29,   // º
50    0,      // »
51    0x2F,   // ¼
52    0x3B,   // ½
53    0x47,   // ¾
54    0,      // ¿
55    0x4100, // À
56    0x4101, // Á
57    0x4102, // Â
58    0x4103, // Ã
59    0x4108, // Ä
60    0x410A, // Å
61    0,      // Æ
62    0x4327, // Ç
63    0x4500, // È
64    0x4501, // É
65    0x4502, // Ê
66    0x4508, // Ë
67    0x4900, // Ì
68    0x4901, // Í
69    0x4902, // Î
70    0x4908, // Ï
71    0,      // Ð
72    0x4E03, // Ñ
73    0x4F00, // Ò
74    0x4F01, // Ó
75    0x4F02, // Ô
76    0x4F03, // Õ
77    0x4F08, // Ö
78    0,      // ×
79    0,      // Ø
80    0x5500, // Ù
81    0x5501, // Ú
82    0x5502, // Û
83    0x5508, // Ü
84    0x5901, // Ý
85    0,      // Þ
86    0,      // ß
87    0x6100, // à
88    0x6101, // á
89    0x6102, // â
90    0x6103, // ã
91    0x6108, // ä
92    0x610A, // å
93    0,      // æ
94    0x6327, // ç
95    0x6500, // è
96    0x6501, // é
97    0x6502, // ê
98    0x6508, // ë
99    0x6900, // ì
100    0x6901, // í
101    0x6902, // î
102    0x6908, // ï
103    0,      // ð
104    0x6E03, // ñ
105    0x6F00, // ò
106    0x6F01, // ó
107    0x6F02, // ô
108    0x6F03, // õ
109    0x6F08, // ö
110    0,      // ÷
111    0,      // ø
112    0x7500, // ù
113    0x7501, // ú
114    0x7502, // û
115    0x7508, // ü
116    0x7901, // ý
117    0,      // þ
118    0x7908, // ÿ
119];
120
121/// Table containing the compatibility decompositions.
122static COMPATIBILITY_DECOMPOSITIONS: [u16; 20] = [
123    0x0020, 0x0308, 0x0061, 0x0020, 0x0304, 0x0020, 0x0301, 0x03BC, 0x0020, 0x0327, 0x006F, 0x0031,
124    0x2044, 0x0034, 0x0031, 0x2044, 0x0032, 0x0033, 0x2044, 0x0034,
125];
126
127const NFKC_BITS: u32 = const {
128    let mut accu = 0;
129    let mut i = 0;
130    while i < 0x20 {
131        if TABLE[i] != 0 {
132            accu |= 1 << (i as u32);
133        }
134        i += 1;
135    }
136    accu
137};
138
139const NFD_BITS: u64 = const {
140    let mut accu = 0;
141    let mut i = 0x20;
142    while i < TABLE.len() {
143        if TABLE[i] != 0 {
144            accu |= 1 << ((i - 0x20) as u32);
145        }
146        i += 1;
147    }
148    accu
149};
150
151const NFKD_BITS: u128 = const {
152    let mut accu = 0;
153    let mut i = 0;
154    while i < TABLE.len() {
155        if TABLE[i] != 0 {
156            accu |= 1 << ((i + 0x20) as u32);
157        }
158        i += 1;
159    }
160    accu
161};
162
163/// Writes the compatibility decomposition of `c` to `sink`.
164#[inline]
165fn compatibility_decomposition(val: u16) -> &'static [u16] {
166    debug_assert!(val <= 0xFF);
167    let len = val & 0b11;
168    let index = val >> 2;
169    COMPATIBILITY_DECOMPOSITIONS
170        .get(index as usize..index as usize + len as usize)
171        .unwrap_or_else(|| {
172            // Internal bug, not even GIGO, never supposed to happen
173            debug_assert!(false);
174            &[]
175        })
176}
177
178/// Normalize Latin1 `text` to NFD UTF-16 written to `sink`.
179#[inline]
180pub fn normalize_nfd_to<W: Write16 + ?Sized>(text: &[u16], sink: &mut W) -> core::fmt::Result {
181    // Indexing is OK, because the index is statically in range.
182    #[expect(clippy::indexing_slicing)]
183    let table = &TABLE[0x20..];
184    let mut text_left = text;
185    let mut iter = text_left.iter();
186    while let Some(u) = iter.next() {
187        let c = *u;
188        if c < 0xC0 {
189            continue;
190        }
191        if let Some(val) = table.get(c.wrapping_sub(0xC0) as usize) {
192            let v = *val;
193            if v != 0 {
194                let remaining = iter.as_slice();
195                // Indexing is OK by construction.
196                #[expect(clippy::indexing_slicing)]
197                sink.write_slice(&text_left[..text_left.len() - remaining.len() - 1])?;
198                text_left = remaining;
199                sink.write_slice(&[v >> 8, (v & 0xFF) + 0x0300])?;
200            }
201        }
202    }
203    sink.write_slice(text_left)?;
204    Ok(())
205}
206
207/// Normalize Latin1 `text` to NFKD UTF-16 written to `sink`.
208#[inline]
209pub fn normalize_nfkd_to<W: Write16 + ?Sized>(text: &[u16], sink: &mut W) -> core::fmt::Result {
210    let mut text_left = text;
211    let mut iter = text_left.iter();
212    while let Some(u) = iter.next() {
213        let c = *u;
214        if c < 0xA0 {
215            continue;
216        }
217        if let Some(val) = TABLE.get(c.wrapping_sub(0xA0) as usize) {
218            let v = *val;
219            if v != 0 {
220                let remaining = iter.as_slice();
221                // Indexing is OK by construction.
222                #[expect(clippy::indexing_slicing)]
223                sink.write_slice(&text_left[..text_left.len() - remaining.len() - 1])?;
224                text_left = remaining;
225                let hi = v >> 8;
226                if hi != 0 {
227                    sink.write_slice(&[hi, (v & 0xFF) + 0x0300])?;
228                } else {
229                    sink.write_slice(compatibility_decomposition(v))?;
230                }
231            }
232        }
233    }
234    sink.write_slice(text_left)?;
235    Ok(())
236}
237
238/// Normalize Latin1 `text` to NFKC UTF-16 written to `sink`.
239#[inline]
240pub fn normalize_nfkc_to<W: Write16 + ?Sized>(text: &[u16], sink: &mut W) -> core::fmt::Result {
241    // Indexing is OK, because the index is statically in range.
242    #[expect(clippy::indexing_slicing)]
243    let table = &TABLE[..0x20];
244    let mut text_left = text;
245    let mut iter = text_left.iter();
246    while let Some(u) = iter.next() {
247        let c = *u;
248        if c < 0xA0 {
249            continue;
250        }
251        if let Some(val) = table.get(c.wrapping_sub(0xA0) as usize) {
252            let v = *val;
253            if v != 0 {
254                let remaining = iter.as_slice();
255                // Indexing is OK by construction.
256                #[expect(clippy::indexing_slicing)]
257                sink.write_slice(&text_left[..text_left.len() - remaining.len() - 1])?;
258                text_left = remaining;
259                sink.write_slice(compatibility_decomposition(v))?;
260            }
261        }
262    }
263    sink.write_slice(text_left)?;
264    Ok(())
265}
266
267/// Split Latin1 `text` into `(head, tail)` such that the first
268/// byte of `tail` is the first byte of input that is not in NFD.
269/// If `text` is fully in NFD, `tail` is empty.
270#[inline]
271pub fn split_normalized_nfd(text: &[u8]) -> (&[u8], &[u8]) {
272    let mut iter = text.iter();
273    while let Some(c) = iter.next() {
274        let b = *c;
275        if let Some(shifted) = 1u64.checked_shl(u32::from(b.wrapping_sub(0xC0))) {
276            if (NFD_BITS & shifted) != 0 {
277                let tail = iter.as_slice();
278                return text
279                    .split_at_checked(text.len() - tail.len() - 1)
280                    .unwrap_or_else(|| {
281                        // Internal bug, not even GIGO, never supposed to happen
282                        debug_assert!(false);
283                        (&[], text)
284                    });
285            }
286        }
287    }
288    (text, &[])
289}
290
291/// Split Latin1 `text` into `(head, tail)` such that the first
292/// byte of `tail` is the first byte of input that is not in NFKD.
293/// If `text` is fully in NFKD, `tail` is empty.
294#[inline]
295pub fn split_normalized_nfkd(text: &[u8]) -> (&[u8], &[u8]) {
296    let mut iter = text.iter();
297    while let Some(c) = iter.next() {
298        let b = *c;
299        if let Some(shifted) = 1u128.checked_shl(u32::from(b.wrapping_sub(0x80))) {
300            if (NFKD_BITS & shifted) != 0 {
301                let tail = iter.as_slice();
302                return text
303                    .split_at_checked(text.len() - tail.len() - 1)
304                    .unwrap_or_else(|| {
305                        // Internal bug, not even GIGO, never supposed to happen
306                        debug_assert!(false);
307                        (&[], text)
308                    });
309            }
310        }
311    }
312    (text, &[])
313}
314
315/// Split Latin1 `text` into `(head, tail)` such that the first
316/// byte of `tail` is the first byte of input that is not in NFKC.
317/// If `text` is fully in NFKC, `tail` is empty.
318#[inline]
319pub fn split_normalized_nfkc(text: &[u8]) -> (&[u8], &[u8]) {
320    let mut iter = text.iter();
321    while let Some(c) = iter.next() {
322        let b = *c;
323        // Make ASCII go one instruction faster.
324        if b < 0xA0 {
325            continue;
326        }
327        if let Some(shifted) = 1u32.checked_shl(u32::from(b.wrapping_sub(0xA0))) {
328            if (NFKC_BITS & shifted) != 0 {
329                let tail = iter.as_slice();
330                return text
331                    .split_at_checked(text.len() - tail.len() - 1)
332                    .unwrap_or_else(|| {
333                        // Internal bug, not even GIGO, never supposed to happen
334                        debug_assert!(false);
335                        (&[], text)
336                    });
337            }
338        }
339    }
340    (text, &[])
341}
icu_normalizer/latin1.rs

icu_normalizer/
latin1.rs
