servo_base/
text.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::iter::Sum;
6use std::ops::{Add, AddAssign, Sub, SubAssign};
7
8use malloc_size_of_derive::MallocSizeOf;
9
10pub use crate::unicode_block::{UnicodeBlock, UnicodeBlockMethod};
11
12pub fn is_bidi_control(c: char) -> bool {
13    matches!(c, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}' | '\u{200E}' | '\u{200F}' | '\u{061C}')
14}
15
16pub fn unicode_plane(codepoint: char) -> u32 {
17    (codepoint as u32) >> 16
18}
19
20pub fn is_cjk(codepoint: char) -> bool {
21    if let Some(
22        UnicodeBlock::CJKRadicalsSupplement |
23        UnicodeBlock::KangxiRadicals |
24        UnicodeBlock::IdeographicDescriptionCharacters |
25        UnicodeBlock::CJKSymbolsandPunctuation |
26        UnicodeBlock::Hiragana |
27        UnicodeBlock::Katakana |
28        UnicodeBlock::Bopomofo |
29        UnicodeBlock::HangulCompatibilityJamo |
30        UnicodeBlock::Kanbun |
31        UnicodeBlock::BopomofoExtended |
32        UnicodeBlock::CJKStrokes |
33        UnicodeBlock::KatakanaPhoneticExtensions |
34        UnicodeBlock::EnclosedCJKLettersandMonths |
35        UnicodeBlock::CJKCompatibility |
36        UnicodeBlock::CJKUnifiedIdeographsExtensionA |
37        UnicodeBlock::YijingHexagramSymbols |
38        UnicodeBlock::CJKUnifiedIdeographs |
39        UnicodeBlock::CJKCompatibilityIdeographs |
40        UnicodeBlock::CJKCompatibilityForms |
41        UnicodeBlock::HalfwidthandFullwidthForms,
42    ) = codepoint.block()
43    {
44        return true;
45    }
46
47    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Supplementary_Ideographic_Plane
48    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Tertiary_Ideographic_Plane
49    unicode_plane(codepoint) == 2 || unicode_plane(codepoint) == 3
50}
51
52macro_rules! unicode_length_type {
53    ($( #[$doc:meta] )+ $type_name:ident) => {
54        $( #[$doc] )+
55        #[derive(Clone, Copy, Debug, Default, Eq, MallocSizeOf, Ord, PartialEq, PartialOrd)]
56        pub struct $type_name(pub usize);
57
58        impl $type_name {
59            pub fn zero() -> Self {
60                Self(0)
61            }
62
63            pub fn one() -> Self {
64                Self(1)
65            }
66
67            pub fn saturating_sub(self, value: Self) -> Self {
68                Self(self.0.saturating_sub(value.0))
69            }
70        }
71
72        impl From<u32> for $type_name {
73            fn from(value: u32) -> Self {
74                Self(value as usize)
75            }
76        }
77
78        impl From<isize> for $type_name {
79            fn from(value: isize) -> Self {
80                Self(value as usize)
81            }
82        }
83
84        impl Add for $type_name {
85            type Output = Self;
86            fn add(self, other: Self) -> Self {
87                Self(self.0 + other.0)
88            }
89        }
90
91        impl AddAssign for $type_name {
92            fn add_assign(&mut self, other: Self) {
93                *self = Self(self.0 + other.0)
94            }
95        }
96
97        impl Sub for $type_name {
98            type Output = Self;
99            fn sub(self, value: Self) -> Self {
100                Self(self.0 - value.0)
101            }
102        }
103
104        impl SubAssign for $type_name {
105            fn sub_assign(&mut self, other: Self) {
106                *self = Self(self.0 - other.0)
107            }
108        }
109
110        impl Sum for $type_name {
111            fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
112                iter.fold(Self::zero(), |a, b| Self(a.0 + b.0))
113            }
114        }
115    };
116}
117
118unicode_length_type! {
119    /// A length or offset counted in 8-bit code units (bytes) in an UTF-8 string.
120    /// This type is used to more reliable work with lengths or offsets in different encodings.
121    Utf8CodeUnits
122}
123
124unicode_length_type! {
125    /// A length or offset counted in 16-bit code units in an UTF-16 string.
126    /// This type is used to more reliable work with lengths or offsets in different encodings.
127    Utf16CodeUnits
128}
129
130unicode_length_type! {
131    /// A length or offset counted in 32-bit code units in UTF-32.
132    /// This is the same as counting Rust `char`s, Unicode scalar values, or Unicode code points.
133    /// This type is used to more reliable work with lengths or offsets in different encodings.
134    Utf32CodeUnits
135}
136
137impl Utf16CodeUnits {
138    pub fn length_of(string: &str) -> Self {
139        Self(string.bytes().map(len_utf16_for_utf8_byte).sum())
140
141        // TODO: after upgrading to a Rust version (1.99?) that includes that PR,
142        // replace the above with:
143
144        // // `EncodeUtf16::count` is optimized in https://github.com/rust-lang/rust/pull/159467
145        // Self(string.encode_utf16().count())
146    }
147
148    pub fn to_utf32_code_units_in(self, string: &str) -> Utf32CodeUnits {
149        let mut current_utf16_offset = Utf16CodeUnits(0);
150        let mut current_utf32_offset = Utf32CodeUnits(0);
151        for utf8_byte in string.bytes() {
152            if current_utf16_offset >= self {
153                break;
154            }
155            let len_utf16 = len_utf16_for_utf8_byte(utf8_byte);
156            current_utf16_offset.0 += len_utf16;
157            // `len_utf16 != 0` means this byte is the first byte of the UTF-8 byte sequence
158            // for one `char` /  UTF-32 code unit
159            current_utf32_offset.0 += (len_utf16 != 0) as usize;
160        }
161        current_utf32_offset
162    }
163}
164
165fn len_utf16_for_utf8_byte(byte: u8) -> usize {
166    if byte < 0b1000_0000 {
167        // 0b0xxx_xxxx: ASCII-compatible U+0000 to U+007F
168        1
169    } else if byte < 0b1100_0000 {
170        // 0b10xx_xxxx: UTF-8 continuation byte, already accounted for by its non-continuation byte
171        0
172    } else if byte < 0b1111_0000 {
173        // 0b110x_xxxx: start of a 2-byte UTF-8 sequence for U+0080 to U+07FF
174        // 0b1110_xxxx: start of a 3-byte UTF-8 sequence for U+0800 to U+FFFF
175        1
176    } else {
177        // 0b1111_0xxx: start of a 4-byte UTF-8 sequence for U+010000 to U+10FFFF
178        // This is exactly the range encoded as a surrogate pair in UTF-16
179        //
180        // 0b1111_1xxx: would fall here but never occurs in valid UTF-8
181        2
182    }
183}
184
185impl Utf32CodeUnits {
186    pub fn length_of(string: &str) -> Self {
187        // `std::str::Chars::count` is optimized in:
188        // https://github.com/rust-lang/rust/blob/main/library/core/src/str/count.rs
189        Self(string.chars().count())
190    }
191}
192
193#[cfg(test)]
194mod test {
195    use super::*;
196
197    #[test]
198    fn test_is_cjk() {
199        // Test characters from different CJK blocks
200        assert_eq!(is_cjk('〇'), true);
201        assert_eq!(is_cjk('㐀'), true);
202        assert_eq!(is_cjk('あ'), true);
203        assert_eq!(is_cjk('ア'), true);
204        assert_eq!(is_cjk('㆒'), true);
205        assert_eq!(is_cjk('ㆣ'), true);
206        assert_eq!(is_cjk('龥'), true);
207        assert_eq!(is_cjk('𰾑'), true);
208        assert_eq!(is_cjk('𰻝'), true);
209
210        // Test characters from outside CJK blocks
211        assert_eq!(is_cjk('a'), false);
212        assert_eq!(is_cjk('🙂'), false);
213        assert_eq!(is_cjk('©'), false);
214    }
215
216    #[test]
217    fn test_utf16_length() {
218        assert_eq!(Utf16CodeUnits::length_of(""), Utf16CodeUnits(0));
219        assert_eq!(Utf16CodeUnits::length_of("a"), Utf16CodeUnits(1));
220        assert_eq!(Utf16CodeUnits::length_of("é"), Utf16CodeUnits(1));
221        assert_eq!(Utf16CodeUnits::length_of("字"), Utf16CodeUnits(1));
222        assert_eq!(Utf16CodeUnits::length_of("\u{1F4A9}"), Utf16CodeUnits(2));
223        assert_eq!(
224            Utf16CodeUnits::length_of("\u{1F4A9}字éa"),
225            Utf16CodeUnits(5)
226        );
227    }
228
229    #[test]
230    fn test_utf16_to_utf32() {
231        let s = "aé字\u{1F4A9}";
232        assert_eq!(
233            Utf16CodeUnits(0).to_utf32_code_units_in(s),
234            Utf32CodeUnits(0)
235        );
236        assert_eq!(
237            Utf16CodeUnits(1).to_utf32_code_units_in(s),
238            Utf32CodeUnits(1)
239        );
240        assert_eq!(
241            Utf16CodeUnits(2).to_utf32_code_units_in(s),
242            Utf32CodeUnits(2)
243        );
244        assert_eq!(
245            Utf16CodeUnits(3).to_utf32_code_units_in(s),
246            Utf32CodeUnits(3)
247        );
248
249        // This 16-bit offset splits the would-be surrogate pair. We return the 32-bit position
250        // after the whole pair. Should this be an error instead?
251        assert_eq!(
252            Utf16CodeUnits(4).to_utf32_code_units_in(s),
253            Utf32CodeUnits(4)
254        );
255
256        assert_eq!(
257            Utf16CodeUnits(5).to_utf32_code_units_in(s),
258            Utf32CodeUnits(4)
259        );
260
261        // This 16-bit offset is out of bounds. We clamp to the nearest valid 32-bit offset,
262        // a.k.a the UTF-32 length. Should this be an error instead?
263        assert_eq!(
264            Utf16CodeUnits(6).to_utf32_code_units_in(s),
265            Utf32CodeUnits(4)
266        );
267        assert_eq!(
268            Utf16CodeUnits(7).to_utf32_code_units_in(s),
269            Utf32CodeUnits(4)
270        );
271    }
272}
servo_base/text.rs

servo_base/
text.rs
