servo_base/
text.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::iter::Sum;
6use std::ops::{Add, AddAssign, Range, Sub, SubAssign};
7
8use malloc_size_of_derive::MallocSizeOf;
9
10pub use crate::unicode_block::{UnicodeBlock, UnicodeBlockMethod};
11
12pub fn is_bidi_control(c: char) -> bool {
13    matches!(c, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}' | '\u{200E}' | '\u{200F}' | '\u{061C}')
14}
15
16pub fn unicode_plane(codepoint: char) -> u32 {
17    (codepoint as u32) >> 16
18}
19
20pub fn is_cjk(codepoint: char) -> bool {
21    if let Some(
22        UnicodeBlock::CJKRadicalsSupplement |
23        UnicodeBlock::KangxiRadicals |
24        UnicodeBlock::IdeographicDescriptionCharacters |
25        UnicodeBlock::CJKSymbolsandPunctuation |
26        UnicodeBlock::Hiragana |
27        UnicodeBlock::Katakana |
28        UnicodeBlock::Bopomofo |
29        UnicodeBlock::HangulCompatibilityJamo |
30        UnicodeBlock::Kanbun |
31        UnicodeBlock::BopomofoExtended |
32        UnicodeBlock::CJKStrokes |
33        UnicodeBlock::KatakanaPhoneticExtensions |
34        UnicodeBlock::EnclosedCJKLettersandMonths |
35        UnicodeBlock::CJKCompatibility |
36        UnicodeBlock::CJKUnifiedIdeographsExtensionA |
37        UnicodeBlock::YijingHexagramSymbols |
38        UnicodeBlock::CJKUnifiedIdeographs |
39        UnicodeBlock::CJKCompatibilityIdeographs |
40        UnicodeBlock::CJKCompatibilityForms |
41        UnicodeBlock::HalfwidthandFullwidthForms,
42    ) = codepoint.block()
43    {
44        return true;
45    }
46
47    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Supplementary_Ideographic_Plane
48    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Tertiary_Ideographic_Plane
49    unicode_plane(codepoint) == 2 || unicode_plane(codepoint) == 3
50}
51
52/// Equivalent to either `Range`, `RangeTo`, `RangeFrom`, or `RangeFull`
53#[derive(Clone, Copy)]
54pub struct RangeAny<T> {
55    /// `None` means zero
56    pub start: Option<T>,
57    /// `None` means the full available length
58    pub end: Option<T>,
59}
60
61impl<T> RangeAny<T> {
62    /// Apply `Option::map` to each bound of this range
63    pub fn map<U>(self, f: impl Fn(T) -> U + Copy) -> RangeAny<U> {
64        let Self { start, end } = self;
65        RangeAny {
66            start: start.map(f),
67            end: end.map(f),
68        }
69    }
70
71    /// Returns the intersection of two ranges, if it is non-empty
72    pub fn intersect(self, other: Self) -> Option<Self>
73    where
74        T: Ord,
75    {
76        // TODO: https://github.com/rust-lang/rust/issues/144273
77        // let start = a.start.reduce(b.start, std::cmp::max);
78        // let end = a.end.reduce(b.end, std::cmp::min);
79        let start = match (self.start, other.start) {
80            (None, None) => None,
81            (None, Some(b)) => Some(b),
82            (Some(a), None) => Some(a),
83            (Some(a), Some(b)) => Some(a.max(b)),
84        };
85        let end = match (self.end, other.end) {
86            (None, None) => None,
87            (None, Some(b)) => Some(b),
88            (Some(a), None) => Some(a),
89            (Some(a), Some(b)) => Some(a.min(b)),
90        };
91        if start
92            .as_ref()
93            .is_none_or(|start| end.as_ref().is_none_or(|end| start < end))
94        {
95            Some(RangeAny { start, end })
96        } else {
97            // `max()..min()` producing a "backwards" range means the intersection is empty
98            None
99        }
100    }
101}
102
103impl<T> From<Range<T>> for RangeAny<T> {
104    fn from(value: Range<T>) -> Self {
105        Self {
106            start: Some(value.start),
107            end: Some(value.end),
108        }
109    }
110}
111
112macro_rules! unicode_length_type {
113    ($( #[$doc:meta] )+ $type_name:ident) => {
114        $( #[$doc] )+
115        #[derive(Clone, Copy, Debug, Default, Eq, MallocSizeOf, Ord, PartialEq, PartialOrd)]
116        pub struct $type_name(pub usize);
117
118        impl $type_name {
119            pub fn zero() -> Self {
120                Self(0)
121            }
122
123            pub fn one() -> Self {
124                Self(1)
125            }
126
127            pub fn saturating_sub(self, value: Self) -> Self {
128                Self(self.0.saturating_sub(value.0))
129            }
130        }
131
132        impl From<u32> for $type_name {
133            fn from(value: u32) -> Self {
134                Self(value as usize)
135            }
136        }
137
138        impl From<isize> for $type_name {
139            fn from(value: isize) -> Self {
140                Self(value as usize)
141            }
142        }
143
144        impl Add for $type_name {
145            type Output = Self;
146            fn add(self, other: Self) -> Self {
147                Self(self.0 + other.0)
148            }
149        }
150
151        impl AddAssign for $type_name {
152            fn add_assign(&mut self, other: Self) {
153                *self = Self(self.0 + other.0)
154            }
155        }
156
157        impl Sub for $type_name {
158            type Output = Self;
159            fn sub(self, value: Self) -> Self {
160                Self(self.0 - value.0)
161            }
162        }
163
164        impl SubAssign for $type_name {
165            fn sub_assign(&mut self, other: Self) {
166                *self = Self(self.0 - other.0)
167            }
168        }
169
170        impl Sum for $type_name {
171            fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
172                iter.fold(Self::zero(), |a, b| Self(a.0 + b.0))
173            }
174        }
175    };
176}
177
178unicode_length_type! {
179    /// A length or offset counted in 8-bit code units (bytes) in an UTF-8 string.
180    /// This type is used to more reliable work with lengths or offsets in different encodings.
181    Utf8CodeUnits
182}
183
184unicode_length_type! {
185    /// A length or offset counted in 16-bit code units in an UTF-16 string.
186    /// This type is used to more reliable work with lengths or offsets in different encodings.
187    Utf16CodeUnits
188}
189
190unicode_length_type! {
191    /// A length or offset counted in 32-bit code units in UTF-32.
192    /// This is the same as counting Rust `char`s, Unicode scalar values, or Unicode code points.
193    /// This type is used to more reliable work with lengths or offsets in different encodings.
194    Utf32CodeUnits
195}
196
197impl Utf16CodeUnits {
198    pub fn length_of(string: &str) -> Self {
199        Self(string.bytes().map(len_utf16_for_utf8_byte).sum())
200
201        // TODO: after upgrading to a Rust version (1.99?) that includes that PR,
202        // replace the above with:
203
204        // // `EncodeUtf16::count` is optimized in https://github.com/rust-lang/rust/pull/159467
205        // Self(string.encode_utf16().count())
206    }
207
208    pub fn to_utf32_code_units_in(self, string: &str) -> Utf32CodeUnits {
209        let mut current_utf16_offset = Utf16CodeUnits(0);
210        let mut current_utf32_offset = Utf32CodeUnits(0);
211        for utf8_byte in string.bytes() {
212            if current_utf16_offset >= self {
213                break;
214            }
215            let len_utf16 = len_utf16_for_utf8_byte(utf8_byte);
216            current_utf16_offset.0 += len_utf16;
217            // `len_utf16 != 0` means this byte is the first byte of the UTF-8 byte sequence
218            // for one `char` /  UTF-32 code unit
219            current_utf32_offset.0 += (len_utf16 != 0) as usize;
220        }
221        current_utf32_offset
222    }
223}
224
225fn len_utf16_for_utf8_byte(byte: u8) -> usize {
226    if byte < 0b1000_0000 {
227        // 0b0xxx_xxxx: ASCII-compatible U+0000 to U+007F
228        1
229    } else if byte < 0b1100_0000 {
230        // 0b10xx_xxxx: UTF-8 continuation byte, already accounted for by its non-continuation byte
231        0
232    } else if byte < 0b1111_0000 {
233        // 0b110x_xxxx: start of a 2-byte UTF-8 sequence for U+0080 to U+07FF
234        // 0b1110_xxxx: start of a 3-byte UTF-8 sequence for U+0800 to U+FFFF
235        1
236    } else {
237        // 0b1111_0xxx: start of a 4-byte UTF-8 sequence for U+010000 to U+10FFFF
238        // This is exactly the range encoded as a surrogate pair in UTF-16
239        //
240        // 0b1111_1xxx: would fall here but never occurs in valid UTF-8
241        2
242    }
243}
244
245impl Utf32CodeUnits {
246    pub fn length_of(string: &str) -> Self {
247        // `std::str::Chars::count` is optimized in:
248        // https://github.com/rust-lang/rust/blob/main/library/core/src/str/count.rs
249        Self(string.chars().count())
250    }
251
252    pub fn to_utf8_code_units_in(self, string: &str) -> Utf8CodeUnits {
253        let mut current_utf32_offset = Utf32CodeUnits(0);
254        for (current_utf8_offset, byte) in string.bytes().enumerate() {
255            if (byte & 0b1100_0000) == 0b1000_0000 {
256                // UTF-8 continuation byte
257                continue;
258            }
259            if current_utf32_offset >= self {
260                return Utf8CodeUnits(current_utf8_offset);
261            }
262            current_utf32_offset.0 += 1;
263        }
264        Utf8CodeUnits(string.len())
265    }
266}
267
268#[cfg(test)]
269mod test {
270    use super::*;
271
272    #[test]
273    fn test_is_cjk() {
274        // Test characters from different CJK blocks
275        assert_eq!(is_cjk('〇'), true);
276        assert_eq!(is_cjk('㐀'), true);
277        assert_eq!(is_cjk('あ'), true);
278        assert_eq!(is_cjk('ア'), true);
279        assert_eq!(is_cjk('㆒'), true);
280        assert_eq!(is_cjk('ㆣ'), true);
281        assert_eq!(is_cjk('龥'), true);
282        assert_eq!(is_cjk('𰾑'), true);
283        assert_eq!(is_cjk('𰻝'), true);
284
285        // Test characters from outside CJK blocks
286        assert_eq!(is_cjk('a'), false);
287        assert_eq!(is_cjk('🙂'), false);
288        assert_eq!(is_cjk('©'), false);
289    }
290
291    #[test]
292    fn test_utf16_length() {
293        assert_eq!(Utf16CodeUnits::length_of(""), Utf16CodeUnits(0));
294        assert_eq!(Utf16CodeUnits::length_of("a"), Utf16CodeUnits(1));
295        assert_eq!(Utf16CodeUnits::length_of("é"), Utf16CodeUnits(1));
296        assert_eq!(Utf16CodeUnits::length_of("字"), Utf16CodeUnits(1));
297        assert_eq!(Utf16CodeUnits::length_of("\u{1F4A9}"), Utf16CodeUnits(2));
298        assert_eq!(
299            Utf16CodeUnits::length_of("\u{1F4A9}字éa"),
300            Utf16CodeUnits(5)
301        );
302    }
303
304    #[test]
305    fn test_utf16_to_utf32() {
306        let s = "aé字\u{1F4A9}";
307        assert_eq!(
308            Utf16CodeUnits(0).to_utf32_code_units_in(s),
309            Utf32CodeUnits(0)
310        );
311        assert_eq!(
312            Utf16CodeUnits(1).to_utf32_code_units_in(s),
313            Utf32CodeUnits(1)
314        );
315        assert_eq!(
316            Utf16CodeUnits(2).to_utf32_code_units_in(s),
317            Utf32CodeUnits(2)
318        );
319        assert_eq!(
320            Utf16CodeUnits(3).to_utf32_code_units_in(s),
321            Utf32CodeUnits(3)
322        );
323
324        // This 16-bit offset splits the would-be surrogate pair. We return the 32-bit position
325        // after the whole pair. Should this be an error instead?
326        assert_eq!(
327            Utf16CodeUnits(4).to_utf32_code_units_in(s),
328            Utf32CodeUnits(4)
329        );
330
331        assert_eq!(
332            Utf16CodeUnits(5).to_utf32_code_units_in(s),
333            Utf32CodeUnits(4)
334        );
335
336        // This 16-bit offset is out of bounds. We clamp to the nearest valid 32-bit offset,
337        // a.k.a the UTF-32 length. Should this be an error instead?
338        assert_eq!(
339            Utf16CodeUnits(6).to_utf32_code_units_in(s),
340            Utf32CodeUnits(4)
341        );
342        assert_eq!(
343            Utf16CodeUnits(7).to_utf32_code_units_in(s),
344            Utf32CodeUnits(4)
345        );
346    }
347}
servo_base/text.rs

servo_base/
text.rs
