servo_base/
text.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::fmt;
6use std::iter::Sum;
7use std::ops::{Add, AddAssign, Range, Sub, SubAssign};
8
9use malloc_size_of_derive::MallocSizeOf;
10
11pub use crate::unicode_block::{UnicodeBlock, UnicodeBlockMethod};
12
13pub fn is_bidi_control(c: char) -> bool {
14    matches!(c, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}' | '\u{200E}' | '\u{200F}' | '\u{061C}')
15}
16
17pub fn unicode_plane(codepoint: char) -> u32 {
18    (codepoint as u32) >> 16
19}
20
21pub fn is_cjk(codepoint: char) -> bool {
22    if let Some(
23        UnicodeBlock::CJKRadicalsSupplement |
24        UnicodeBlock::KangxiRadicals |
25        UnicodeBlock::IdeographicDescriptionCharacters |
26        UnicodeBlock::CJKSymbolsandPunctuation |
27        UnicodeBlock::Hiragana |
28        UnicodeBlock::Katakana |
29        UnicodeBlock::Bopomofo |
30        UnicodeBlock::HangulCompatibilityJamo |
31        UnicodeBlock::Kanbun |
32        UnicodeBlock::BopomofoExtended |
33        UnicodeBlock::CJKStrokes |
34        UnicodeBlock::KatakanaPhoneticExtensions |
35        UnicodeBlock::EnclosedCJKLettersandMonths |
36        UnicodeBlock::CJKCompatibility |
37        UnicodeBlock::CJKUnifiedIdeographsExtensionA |
38        UnicodeBlock::YijingHexagramSymbols |
39        UnicodeBlock::CJKUnifiedIdeographs |
40        UnicodeBlock::CJKCompatibilityIdeographs |
41        UnicodeBlock::CJKCompatibilityForms |
42        UnicodeBlock::HalfwidthandFullwidthForms,
43    ) = codepoint.block()
44    {
45        return true;
46    }
47
48    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Supplementary_Ideographic_Plane
49    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Tertiary_Ideographic_Plane
50    unicode_plane(codepoint) == 2 || unicode_plane(codepoint) == 3
51}
52
53/// Equivalent to either `Range`, `RangeTo`, `RangeFrom`, or `RangeFull`
54#[derive(Clone, Copy, Eq, PartialEq, MallocSizeOf)]
55pub struct RangeAny<T> {
56    /// `None` means zero
57    pub start: Option<T>,
58    /// `None` means the full available length
59    pub end: Option<T>,
60}
61
62impl<T: fmt::Debug> fmt::Debug for RangeAny<T> {
63    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
64        match (&self.start, &self.end) {
65            (Some(start), Some(end)) => write!(f, "{start:?}..{end:?}"),
66            (Some(start), None) => write!(f, "{start:?}.."),
67            (None, Some(end)) => write!(f, "..{end:?}"),
68            (None, None) => write!(f, ".."),
69        }
70    }
71}
72
73impl<T> RangeAny<T> {
74    /// Returns a `RangeAny` that represents the full range: both bounds unset
75    pub fn full() -> Self {
76        Self {
77            start: None,
78            end: None,
79        }
80    }
81
82    /// Apply `Option::map` to each bound of this range
83    pub fn map<U>(self, f: impl Fn(T) -> U + Copy) -> RangeAny<U> {
84        let Self { start, end } = self;
85        RangeAny {
86            start: start.map(f),
87            end: end.map(f),
88        }
89    }
90
91    /// Returns the intersection of two ranges, if it is non-empty
92    pub fn intersect(self, other: Self) -> Option<Self>
93    where
94        T: Ord,
95    {
96        // TODO: https://github.com/rust-lang/rust/issues/144273
97        // let start = a.start.reduce(b.start, std::cmp::max);
98        // let end = a.end.reduce(b.end, std::cmp::min);
99        let start = match (self.start, other.start) {
100            (None, None) => None,
101            (None, Some(b)) => Some(b),
102            (Some(a), None) => Some(a),
103            (Some(a), Some(b)) => Some(a.max(b)),
104        };
105        let end = match (self.end, other.end) {
106            (None, None) => None,
107            (None, Some(b)) => Some(b),
108            (Some(a), None) => Some(a),
109            (Some(a), Some(b)) => Some(a.min(b)),
110        };
111        if start
112            .as_ref()
113            .is_none_or(|start| end.as_ref().is_none_or(|end| start < end))
114        {
115            Some(RangeAny { start, end })
116        } else {
117            // `max()..min()` producing a "backwards" range means the intersection is empty
118            None
119        }
120    }
121}
122
123impl<T> From<Range<T>> for RangeAny<T> {
124    fn from(value: Range<T>) -> Self {
125        Self {
126            start: Some(value.start),
127            end: Some(value.end),
128        }
129    }
130}
131
132macro_rules! unicode_length_type {
133    ($( #[$doc:meta] )+ $type_name:ident) => {
134        $( #[$doc] )+
135        #[derive(Clone, Copy, Default, Eq, MallocSizeOf, Ord, PartialEq, PartialOrd)]
136        pub struct $type_name(pub usize);
137
138        impl $type_name {
139            pub fn zero() -> Self {
140                Self(0)
141            }
142
143            pub fn one() -> Self {
144                Self(1)
145            }
146
147            pub fn saturating_sub(self, value: Self) -> Self {
148                Self(self.0.saturating_sub(value.0))
149            }
150        }
151
152        impl From<u32> for $type_name {
153            fn from(value: u32) -> Self {
154                Self(value as usize)
155            }
156        }
157
158        impl From<isize> for $type_name {
159            fn from(value: isize) -> Self {
160                Self(value as usize)
161            }
162        }
163
164        impl Add for $type_name {
165            type Output = Self;
166            fn add(self, other: Self) -> Self {
167                Self(self.0 + other.0)
168            }
169        }
170
171        impl AddAssign for $type_name {
172            fn add_assign(&mut self, other: Self) {
173                *self = Self(self.0 + other.0)
174            }
175        }
176
177        impl Sub for $type_name {
178            type Output = Self;
179            fn sub(self, value: Self) -> Self {
180                Self(self.0 - value.0)
181            }
182        }
183
184        impl SubAssign for $type_name {
185            fn sub_assign(&mut self, other: Self) {
186                *self = Self(self.0 - other.0)
187            }
188        }
189
190        impl Sum for $type_name {
191            fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
192                iter.fold(Self::zero(), |a, b| Self(a.0 + b.0))
193            }
194        }
195
196        /// Use compact formatting regardless of `Formatter::alternate`
197        impl fmt::Debug for $type_name {
198            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
199                write!(f, concat!(stringify!($type_name), "({:?})"), self.0)
200            }
201        }
202    };
203}
204
205unicode_length_type! {
206    /// A length or offset counted in 8-bit code units (bytes) in an UTF-8 string.
207    /// This type is used to more reliable work with lengths or offsets in different encodings.
208    Utf8CodeUnits
209}
210
211unicode_length_type! {
212    /// A length or offset counted in 16-bit code units in an UTF-16 string.
213    /// This type is used to more reliable work with lengths or offsets in different encodings.
214    Utf16CodeUnits
215}
216
217unicode_length_type! {
218    /// A length or offset counted in 32-bit code units in UTF-32.
219    /// This is the same as counting Rust `char`s, Unicode scalar values, or Unicode code points.
220    /// This type is used to more reliable work with lengths or offsets in different encodings.
221    Utf32CodeUnits
222}
223
224unicode_length_type! {
225    /// A length or offset counted in 32-bit code units in UTF-32 or a node offset in a container
226    /// node counted in previous siblings.
227    Utf32CodeUnitsOrNodeOffset
228}
229
230impl Utf16CodeUnits {
231    pub fn length_of(string: &str) -> Self {
232        Self(string.bytes().map(len_utf16_for_utf8_byte).sum())
233
234        // TODO: after upgrading to a Rust version (1.99?) that includes that PR,
235        // replace the above with:
236
237        // // `EncodeUtf16::count` is optimized in https://github.com/rust-lang/rust/pull/159467
238        // Self(string.encode_utf16().count())
239    }
240
241    pub fn to_utf32_code_units_in(self, string: &str) -> Utf32CodeUnits {
242        let mut current_utf16_offset = Utf16CodeUnits(0);
243        let mut current_utf32_offset = Utf32CodeUnits(0);
244        for utf8_byte in string.bytes() {
245            if current_utf16_offset >= self {
246                break;
247            }
248            increment_offsets_for_utf8_byte(
249                utf8_byte,
250                &mut current_utf16_offset,
251                &mut current_utf32_offset,
252            );
253        }
254        current_utf32_offset
255    }
256}
257
258fn len_utf16_for_utf8_byte(byte: u8) -> usize {
259    if byte < 0b1000_0000 {
260        // 0b0xxx_xxxx: ASCII-compatible U+0000 to U+007F
261        1
262    } else if byte < 0b1100_0000 {
263        // 0b10xx_xxxx: UTF-8 continuation byte, already accounted for by its non-continuation byte
264        0
265    } else if byte < 0b1111_0000 {
266        // 0b110x_xxxx: start of a 2-byte UTF-8 sequence for U+0080 to U+07FF
267        // 0b1110_xxxx: start of a 3-byte UTF-8 sequence for U+0800 to U+FFFF
268        1
269    } else {
270        // 0b1111_0xxx: start of a 4-byte UTF-8 sequence for U+010000 to U+10FFFF
271        // This is exactly the range encoded as a surrogate pair in UTF-16
272        //
273        // 0b1111_1xxx: would fall here but never occurs in valid UTF-8
274        2
275    }
276}
277
278fn increment_offsets_for_utf8_byte(
279    utf8_byte: u8,
280    utf16_offset: &mut Utf16CodeUnits,
281    utf32_offset: &mut Utf32CodeUnits,
282) {
283    let len_utf16 = len_utf16_for_utf8_byte(utf8_byte);
284    utf16_offset.0 += len_utf16;
285    // `len_utf16 != 0` means this byte is the first byte of the UTF-8 byte sequence
286    // for one `char` /  UTF-32 code unit
287    utf32_offset.0 += (len_utf16 != 0) as usize;
288}
289
290impl Utf32CodeUnits {
291    pub fn length_of(string: &str) -> Self {
292        // `std::str::Chars::count` is optimized in:
293        // https://github.com/rust-lang/rust/blob/main/library/core/src/str/count.rs
294        Self(string.chars().count())
295    }
296
297    pub fn to_utf8_code_units_in(self, string: &str) -> Utf8CodeUnits {
298        let mut current_utf32_offset = Utf32CodeUnits(0);
299        for (current_utf8_offset, utf8_byte) in string.bytes().enumerate() {
300            if (utf8_byte & 0b1100_0000) == 0b1000_0000 {
301                // UTF-8 continuation byte
302                continue;
303            }
304            if current_utf32_offset >= self {
305                return Utf8CodeUnits(current_utf8_offset);
306            }
307            current_utf32_offset.0 += 1;
308        }
309        Utf8CodeUnits(string.len())
310    }
311
312    pub fn to_utf16_code_units_in(self, string: &str) -> Utf16CodeUnits {
313        let mut current_utf32_offset = Utf32CodeUnits(0);
314        let mut current_utf16_offset = Utf16CodeUnits(0);
315        for utf8_byte in string.bytes() {
316            if current_utf32_offset >= self {
317                break;
318            }
319            increment_offsets_for_utf8_byte(
320                utf8_byte,
321                &mut current_utf16_offset,
322                &mut current_utf32_offset,
323            );
324        }
325        current_utf16_offset
326    }
327}
328
329impl Utf32CodeUnitsOrNodeOffset {
330    pub fn to_utf16_code_units_in(self, string: &str) -> Utf16CodeUnits {
331        Utf32CodeUnits(self.0).to_utf16_code_units_in(string)
332    }
333}
334
335#[cfg(test)]
336mod test {
337    use super::*;
338
339    #[test]
340    fn test_is_cjk() {
341        // Test characters from different CJK blocks
342        assert_eq!(is_cjk('〇'), true);
343        assert_eq!(is_cjk('㐀'), true);
344        assert_eq!(is_cjk('あ'), true);
345        assert_eq!(is_cjk('ア'), true);
346        assert_eq!(is_cjk('㆒'), true);
347        assert_eq!(is_cjk('ㆣ'), true);
348        assert_eq!(is_cjk('龥'), true);
349        assert_eq!(is_cjk('𰾑'), true);
350        assert_eq!(is_cjk('𰻝'), true);
351
352        // Test characters from outside CJK blocks
353        assert_eq!(is_cjk('a'), false);
354        assert_eq!(is_cjk('🙂'), false);
355        assert_eq!(is_cjk('©'), false);
356    }
357
358    #[test]
359    fn test_utf16_length() {
360        assert_eq!(Utf16CodeUnits::length_of(""), Utf16CodeUnits(0));
361        assert_eq!(Utf16CodeUnits::length_of("a"), Utf16CodeUnits(1));
362        assert_eq!(Utf16CodeUnits::length_of("é"), Utf16CodeUnits(1));
363        assert_eq!(Utf16CodeUnits::length_of("字"), Utf16CodeUnits(1));
364        assert_eq!(Utf16CodeUnits::length_of("\u{1F4A9}"), Utf16CodeUnits(2));
365        assert_eq!(
366            Utf16CodeUnits::length_of("\u{1F4A9}字éa"),
367            Utf16CodeUnits(5)
368        );
369    }
370
371    #[test]
372    fn test_utf16_to_utf32() {
373        let s = "aé字\u{1F4A9}";
374        assert_eq!(
375            Utf16CodeUnits(0).to_utf32_code_units_in(s),
376            Utf32CodeUnits(0)
377        );
378        assert_eq!(
379            Utf16CodeUnits(1).to_utf32_code_units_in(s),
380            Utf32CodeUnits(1)
381        );
382        assert_eq!(
383            Utf16CodeUnits(2).to_utf32_code_units_in(s),
384            Utf32CodeUnits(2)
385        );
386        assert_eq!(
387            Utf16CodeUnits(3).to_utf32_code_units_in(s),
388            Utf32CodeUnits(3)
389        );
390
391        // This 16-bit offset splits the would-be surrogate pair. We return the 32-bit position
392        // after the whole pair. Should this be an error instead?
393        assert_eq!(
394            Utf16CodeUnits(4).to_utf32_code_units_in(s),
395            Utf32CodeUnits(4)
396        );
397
398        assert_eq!(
399            Utf16CodeUnits(5).to_utf32_code_units_in(s),
400            Utf32CodeUnits(4)
401        );
402
403        // This 16-bit offset is out of bounds. We clamp to the nearest valid 32-bit offset,
404        // a.k.a the UTF-32 length. Should this be an error instead?
405        assert_eq!(
406            Utf16CodeUnits(6).to_utf32_code_units_in(s),
407            Utf32CodeUnits(4)
408        );
409        assert_eq!(
410            Utf16CodeUnits(7).to_utf32_code_units_in(s),
411            Utf32CodeUnits(4)
412        );
413    }
414
415    #[test]
416    fn test_utf32_to_utf16() {
417        let string = "aé字\u{1F4A9}";
418        assert_eq!(
419            Utf32CodeUnits(0).to_utf16_code_units_in(string),
420            Utf16CodeUnits(0),
421        );
422        assert_eq!(
423            Utf32CodeUnits(1).to_utf16_code_units_in(string),
424            Utf16CodeUnits(1),
425        );
426        assert_eq!(
427            Utf32CodeUnits(2).to_utf16_code_units_in(string),
428            Utf16CodeUnits(2),
429        );
430        assert_eq!(
431            Utf32CodeUnits(3).to_utf16_code_units_in(string),
432            Utf16CodeUnits(3),
433        );
434
435        assert_eq!(
436            Utf32CodeUnits(4).to_utf16_code_units_in(string),
437            Utf16CodeUnits(5),
438        );
439
440        // This 32-bit offset is out of bounds. We clamp to the nearest valid 16-bit offset,
441        // a.k.a the UTF-16 length. Should this be an error instead?
442        assert_eq!(
443            Utf32CodeUnits(6).to_utf16_code_units_in(string),
444            Utf16CodeUnits(5),
445        );
446        assert_eq!(
447            Utf32CodeUnits(1000).to_utf16_code_units_in(string),
448            Utf16CodeUnits(5),
449        );
450    }
451}
servo_base/text.rs

servo_base/
text.rs
