mozjs_utf16_iter/
lib.rs

1// Copyright Mozilla Foundation
2//
3// Licensed under the Apache License (Version 2.0), or the MIT license,
4// (the "Licenses") at your option. You may not use this file except in
5// compliance with one of the Licenses. You may obtain copies of the
6// Licenses at:
7//
8//    https://www.apache.org/licenses/LICENSE-2.0
9//    https://opensource.org/licenses/MIT
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the Licenses is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the Licenses for the specific language governing permissions and
15// limitations under the Licenses.
16
17#![no_std]
18
19//! Provides iteration by `char` over `&[u16]` containing potentially-invalid
20//! UTF-16 such that errors are replaced with the REPLACEMENT CHARACTER.
21//!
22//! The trait `Utf16CharsEx` provides the convenience method `chars()` on
23//! byte slices themselves instead of having to use the more verbose
24//! `Utf16Chars::new(slice)`.
25
26#[cfg(feature = "icu_collections")]
27mod cptrie;
28#[cfg(feature = "icu_collections")]
29mod cptrie_indices;
30mod indices;
31mod report;
32
33#[cfg(feature = "icu_collections")]
34pub use crate::cptrie::Utf16CharsWithTrie;
35#[cfg(feature = "icu_collections")]
36pub use crate::cptrie::Utf16CharsWithTrieEx;
37#[cfg(feature = "icu_collections")]
38pub use crate::cptrie_indices::Utf16CharIndicesWithTrie;
39pub use crate::indices::Utf16CharIndices;
40pub use crate::report::ErrorReportingUtf16Chars;
41pub use crate::report::Utf16CharsError;
42use core::iter::FusedIterator;
43
44#[inline(always)]
45pub(crate) fn in_inclusive_range16(i: u16, start: u16, end: u16) -> bool {
46    i.wrapping_sub(start) <= (end - start)
47}
48
49/// Iterator by `char` over `&[u16]` that contains
50/// potentially-invalid UTF-16. See the crate documentation.
51#[derive(Debug, Clone)]
52pub struct Utf16Chars<'a> {
53    remaining: &'a [u16],
54}
55
56impl<'a> Utf16Chars<'a> {
57    #[inline(always)]
58    /// Creates the iterator from a `u16` slice.
59    pub fn new(code_units: &'a [u16]) -> Self {
60        Utf16Chars::<'a> {
61            remaining: code_units,
62        }
63    }
64
65    /// Views the current remaining data in the iterator as a subslice
66    /// of the original slice.
67    #[inline(always)]
68    pub fn as_slice(&self) -> &'a [u16] {
69        self.remaining
70    }
71
72    #[inline(never)]
73    fn surrogate_next(&mut self, surrogate_base: u16, first: u16) -> char {
74        if surrogate_base <= (0xDBFF - 0xD800) {
75            if let Some((&low, tail_tail)) = self.remaining.split_first() {
76                if in_inclusive_range16(low, 0xDC00, 0xDFFF) {
77                    self.remaining = tail_tail;
78                    return unsafe {
79                        char::from_u32_unchecked(
80                            (u32::from(first) << 10) + u32::from(low)
81                                - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32),
82                        )
83                    };
84                }
85            }
86        }
87        '\u{FFFD}'
88    }
89
90    #[inline(never)]
91    fn surrogate_next_back(&mut self, last: u16) -> char {
92        if in_inclusive_range16(last, 0xDC00, 0xDFFF) {
93            if let Some((&high, head_head)) = self.remaining.split_last() {
94                if in_inclusive_range16(high, 0xD800, 0xDBFF) {
95                    self.remaining = head_head;
96                    return unsafe {
97                        char::from_u32_unchecked(
98                            (u32::from(high) << 10) + u32::from(last)
99                                - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32),
100                        )
101                    };
102                }
103            }
104        }
105        '\u{FFFD}'
106    }
107}
108
109impl<'a> Iterator for Utf16Chars<'a> {
110    type Item = char;
111
112    #[inline(always)]
113    fn next(&mut self) -> Option<char> {
114        // It might be OK to delegate to `ErrorReportingUtf16Chars`, but since
115        // the methods are rather small, copypaste is probably clearer. Also,
116        // copypaste would _not_ be equivalent if any part of this was delegated
117        // to an `inline(never)` helper. However, previous experimentation indicated
118        // that such a helper didn't help performance here.
119        let (&first, tail) = self.remaining.split_first()?;
120        self.remaining = tail;
121        let surrogate_base = first.wrapping_sub(0xD800);
122        if surrogate_base > (0xDFFF - 0xD800) {
123            return Some(unsafe { char::from_u32_unchecked(u32::from(first)) });
124        }
125        Some(self.surrogate_next(surrogate_base, first))
126    }
127}
128
129impl<'a> DoubleEndedIterator for Utf16Chars<'a> {
130    #[inline(always)]
131    fn next_back(&mut self) -> Option<char> {
132        let (&last, head) = self.remaining.split_last()?;
133        self.remaining = head;
134        if !in_inclusive_range16(last, 0xD800, 0xDFFF) {
135            return Some(unsafe { char::from_u32_unchecked(u32::from(last)) });
136        }
137        Some(self.surrogate_next_back(last))
138    }
139}
140
141impl FusedIterator for Utf16Chars<'_> {}
142
143/// Convenience trait that adds `chars()` and `char_indices()` methods
144/// similar to the ones on string slices to `u16` slices.
145pub trait Utf16CharsEx {
146    /// Convenience method for creating an UTF-16 iterator
147    /// for the slice.
148    fn chars(&self) -> Utf16Chars<'_>;
149
150    /// Convenience method for creating a code unit index and
151    /// UTF-16 iterator for the slice.
152    fn char_indices(&self) -> Utf16CharIndices<'_>;
153}
154
155impl Utf16CharsEx for [u16] {
156    /// Convenience method for creating an UTF-16 iterator
157    /// for the slice.
158    #[inline]
159    fn chars(&self) -> Utf16Chars<'_> {
160        Utf16Chars::new(self)
161    }
162    /// Convenience method for creating a code unit index and
163    /// UTF-16 iterator for the slice.
164    #[inline]
165    fn char_indices(&self) -> Utf16CharIndices<'_> {
166        Utf16CharIndices::new(self)
167    }
168}
169
170#[cfg(test)]
171mod tests {
172    use crate::Utf16CharsEx;
173
174    #[test]
175    fn test_boundaries() {
176        assert!([0xD7FFu16]
177            .as_slice()
178            .chars()
179            .eq(core::iter::once('\u{D7FF}')));
180        assert!([0xE000u16]
181            .as_slice()
182            .chars()
183            .eq(core::iter::once('\u{E000}')));
184        assert!([0xD800u16]
185            .as_slice()
186            .chars()
187            .eq(core::iter::once('\u{FFFD}')));
188        assert!([0xDFFFu16]
189            .as_slice()
190            .chars()
191            .eq(core::iter::once('\u{FFFD}')));
192    }
193
194    #[test]
195    fn test_unpaired() {
196        assert!([0xD800u16, 0x0061u16]
197            .as_slice()
198            .chars()
199            .eq([0xFFFDu16, 0x0061u16].as_slice().chars()));
200        assert!([0xDFFFu16, 0x0061u16]
201            .as_slice()
202            .chars()
203            .eq([0xFFFDu16, 0x0061u16].as_slice().chars()));
204    }
205
206    #[test]
207    fn test_unpaired_rev() {
208        assert!([0xD800u16, 0x0061u16]
209            .as_slice()
210            .chars()
211            .rev()
212            .eq([0xFFFDu16, 0x0061u16].as_slice().chars().rev()));
213        assert!([0xDFFFu16, 0x0061u16]
214            .as_slice()
215            .chars()
216            .rev()
217            .eq([0xFFFDu16, 0x0061u16].as_slice().chars().rev()));
218    }
219
220    #[test]
221    fn test_paired() {
222        assert!([0xD83Eu16, 0xDD73u16]
223            .as_slice()
224            .chars()
225            .eq(core::iter::once('🥳')));
226    }
227
228    #[test]
229    fn test_paired_rev() {
230        assert!([0xD83Eu16, 0xDD73u16]
231            .as_slice()
232            .chars()
233            .rev()
234            .eq(core::iter::once('🥳')));
235    }
236
237    #[test]
238    fn test_as_slice() {
239        let mut iter = [0x0061u16, 0x0062u16].as_slice().chars();
240        let at_start = iter.as_slice();
241        assert_eq!(iter.next(), Some('a'));
242        let in_middle = iter.as_slice();
243        assert_eq!(iter.next(), Some('b'));
244        let at_end = iter.as_slice();
245        assert_eq!(at_start.len(), 2);
246        assert_eq!(in_middle.len(), 1);
247        assert_eq!(at_end.len(), 0);
248        assert_eq!(at_start[0], 0x0061u16);
249        assert_eq!(at_start[1], 0x0062u16);
250        assert_eq!(in_middle[0], 0x0062u16);
251    }
252}
mozjs_utf16_iter/lib.rs

mozjs_utf16_iter/
lib.rs
