mozjs_utf16_iter/
cptrie.rs

1// Copyright Mozilla Foundation
2//
3// Licensed under the Apache License (Version 2.0), or the MIT license,
4// (the "Licenses") at your option. You may not use this file except in
5// compliance with one of the Licenses. You may obtain copies of the
6// Licenses at:
7//
8//    https://www.apache.org/licenses/LICENSE-2.0
9//    https://opensource.org/licenses/MIT
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the Licenses is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the Licenses for the specific language governing permissions and
15// limitations under the Licenses.
16
17use crate::Utf16CharIndicesWithTrie;
18use core::iter::FusedIterator;
19use core::marker::PhantomData;
20use icu_collections::codepointtrie::AbstractCodePointTrie;
21use icu_collections::codepointtrie::TrieValue;
22use icu_collections::codepointtrie::WithTrie;
23
24/// Iterator by `char` and `icu_collections::codepointtrie::TrieValue`
25/// over `&[u16]` that contains potentially-invalid UTF-16. See the
26/// crate documentation.
27#[derive(Debug)]
28pub struct Utf16CharsWithTrie<'slice, 'trie, T, V>
29where
30    V: TrieValue,
31    T: AbstractCodePointTrie<'trie, V>,
32{
33    remaining: &'slice [u16],
34    trie: &'trie T,
35    phantom: PhantomData<V>,
36}
37
38impl<'slice, 'trie, T, V> Utf16CharsWithTrie<'slice, 'trie, T, V>
39where
40    V: TrieValue,
41    T: AbstractCodePointTrie<'trie, V>,
42{
43    #[inline(always)]
44    /// Creates the iterator from a `u16` slice.
45    pub fn new(code_units: &'slice [u16], trie: &'trie T) -> Self {
46        Self {
47            remaining: code_units,
48            trie,
49            phantom: PhantomData,
50        }
51    }
52
53    /// Views the current remaining data in the iterator as a subslice
54    /// of the original slice.
55    #[inline(always)]
56    pub fn as_slice(&self) -> &'slice [u16] {
57        self.remaining
58    }
59
60    #[inline(never)]
61    fn surrogate_next(&mut self, surrogate_base: u16, first: u16) -> (char, V) {
62        if surrogate_base <= (0xDBFF - 0xD800) {
63            if let Some((&low, tail_tail)) = self.remaining.split_first() {
64                if crate::in_inclusive_range16(low, 0xDC00, 0xDFFF) {
65                    self.remaining = tail_tail;
66                    let code_point = (u32::from(first) << 10) + u32::from(low)
67                        - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32);
68                    return unsafe {
69                        (
70                            char::from_u32_unchecked(code_point),
71                            self.trie.supplementary(code_point),
72                        )
73                    };
74                }
75            }
76        }
77        ('\u{FFFD}', self.trie.bmp(0xFFFD))
78    }
79
80    #[inline(never)]
81    fn surrogate_next_back(&mut self, last: u16) -> (char, V) {
82        if crate::in_inclusive_range16(last, 0xDC00, 0xDFFF) {
83            if let Some((&high, head_head)) = self.remaining.split_last() {
84                if crate::in_inclusive_range16(high, 0xD800, 0xDBFF) {
85                    self.remaining = head_head;
86                    let code_point = (u32::from(high) << 10) + u32::from(last)
87                        - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32);
88                    return unsafe {
89                        (
90                            char::from_u32_unchecked(code_point),
91                            self.trie.supplementary(code_point),
92                        )
93                    };
94                }
95            }
96        }
97        ('\u{FFFD}', self.trie.bmp(0xFFFD))
98    }
99}
100
101impl<'slice, 'trie, T, V> Clone for Utf16CharsWithTrie<'slice, 'trie, T, V>
102where
103    V: TrieValue,
104    T: AbstractCodePointTrie<'trie, V>,
105{
106    #[inline]
107    fn clone(&self) -> Self {
108        Self {
109            remaining: self.remaining,
110            trie: self.trie,
111            phantom: PhantomData,
112        }
113    }
114}
115
116impl<'slice, 'trie, T, V> WithTrie<'trie, T, V> for Utf16CharsWithTrie<'slice, 'trie, T, V>
117where
118    V: TrieValue,
119    T: AbstractCodePointTrie<'trie, V>,
120{
121    #[inline]
122    fn trie(&self) -> &'trie T {
123        self.trie
124    }
125}
126
127impl<'slice, 'trie, T, V> Iterator for Utf16CharsWithTrie<'slice, 'trie, T, V>
128where
129    V: TrieValue,
130    T: AbstractCodePointTrie<'trie, V>,
131{
132    type Item = (char, V);
133
134    #[inline(always)]
135    fn next(&mut self) -> Option<Self::Item> {
136        let (&first, tail) = self.remaining.split_first()?;
137        self.remaining = tail;
138        let surrogate_base = first.wrapping_sub(0xD800);
139        if surrogate_base > (0xDFFF - 0xD800) {
140            return Some((
141                unsafe { char::from_u32_unchecked(u32::from(first)) },
142                self.trie.bmp(first),
143            ));
144        }
145        Some(self.surrogate_next(surrogate_base, first))
146    }
147}
148
149impl<'slice, 'trie, T, V> DoubleEndedIterator for Utf16CharsWithTrie<'slice, 'trie, T, V>
150where
151    V: TrieValue,
152    T: AbstractCodePointTrie<'trie, V>,
153{
154    #[inline(always)]
155    fn next_back(&mut self) -> Option<Self::Item> {
156        let (&last, head) = self.remaining.split_last()?;
157        self.remaining = head;
158        if !crate::in_inclusive_range16(last, 0xD800, 0xDFFF) {
159            return Some((
160                unsafe { char::from_u32_unchecked(u32::from(last)) },
161                self.trie.bmp(last),
162            ));
163        }
164        Some(self.surrogate_next_back(last))
165    }
166}
167
168impl<'slice, 'trie, T, V> FusedIterator for Utf16CharsWithTrie<'slice, 'trie, T, V>
169where
170    V: TrieValue,
171    T: AbstractCodePointTrie<'trie, V>,
172{
173}
174
175/// Convenience trait that adds `chars_with_trie()` and `char_indices_with_trie()` methods
176/// similar to the ones `icu_collections::codepointtrie::CharsWithTrieEx` adds to string
177/// slices to `u16` slices.
178pub trait Utf16CharsWithTrieEx<'slice, 'trie, T, V>
179where
180    V: TrieValue,
181    T: AbstractCodePointTrie<'trie, V>,
182{
183    /// Convenience method for creating an UTF-16 iterator
184    /// with trie values for the slice.
185    fn chars_with_trie(&'slice self, trie: &'trie T) -> Utf16CharsWithTrie<'slice, 'trie, T, V>;
186    /// Convenience method for creating a code unit index and
187    /// UTF-16 iterator with trie values for the slice.
188    fn char_indices_with_trie(
189        &'slice self,
190        trie: &'trie T,
191    ) -> Utf16CharIndicesWithTrie<'slice, 'trie, T, V>;
192}
193
194impl<'slice, 'trie, T, V> Utf16CharsWithTrieEx<'slice, 'trie, T, V> for [u16]
195where
196    V: TrieValue,
197    T: AbstractCodePointTrie<'trie, V>,
198{
199    /// Convenience method for creating an UTF-16 iterator
200    /// with trie values for the slice.
201    #[inline]
202    fn chars_with_trie(&'slice self, trie: &'trie T) -> Utf16CharsWithTrie<'slice, 'trie, T, V> {
203        Utf16CharsWithTrie::new(self, trie)
204    }
205
206    /// Convenience method for creating a code unit index and
207    /// UTF-16 iterator with trie values for the slice.
208    #[inline]
209    fn char_indices_with_trie(
210        &'slice self,
211        trie: &'trie T,
212    ) -> Utf16CharIndicesWithTrie<'slice, 'trie, T, V> {
213        Utf16CharIndicesWithTrie::new(self, trie)
214    }
215}
216
217#[cfg(test)]
218mod tests {
219    use super::*;
220
221    #[test]
222    fn test_forward() {
223        let trie = icu_collections::codepointtrie::planes::get_planes_trie();
224        let s = &[0xD83Eu16, 0xDD73u16, 0xD83Eu16, 0x00E4u16, 0xD83Eu16];
225        let mut iter = s.chars_with_trie(&trie);
226        assert_eq!(iter.next(), Some(('🥳', 1)));
227        assert_eq!(iter.next(), Some(('\u{FFFD}', 0)));
228        assert_eq!(iter.next(), Some(('\u{00E4}', 0)));
229        assert_eq!(iter.next(), Some(('\u{FFFD}', 0)));
230        assert_eq!(iter.next(), None);
231    }
232
233    #[test]
234    fn test_backwards() {
235        let trie = icu_collections::codepointtrie::planes::get_planes_trie();
236        let s = &[0xD83Eu16, 0xDD73u16, 0xD83Eu16, 0x00E4u16, 0xD83Eu16];
237        let mut iter = s.chars_with_trie(&trie);
238        assert_eq!(iter.next_back(), Some(('\u{FFFD}', 0)));
239        assert_eq!(iter.next_back(), Some(('\u{00E4}', 0)));
240        assert_eq!(iter.next_back(), Some(('\u{FFFD}', 0)));
241        assert_eq!(iter.next_back(), Some(('🥳', 1)));
242        assert_eq!(iter.next_back(), None);
243    }
244}
mozjs_utf16_iter/cptrie.rs

mozjs_utf16_iter/
cptrie.rs
