1use crate::in_inclusive_range8;
18use crate::Utf8CharIndicesWithTrie;
19use crate::UTF8_DATA;
20use core::iter::FusedIterator;
21use core::marker::PhantomData;
22use icu_collections::codepointtrie::AbstractCodePointTrie;
23use icu_collections::codepointtrie::TrieValue;
24use icu_collections::codepointtrie::WithTrie;
25
26#[derive(Debug)]
30pub struct Utf8CharsWithTrie<'slice, 'trie, T, V>
31where
32 V: TrieValue,
33 T: AbstractCodePointTrie<'trie, V>,
34{
35 remaining: &'slice [u8],
36 trie: &'trie T,
37 phantom: PhantomData<V>,
38}
39
40impl<'slice, 'trie, T, V> Utf8CharsWithTrie<'slice, 'trie, T, V>
41where
42 V: TrieValue,
43 T: AbstractCodePointTrie<'trie, V>,
44{
45 #[inline(always)]
46 pub fn new(bytes: &'slice [u8], trie: &'trie T) -> Self {
48 Self {
49 remaining: bytes,
50 trie,
51 phantom: PhantomData,
52 }
53 }
54
55 #[inline(always)]
58 pub fn as_slice(&self) -> &'slice [u8] {
59 self.remaining
60 }
61
62 #[inline(never)]
63 fn next_fallback(&mut self) -> Option<(char, V)> {
64 if self.remaining.is_empty() {
65 return None;
66 }
67 let first = self.remaining[0];
68 if first < 0x80 {
69 self.remaining = &self.remaining[1..];
70 return Some((char::from(first), unsafe { self.trie.ascii(first) }));
72 }
73 if !in_inclusive_range8(first, 0xC2, 0xF4) || self.remaining.len() == 1 {
74 self.remaining = &self.remaining[1..];
75 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
76 }
77 let second = self.remaining[1];
78 let (lower_bound, upper_bound) = match first {
79 0xE0 => (0xA0, 0xBF),
80 0xED => (0x80, 0x9F),
81 0xF0 => (0x90, 0xBF),
82 0xF4 => (0x80, 0x8F),
83 _ => (0x80, 0xBF),
84 };
85 if !in_inclusive_range8(second, lower_bound, upper_bound) {
86 self.remaining = &self.remaining[1..];
87 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
88 }
89 if first < 0xE0 {
90 self.remaining = &self.remaining[2..];
91 let high_five = u32::from(first) & 0b11_111;
92 let low_six = u32::from(second) & 0b111_111;
93 let v = unsafe { self.trie.utf8_two_byte(high_five, low_six) };
96 let point = (high_five << 6) | low_six;
97 return Some((unsafe { char::from_u32_unchecked(point) }, v));
102 }
103 if self.remaining.len() == 2 {
104 self.remaining = &self.remaining[2..];
105 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
106 }
107 let third = self.remaining[2];
108 if !in_inclusive_range8(third, 0x80, 0xBF) {
109 self.remaining = &self.remaining[2..];
110 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
111 }
112 if first < 0xF0 {
113 self.remaining = &self.remaining[3..];
114 let high_ten = ((u32::from(first) & 0b1111) << 6) | (u32::from(second) & 0b111_111);
115 let low_six = u32::from(third) & 0b111_111;
116 let v = unsafe { self.trie.utf8_three_byte(high_ten, low_six) };
119 let point = (high_ten << 6) | low_six;
120 return Some((unsafe { char::from_u32_unchecked(point) }, v));
125 }
126 self.remaining = &self.remaining[3..];
130 Some(('\u{FFFD}', self.trie.bmp(0xFFFD)))
131 }
132}
133
134impl<'slice, 'trie, T, V> Clone for Utf8CharsWithTrie<'slice, 'trie, T, V>
135where
136 V: TrieValue,
137 T: AbstractCodePointTrie<'trie, V>,
138{
139 #[inline]
140 fn clone(&self) -> Self {
141 Self {
142 remaining: self.remaining,
143 trie: self.trie,
144 phantom: PhantomData,
145 }
146 }
147}
148
149impl<'slice, 'trie, T, V> WithTrie<'trie, T, V> for Utf8CharsWithTrie<'slice, 'trie, T, V>
150where
151 V: TrieValue,
152 T: AbstractCodePointTrie<'trie, V>,
153{
154 #[inline]
155 fn trie(&self) -> &'trie T {
156 self.trie
157 }
158}
159
160impl<'slice, 'trie, T, V> Iterator for Utf8CharsWithTrie<'slice, 'trie, T, V>
161where
162 V: TrieValue,
163 T: AbstractCodePointTrie<'trie, V>,
164{
165 type Item = (char, V);
166
167 #[inline]
168 fn next(&mut self) -> Option<Self::Item> {
169 #[allow(clippy::never_loop)]
171 loop {
172 if self.remaining.len() < 4 {
173 break;
174 }
175 let first = self.remaining[0];
176 if first < 0x80 {
177 self.remaining = &self.remaining[1..];
178 return Some((char::from(first), unsafe { self.trie.ascii(first) }));
180 }
181 let second = self.remaining[1];
182 if in_inclusive_range8(first, 0xC2, 0xDF) {
183 if !in_inclusive_range8(second, 0x80, 0xBF) {
184 break;
185 }
186 self.remaining = &self.remaining[2..];
187 let high_five = u32::from(first) & 0b11_111;
188 let low_six = u32::from(second) & 0b111_111;
189 let v = unsafe { self.trie.utf8_two_byte(high_five, low_six) };
192 let point = (high_five << 6) | low_six;
193 return Some((unsafe { char::from_u32_unchecked(point) }, v));
198 }
199 let third = self.remaining[2];
202 if first < 0xF0 {
203 if ((UTF8_DATA.table[usize::from(second)]
204 & UTF8_DATA.table[usize::from(first) + 0x80])
205 | (third >> 6))
206 != 2
207 {
208 break;
209 }
210 self.remaining = &self.remaining[3..];
211 let high_ten = ((u32::from(first) & 0b1111) << 6) | (u32::from(second) & 0b111_111);
212 let low_six = u32::from(third) & 0b111_111;
213 let v = unsafe { self.trie.utf8_three_byte(high_ten, low_six) };
216 let point = (high_ten << 6) | low_six;
217 return Some((unsafe { char::from_u32_unchecked(point) }, v));
222 }
223 let fourth = self.remaining[3];
224 if (u16::from(
225 UTF8_DATA.table[usize::from(second)] & UTF8_DATA.table[usize::from(first) + 0x80],
226 ) | u16::from(third >> 6)
227 | (u16::from(fourth & 0xC0) << 2))
228 != 0x202
229 {
230 break;
231 }
232 let point = ((u32::from(first) & 0x7) << 18)
233 | ((u32::from(second) & 0x3F) << 12)
234 | ((u32::from(third) & 0x3F) << 6)
235 | (u32::from(fourth) & 0x3F);
236 self.remaining = &self.remaining[4..];
237 return Some((
240 unsafe { char::from_u32_unchecked(point) },
241 self.trie.supplementary(point),
242 ));
243 }
244 self.next_fallback()
245 }
246}
247
248impl<'slice, 'trie, T, V> DoubleEndedIterator for Utf8CharsWithTrie<'slice, 'trie, T, V>
249where
250 V: TrieValue,
251 T: AbstractCodePointTrie<'trie, V>,
252{
253 #[inline]
254 fn next_back(&mut self) -> Option<(char, V)> {
255 if self.remaining.is_empty() {
256 return None;
257 }
258 let mut attempt = 1;
259 for b in self.remaining.iter().rev() {
260 if b & 0xC0 != 0x80 {
261 let (head, tail) = self.remaining.split_at(self.remaining.len() - attempt);
262 let mut inner = Utf8CharsWithTrie::new(tail, self.trie);
263 let candidate = inner.next();
264 if inner.as_slice().is_empty() {
265 self.remaining = head;
266 return candidate;
267 }
268 break;
269 }
270 if attempt == 4 {
271 break;
272 }
273 attempt += 1;
274 }
275
276 self.remaining = &self.remaining[..self.remaining.len() - 1];
277 Some(('\u{FFFD}', self.trie.bmp(0xFFFD)))
278 }
279}
280
281impl<'slice, 'trie, T, V> FusedIterator for Utf8CharsWithTrie<'slice, 'trie, T, V>
282where
283 V: TrieValue,
284 T: AbstractCodePointTrie<'trie, V>,
285{
286}
287
288pub trait Utf8CharsWithTrieEx<'slice, 'trie, T, V>
292where
293 V: TrieValue,
294 T: AbstractCodePointTrie<'trie, V>,
295{
296 fn chars_with_trie(&'slice self, trie: &'trie T) -> Utf8CharsWithTrie<'slice, 'trie, T, V>;
299 fn char_indices_with_trie(
302 &'slice self,
303 trie: &'trie T,
304 ) -> Utf8CharIndicesWithTrie<'slice, 'trie, T, V>;
305}
306
307impl<'slice, 'trie, T, V> Utf8CharsWithTrieEx<'slice, 'trie, T, V> for [u8]
308where
309 V: TrieValue,
310 T: AbstractCodePointTrie<'trie, V>,
311{
312 #[inline]
315 fn chars_with_trie(&'slice self, trie: &'trie T) -> Utf8CharsWithTrie<'slice, 'trie, T, V> {
316 Utf8CharsWithTrie::new(self, trie)
317 }
318
319 #[inline]
322 fn char_indices_with_trie(
323 &'slice self,
324 trie: &'trie T,
325 ) -> Utf8CharIndicesWithTrie<'slice, 'trie, T, V> {
326 Utf8CharIndicesWithTrie::new(self, trie)
327 }
328}
329
330#[derive(Debug)]
337pub struct Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>
338where
339 V: TrieValue + Default,
340 T: AbstractCodePointTrie<'trie, V>,
341{
342 remaining: &'slice [u8],
343 trie: &'trie T,
344 phantom: PhantomData<V>,
345}
346
347impl<'slice, 'trie, T, V> Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>
348where
349 V: TrieValue + Default,
350 T: AbstractCodePointTrie<'trie, V>,
351{
352 #[inline(always)]
353 pub fn new(bytes: &'slice [u8], trie: &'trie T) -> Self {
355 Self {
356 remaining: bytes,
357 trie,
358 phantom: PhantomData,
359 }
360 }
361
362 #[inline(always)]
365 pub fn as_slice(&self) -> &'slice [u8] {
366 self.remaining
367 }
368
369 #[inline(never)]
370 fn next_fallback(&mut self) -> Option<(char, V)> {
371 if self.remaining.is_empty() {
372 return None;
373 }
374 let first = self.remaining[0];
375 if first < 0x80 {
376 self.remaining = &self.remaining[1..];
377 return Some((char::from(first), V::default()));
378 }
379 if !in_inclusive_range8(first, 0xC2, 0xF4) || self.remaining.len() == 1 {
380 self.remaining = &self.remaining[1..];
381 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
382 }
383 let second = self.remaining[1];
384 let (lower_bound, upper_bound) = match first {
385 0xE0 => (0xA0, 0xBF),
386 0xED => (0x80, 0x9F),
387 0xF0 => (0x90, 0xBF),
388 0xF4 => (0x80, 0x8F),
389 _ => (0x80, 0xBF),
390 };
391 if !in_inclusive_range8(second, lower_bound, upper_bound) {
392 self.remaining = &self.remaining[1..];
393 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
394 }
395 if first < 0xE0 {
396 self.remaining = &self.remaining[2..];
397 let high_five = u32::from(first) & 0b11_111;
398 let low_six = u32::from(second) & 0b111_111;
399 let v = unsafe { self.trie.utf8_two_byte(high_five, low_six) };
402 let point = (high_five << 6) | low_six;
403 return Some((unsafe { char::from_u32_unchecked(point) }, v));
408 }
409 if self.remaining.len() == 2 {
410 self.remaining = &self.remaining[2..];
411 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
412 }
413 let third = self.remaining[2];
414 if !in_inclusive_range8(third, 0x80, 0xBF) {
415 self.remaining = &self.remaining[2..];
416 return Some(('\u{FFFD}', self.trie.bmp(0xFFFD)));
417 }
418 if first < 0xF0 {
419 self.remaining = &self.remaining[3..];
420 let high_ten = ((u32::from(first) & 0b1111) << 6) | (u32::from(second) & 0b111_111);
421 let low_six = u32::from(third) & 0b111_111;
422 let v = unsafe { self.trie.utf8_three_byte(high_ten, low_six) };
425 let point = (high_ten << 6) | low_six;
426 return Some((unsafe { char::from_u32_unchecked(point) }, v));
431 }
432 self.remaining = &self.remaining[3..];
436 Some(('\u{FFFD}', self.trie.bmp(0xFFFD)))
437 }
438}
439
440impl<'slice, 'trie, T, V> Clone for Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>
441where
442 V: TrieValue + Default,
443 T: AbstractCodePointTrie<'trie, V>,
444{
445 #[inline]
446 fn clone(&self) -> Self {
447 Self {
448 remaining: self.remaining,
449 trie: self.trie,
450 phantom: PhantomData,
451 }
452 }
453}
454
455impl<'slice, 'trie, T, V> WithTrie<'trie, T, V>
456 for Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>
457where
458 V: TrieValue + Default,
459 T: AbstractCodePointTrie<'trie, V>,
460{
461 #[inline]
462 fn trie(&self) -> &'trie T {
463 self.trie
464 }
465}
466
467impl<'slice, 'trie, T, V> Iterator for Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>
468where
469 V: TrieValue + Default,
470 T: AbstractCodePointTrie<'trie, V>,
471{
472 type Item = (char, V);
473
474 #[inline]
475 fn next(&mut self) -> Option<Self::Item> {
476 #[allow(clippy::never_loop)]
478 loop {
479 if self.remaining.len() < 4 {
480 break;
481 }
482 let first = self.remaining[0];
483 if first < 0x80 {
484 self.remaining = &self.remaining[1..];
485 return Some((char::from(first), V::default()));
486 }
487 let second = self.remaining[1];
488 if in_inclusive_range8(first, 0xC2, 0xDF) {
489 if !in_inclusive_range8(second, 0x80, 0xBF) {
490 break;
491 }
492 self.remaining = &self.remaining[2..];
493 let high_five = u32::from(first) & 0b11_111;
494 let low_six = u32::from(second) & 0b111_111;
495 let v = unsafe { self.trie.utf8_two_byte(high_five, low_six) };
498 let point = (high_five << 6) | low_six;
499 return Some((unsafe { char::from_u32_unchecked(point) }, v));
504 }
505 let third = self.remaining[2];
508 if first < 0xF0 {
509 if ((UTF8_DATA.table[usize::from(second)]
510 & UTF8_DATA.table[usize::from(first) + 0x80])
511 | (third >> 6))
512 != 2
513 {
514 break;
515 }
516 self.remaining = &self.remaining[3..];
517 let high_ten = ((u32::from(first) & 0b1111) << 6) | (u32::from(second) & 0b111_111);
518 let low_six = u32::from(third) & 0b111_111;
519 let v = unsafe { self.trie.utf8_three_byte(high_ten, low_six) };
522 let point = (high_ten << 6) | low_six;
523 return Some((unsafe { char::from_u32_unchecked(point) }, v));
528 }
529 let fourth = self.remaining[3];
530 if (u16::from(
531 UTF8_DATA.table[usize::from(second)] & UTF8_DATA.table[usize::from(first) + 0x80],
532 ) | u16::from(third >> 6)
533 | (u16::from(fourth & 0xC0) << 2))
534 != 0x202
535 {
536 break;
537 }
538 let point = ((u32::from(first) & 0x7) << 18)
539 | ((u32::from(second) & 0x3F) << 12)
540 | ((u32::from(third) & 0x3F) << 6)
541 | (u32::from(fourth) & 0x3F);
542 self.remaining = &self.remaining[4..];
543 return Some((
546 unsafe { char::from_u32_unchecked(point) },
547 self.trie.supplementary(point),
548 ));
549 }
550 self.next_fallback()
551 }
552}
553
554impl<'slice, 'trie, T, V> DoubleEndedIterator
555 for Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>
556where
557 V: TrieValue + Default,
558 T: AbstractCodePointTrie<'trie, V>,
559{
560 #[inline]
561 fn next_back(&mut self) -> Option<(char, V)> {
562 if self.remaining.is_empty() {
563 return None;
564 }
565 let mut attempt = 1;
566 for b in self.remaining.iter().rev() {
567 if b & 0xC0 != 0x80 {
568 let (head, tail) = self.remaining.split_at(self.remaining.len() - attempt);
569 let mut inner = Utf8CharsWithTrieDefaultForAscii::new(tail, self.trie);
570 let candidate = inner.next();
571 if inner.as_slice().is_empty() {
572 self.remaining = head;
573 return candidate;
574 }
575 break;
576 }
577 if attempt == 4 {
578 break;
579 }
580 attempt += 1;
581 }
582
583 self.remaining = &self.remaining[..self.remaining.len() - 1];
584 Some(('\u{FFFD}', self.trie.bmp(0xFFFD)))
585 }
586}
587
588impl<'slice, 'trie, T, V> FusedIterator for Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>
589where
590 V: TrieValue + Default,
591 T: AbstractCodePointTrie<'trie, V>,
592{
593}
594
595pub trait Utf8CharsWithTrieDefaultForAsciiEx<'slice, 'trie, T, V>
599where
600 V: TrieValue + Default,
601 T: AbstractCodePointTrie<'trie, V>,
602{
603 fn chars_with_trie_default_for_ascii(
606 &'slice self,
607 trie: &'trie T,
608 ) -> Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V>;
609 fn char_indices_with_trie_default_for_ascii(
612 &'slice self,
613 trie: &'trie T,
614 ) -> Utf8CharIndicesWithTrie<'slice, 'trie, T, V>;
615}
616
617impl<'slice, 'trie, T, V> Utf8CharsWithTrieDefaultForAsciiEx<'slice, 'trie, T, V> for [u8]
618where
619 V: TrieValue + Default,
620 T: AbstractCodePointTrie<'trie, V>,
621{
622 #[inline]
625 fn chars_with_trie_default_for_ascii(
626 &'slice self,
627 trie: &'trie T,
628 ) -> Utf8CharsWithTrieDefaultForAscii<'slice, 'trie, T, V> {
629 Utf8CharsWithTrieDefaultForAscii::new(self, trie)
630 }
631
632 #[inline]
635 fn char_indices_with_trie_default_for_ascii(
636 &'slice self,
637 trie: &'trie T,
638 ) -> Utf8CharIndicesWithTrie<'slice, 'trie, T, V> {
639 Utf8CharIndicesWithTrie::new(self, trie)
640 }
641}