encoding_rs/
handles.rs

1// Copyright Mozilla Foundation. See the COPYRIGHT
2// file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10//! This module provides structs that use lifetimes to couple bounds checking
11//! and space availability checking and detaching those from actual slice
12//! reading/writing.
13//!
14//! At present, the internals of the implementation are safe code, so the
15//! bound checks currently also happen on read/write. Once this code works,
16//! the plan is to replace the internals with unsafe code that omits the
17//! bound check at the read/write time.
18
19#[cfg(all(feature = "simd-accel", target_endian = "little"))]
20use crate::simd_funcs::*;
21
22#[cfg(all(feature = "simd-accel", target_endian = "little"))]
23use core::simd::u16x8;
24
25use super::DecoderResult;
26use super::EncoderResult;
27use crate::ascii::*;
28use crate::utf_8::convert_utf8_to_utf16_up_to_invalid;
29use crate::utf_8::utf8_valid_up_to;
30
31#[cfg(all(feature = "simd-accel", target_endian = "little"))]
32const SIMD_STRIDE_SIZE: usize = crate::ascii::STRIDE;
33
34pub enum Space<T> {
35    Available(T),
36    Full(usize),
37}
38
39pub enum CopyAsciiResult<T, U> {
40    Stop(T),
41    GoOn(U),
42}
43
44pub enum NonAscii {
45    BmpExclAscii(u16),
46    Astral(char),
47}
48
49pub enum Unicode {
50    Ascii(u8),
51    NonAscii(NonAscii),
52}
53
54// Start UTF-16LE/BE fast path
55
56pub trait Endian {
57    const OPPOSITE_ENDIAN: bool;
58}
59
60pub struct BigEndian;
61
62impl Endian for BigEndian {
63    #[cfg(target_endian = "little")]
64    const OPPOSITE_ENDIAN: bool = true;
65
66    #[cfg(target_endian = "big")]
67    const OPPOSITE_ENDIAN: bool = false;
68}
69
70pub struct LittleEndian;
71
72impl Endian for LittleEndian {
73    #[cfg(target_endian = "little")]
74    const OPPOSITE_ENDIAN: bool = false;
75
76    #[cfg(target_endian = "big")]
77    const OPPOSITE_ENDIAN: bool = true;
78}
79
80#[derive(Debug, Copy, Clone)]
81struct UnalignedU16Slice {
82    // Safety invariant: ptr must be valid for reading 2*len bytes
83    ptr: *const u8,
84    len: usize,
85}
86
87impl UnalignedU16Slice {
88    /// Safety: ptr must be valid for reading 2*len bytes
89    #[inline(always)]
90    pub unsafe fn new(ptr: *const u8, len: usize) -> UnalignedU16Slice {
91        // Safety: field invariant passed up to caller here
92        UnalignedU16Slice { ptr, len }
93    }
94
95    #[inline(always)]
96    pub fn trim_last(&mut self) {
97        assert!(self.len > 0);
98        // Safety: invariant upheld here: a slice is still valid with a shorter len
99        self.len -= 1;
100    }
101
102    #[inline(always)]
103    pub fn at(&self, i: usize) -> u16 {
104        use core::mem::MaybeUninit;
105
106        assert!(i < self.len);
107        unsafe {
108            let mut u: MaybeUninit<u16> = MaybeUninit::uninit();
109            // Safety: i is at most len - 1, which works here
110            ::core::ptr::copy_nonoverlapping(self.ptr.add(i * 2), u.as_mut_ptr() as *mut u8, 2);
111            // Safety: valid read above lets us do this
112            u.assume_init()
113        }
114    }
115
116    #[cfg(all(feature = "simd-accel", target_endian = "little"))]
117    #[inline(always)]
118    pub fn simd_at(&self, i: usize) -> u16x8 {
119        // Safety: i/len are on the scale of u16s, each one corresponds to 2 u8s
120        assert!(i + SIMD_STRIDE_SIZE / 2 <= self.len);
121        let byte_index = i * 2;
122        // Safety: load16_unaligned needs SIMD_STRIDE_SIZE=16 u8 elements to read,
123        // or 16/2 = 8 u16 elements to read.
124        // We have checked that we have at least that many above.
125
126        unsafe { to_u16_lanes(load16_unaligned(self.ptr.add(byte_index))) }
127    }
128
129    #[inline(always)]
130    pub fn len(&self) -> usize {
131        self.len
132    }
133
134    #[inline(always)]
135    pub fn tail(&self, from: usize) -> UnalignedU16Slice {
136        // XXX the return value should be restricted not to
137        // outlive self.
138        assert!(from <= self.len);
139        // Safety: This upholds the same invariant: `from` is in bounds and we're returning a shorter slice
140        unsafe { UnalignedU16Slice::new(self.ptr.add(from * 2), self.len - from) }
141    }
142
143    #[cfg(all(feature = "simd-accel", target_endian = "little"))]
144    #[inline(always)]
145    pub fn copy_bmp_to<E: Endian>(&self, other: &mut [u16]) -> Option<(u16, usize)> {
146        assert!(self.len <= other.len());
147        let mut offset = 0;
148        // Safety: SIMD_STRIDE_SIZE is measured in bytes, whereas len is in u16s. We check we can
149        // munch SIMD_STRIDE_SIZE / 2 u16s which means we can write SIMD_STRIDE_SIZE u8s
150        if SIMD_STRIDE_SIZE / 2 <= self.len {
151            let len_minus_stride = self.len - SIMD_STRIDE_SIZE / 2;
152            loop {
153                let mut simd = self.simd_at(offset);
154                if E::OPPOSITE_ENDIAN {
155                    simd = simd_byte_swap(simd);
156                }
157                // Safety: we have enough space on the other side to write this
158                unsafe {
159                    store8_unaligned(other.as_mut_ptr().add(offset), simd);
160                }
161                if contains_surrogates(simd) {
162                    break;
163                }
164                offset += SIMD_STRIDE_SIZE / 2;
165                // Safety: This ensures we still have space for writing SIMD_STRIDE_SIZE u8s
166                if offset > len_minus_stride {
167                    break;
168                }
169            }
170        }
171        while offset < self.len {
172            let unit = swap_if_opposite_endian::<E>(self.at(offset));
173            other[offset] = unit;
174            if super::in_range16(unit, 0xD800, 0xE000) {
175                return Some((unit, offset));
176            }
177            offset += 1;
178        }
179        None
180    }
181
182    #[cfg(not(all(feature = "simd-accel", target_endian = "little")))]
183    #[inline(always)]
184    fn copy_bmp_to<E: Endian>(&self, other: &mut [u16]) -> Option<(u16, usize)> {
185        assert!(self.len <= other.len());
186        for (i, target) in other.iter_mut().enumerate().take(self.len) {
187            let unit = swap_if_opposite_endian::<E>(self.at(i));
188            *target = unit;
189            if super::in_range16(unit, 0xD800, 0xE000) {
190                return Some((unit, i));
191            }
192        }
193        None
194    }
195}
196
197#[inline(always)]
198fn copy_unaligned_basic_latin_to_ascii_alu<E: Endian>(
199    src: UnalignedU16Slice,
200    dst: &mut [u8],
201    offset: usize,
202) -> CopyAsciiResult<usize, (u16, usize)> {
203    let len = ::core::cmp::min(src.len(), dst.len());
204    let mut i = 0usize;
205    loop {
206        if i == len {
207            return CopyAsciiResult::Stop(i + offset);
208        }
209        let unit = swap_if_opposite_endian::<E>(src.at(i));
210        if unit > 0x7F {
211            return CopyAsciiResult::GoOn((unit, i + offset));
212        }
213        dst[i] = unit as u8;
214        i += 1;
215    }
216}
217
218#[inline(always)]
219fn swap_if_opposite_endian<E: Endian>(unit: u16) -> u16 {
220    if E::OPPOSITE_ENDIAN {
221        unit.swap_bytes()
222    } else {
223        unit
224    }
225}
226
227#[cfg(not(all(feature = "simd-accel", target_endian = "little")))]
228#[inline(always)]
229fn copy_unaligned_basic_latin_to_ascii<E: Endian>(
230    src: UnalignedU16Slice,
231    dst: &mut [u8],
232) -> CopyAsciiResult<usize, (u16, usize)> {
233    copy_unaligned_basic_latin_to_ascii_alu::<E>(src, dst, 0)
234}
235
236#[cfg(all(feature = "simd-accel", target_endian = "little"))]
237#[inline(always)]
238fn copy_unaligned_basic_latin_to_ascii<E: Endian>(
239    src: UnalignedU16Slice,
240    dst: &mut [u8],
241) -> CopyAsciiResult<usize, (u16, usize)> {
242    let len = ::core::cmp::min(src.len(), dst.len());
243    let mut offset = 0;
244    // Safety: This check ensures we are able to read/write at least SIMD_STRIDE_SIZE elements
245    if SIMD_STRIDE_SIZE <= len {
246        let len_minus_stride = len - SIMD_STRIDE_SIZE;
247        loop {
248            let mut first = src.simd_at(offset);
249            let mut second = src.simd_at(offset + (SIMD_STRIDE_SIZE / 2));
250            if E::OPPOSITE_ENDIAN {
251                first = simd_byte_swap(first);
252                second = simd_byte_swap(second);
253            }
254            if !simd_is_basic_latin(first | second) {
255                break;
256            }
257            let packed = simd_pack(first, second);
258            // Safety: We are able to write SIMD_STRIDE_SIZE elements in this iteration
259            unsafe {
260                store16_unaligned(dst.as_mut_ptr().add(offset), packed);
261            }
262            offset += SIMD_STRIDE_SIZE;
263            // Safety: This is `offset > len - SIMD_STRIDE_SIZE`, which ensures that we can write at least SIMD_STRIDE_SIZE elements
264            // in the next iteration
265            if offset > len_minus_stride {
266                break;
267            }
268        }
269    }
270    copy_unaligned_basic_latin_to_ascii_alu::<E>(src.tail(offset), &mut dst[offset..], offset)
271}
272
273#[inline(always)]
274fn convert_unaligned_utf16_to_utf8<E: Endian>(
275    src: UnalignedU16Slice,
276    dst: &mut [u8],
277) -> (usize, usize, bool) {
278    if dst.len() < 4 {
279        return (0, 0, false);
280    }
281    let mut src_pos = 0usize;
282    let mut dst_pos = 0usize;
283    let src_len = src.len();
284    let dst_len_minus_three = dst.len() - 3;
285    'outer: loop {
286        let mut non_ascii = match copy_unaligned_basic_latin_to_ascii::<E>(
287            src.tail(src_pos),
288            &mut dst[dst_pos..],
289        ) {
290            CopyAsciiResult::GoOn((unit, read_written)) => {
291                src_pos += read_written;
292                dst_pos += read_written;
293                unit
294            }
295            CopyAsciiResult::Stop(read_written) => {
296                return (src_pos + read_written, dst_pos + read_written, false);
297            }
298        };
299        if dst_pos >= dst_len_minus_three {
300            break 'outer;
301        }
302        // We have enough destination space to commit to
303        // having read `non_ascii`.
304        src_pos += 1;
305        'inner: loop {
306            let non_ascii_minus_surrogate_start = non_ascii.wrapping_sub(0xD800);
307            if non_ascii_minus_surrogate_start > (0xDFFF - 0xD800) {
308                if non_ascii < 0x800 {
309                    dst[dst_pos] = ((non_ascii >> 6) | 0xC0) as u8;
310                    dst_pos += 1;
311                    dst[dst_pos] = ((non_ascii & 0x3F) | 0x80) as u8;
312                    dst_pos += 1;
313                } else {
314                    dst[dst_pos] = ((non_ascii >> 12) | 0xE0) as u8;
315                    dst_pos += 1;
316                    dst[dst_pos] = (((non_ascii & 0xFC0) >> 6) | 0x80) as u8;
317                    dst_pos += 1;
318                    dst[dst_pos] = ((non_ascii & 0x3F) | 0x80) as u8;
319                    dst_pos += 1;
320                }
321            } else if non_ascii_minus_surrogate_start <= (0xDBFF - 0xD800) {
322                // high surrogate
323                if src_pos < src_len {
324                    let second = swap_if_opposite_endian::<E>(src.at(src_pos));
325                    let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
326                    if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
327                        // The next code unit is a low surrogate. Advance position.
328                        src_pos += 1;
329                        let point = (u32::from(non_ascii) << 10) + u32::from(second)
330                            - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32);
331
332                        dst[dst_pos] = ((point >> 18) | 0xF0u32) as u8;
333                        dst_pos += 1;
334                        dst[dst_pos] = (((point & 0x3F000u32) >> 12) | 0x80u32) as u8;
335                        dst_pos += 1;
336                        dst[dst_pos] = (((point & 0xFC0u32) >> 6) | 0x80u32) as u8;
337                        dst_pos += 1;
338                        dst[dst_pos] = ((point & 0x3Fu32) | 0x80u32) as u8;
339                        dst_pos += 1;
340                    } else {
341                        // The next code unit is not a low surrogate. Don't advance
342                        // position and treat the high surrogate as unpaired.
343                        return (src_pos, dst_pos, true);
344                    }
345                } else {
346                    // Unpaired surrogate at the end of buffer
347                    return (src_pos, dst_pos, true);
348                }
349            } else {
350                // Unpaired low surrogate
351                return (src_pos, dst_pos, true);
352            }
353            if dst_pos >= dst_len_minus_three || src_pos == src_len {
354                break 'outer;
355            }
356            let unit = swap_if_opposite_endian::<E>(src.at(src_pos));
357            src_pos += 1;
358            if unit > 0x7F {
359                non_ascii = unit;
360                continue 'inner;
361            }
362            dst[dst_pos] = unit as u8;
363            dst_pos += 1;
364            continue 'outer;
365        }
366    }
367    (src_pos, dst_pos, false)
368}
369
370// Byte source
371
372pub struct ByteSource<'a> {
373    slice: &'a [u8],
374    pos: usize,
375}
376
377impl<'a> ByteSource<'a> {
378    #[inline(always)]
379    pub fn new(src: &'a [u8]) -> ByteSource<'a> {
380        ByteSource { slice: src, pos: 0 }
381    }
382    #[inline(always)]
383    pub fn check_available<'b>(&'b mut self) -> Space<ByteReadHandle<'b, 'a>> {
384        if self.pos < self.slice.len() {
385            Space::Available(ByteReadHandle::new(self))
386        } else {
387            Space::Full(self.consumed())
388        }
389    }
390    #[inline(always)]
391    fn read(&mut self) -> u8 {
392        let ret = self.slice[self.pos];
393        self.pos += 1;
394        ret
395    }
396    #[inline(always)]
397    fn unread(&mut self) -> usize {
398        self.pos -= 1;
399        self.pos
400    }
401    #[inline(always)]
402    pub fn consumed(&self) -> usize {
403        self.pos
404    }
405}
406
407pub struct ByteReadHandle<'a, 'b>
408where
409    'b: 'a,
410{
411    source: &'a mut ByteSource<'b>,
412}
413
414impl<'a, 'b> ByteReadHandle<'a, 'b>
415where
416    'b: 'a,
417{
418    #[inline(always)]
419    fn new(src: &'a mut ByteSource<'b>) -> ByteReadHandle<'a, 'b> {
420        ByteReadHandle { source: src }
421    }
422    #[inline(always)]
423    pub fn read(self) -> (u8, ByteUnreadHandle<'a, 'b>) {
424        let byte = self.source.read();
425        let handle = ByteUnreadHandle::new(self.source);
426        (byte, handle)
427    }
428    #[inline(always)]
429    pub fn consumed(&self) -> usize {
430        self.source.consumed()
431    }
432}
433
434pub struct ByteUnreadHandle<'a, 'b>
435where
436    'b: 'a,
437{
438    source: &'a mut ByteSource<'b>,
439}
440
441impl<'a, 'b> ByteUnreadHandle<'a, 'b>
442where
443    'b: 'a,
444{
445    #[inline(always)]
446    fn new(src: &'a mut ByteSource<'b>) -> ByteUnreadHandle<'a, 'b> {
447        ByteUnreadHandle { source: src }
448    }
449    #[inline(always)]
450    pub fn unread(self) -> usize {
451        self.source.unread()
452    }
453    #[inline(always)]
454    pub fn consumed(&self) -> usize {
455        self.source.consumed()
456    }
457    #[inline(always)]
458    pub fn commit(self) -> &'a mut ByteSource<'b> {
459        self.source
460    }
461}
462
463// UTF-16 destination
464
465pub struct Utf16BmpHandle<'a, 'b>
466where
467    'b: 'a,
468{
469    dest: &'a mut Utf16Destination<'b>,
470}
471
472impl<'a, 'b> Utf16BmpHandle<'a, 'b>
473where
474    'b: 'a,
475{
476    #[inline(always)]
477    fn new(dst: &'a mut Utf16Destination<'b>) -> Utf16BmpHandle<'a, 'b> {
478        Utf16BmpHandle { dest: dst }
479    }
480    #[inline(always)]
481    pub fn written(&self) -> usize {
482        self.dest.written()
483    }
484    #[inline(always)]
485    pub fn write_ascii(self, ascii: u8) -> &'a mut Utf16Destination<'b> {
486        self.dest.write_ascii(ascii);
487        self.dest
488    }
489    #[inline(always)]
490    pub fn write_bmp(self, bmp: u16) -> &'a mut Utf16Destination<'b> {
491        self.dest.write_bmp(bmp);
492        self.dest
493    }
494    #[inline(always)]
495    pub fn write_bmp_excl_ascii(self, bmp: u16) -> &'a mut Utf16Destination<'b> {
496        self.dest.write_bmp_excl_ascii(bmp);
497        self.dest
498    }
499    #[inline(always)]
500    pub fn write_mid_bmp(self, bmp: u16) -> &'a mut Utf16Destination<'b> {
501        self.dest.write_mid_bmp(bmp);
502        self.dest
503    }
504    #[inline(always)]
505    pub fn write_upper_bmp(self, bmp: u16) -> &'a mut Utf16Destination<'b> {
506        self.dest.write_upper_bmp(bmp);
507        self.dest
508    }
509    #[inline(always)]
510    pub fn commit(self) -> &'a mut Utf16Destination<'b> {
511        self.dest
512    }
513}
514
515pub struct Utf16AstralHandle<'a, 'b>
516where
517    'b: 'a,
518{
519    dest: &'a mut Utf16Destination<'b>,
520}
521
522impl<'a, 'b> Utf16AstralHandle<'a, 'b>
523where
524    'b: 'a,
525{
526    #[inline(always)]
527    fn new(dst: &'a mut Utf16Destination<'b>) -> Utf16AstralHandle<'a, 'b> {
528        Utf16AstralHandle { dest: dst }
529    }
530    #[inline(always)]
531    pub fn written(&self) -> usize {
532        self.dest.written()
533    }
534    #[inline(always)]
535    pub fn write_ascii(self, ascii: u8) -> &'a mut Utf16Destination<'b> {
536        self.dest.write_ascii(ascii);
537        self.dest
538    }
539    #[inline(always)]
540    pub fn write_bmp(self, bmp: u16) -> &'a mut Utf16Destination<'b> {
541        self.dest.write_bmp(bmp);
542        self.dest
543    }
544    #[inline(always)]
545    pub fn write_bmp_excl_ascii(self, bmp: u16) -> &'a mut Utf16Destination<'b> {
546        self.dest.write_bmp_excl_ascii(bmp);
547        self.dest
548    }
549    #[inline(always)]
550    pub fn write_upper_bmp(self, bmp: u16) -> &'a mut Utf16Destination<'b> {
551        self.dest.write_upper_bmp(bmp);
552        self.dest
553    }
554    #[inline(always)]
555    pub fn write_astral(self, astral: u32) -> &'a mut Utf16Destination<'b> {
556        self.dest.write_astral(astral);
557        self.dest
558    }
559    #[inline(always)]
560    pub fn write_surrogate_pair(self, high: u16, low: u16) -> &'a mut Utf16Destination<'b> {
561        self.dest.write_surrogate_pair(high, low);
562        self.dest
563    }
564    #[inline(always)]
565    pub fn write_big5_combination(
566        self,
567        combined: u16,
568        combining: u16,
569    ) -> &'a mut Utf16Destination<'b> {
570        self.dest.write_big5_combination(combined, combining);
571        self.dest
572    }
573    #[inline(always)]
574    pub fn commit(self) -> &'a mut Utf16Destination<'b> {
575        self.dest
576    }
577}
578
579pub struct Utf16Destination<'a> {
580    slice: &'a mut [u16],
581    pos: usize,
582}
583
584impl<'a> Utf16Destination<'a> {
585    #[inline(always)]
586    pub fn new(dst: &mut [u16]) -> Utf16Destination<'_> {
587        Utf16Destination { slice: dst, pos: 0 }
588    }
589    #[inline(always)]
590    pub fn check_space_bmp<'b>(&'b mut self) -> Space<Utf16BmpHandle<'b, 'a>> {
591        if self.pos < self.slice.len() {
592            Space::Available(Utf16BmpHandle::new(self))
593        } else {
594            Space::Full(self.written())
595        }
596    }
597    #[inline(always)]
598    pub fn check_space_astral<'b>(&'b mut self) -> Space<Utf16AstralHandle<'b, 'a>> {
599        if self.pos + 1 < self.slice.len() {
600            Space::Available(Utf16AstralHandle::new(self))
601        } else {
602            Space::Full(self.written())
603        }
604    }
605    #[inline(always)]
606    pub fn written(&self) -> usize {
607        self.pos
608    }
609    #[inline(always)]
610    fn write_code_unit(&mut self, u: u16) {
611        unsafe {
612            // OK, because we checked before handing out a handle.
613            *(self.slice.get_unchecked_mut(self.pos)) = u;
614        }
615        self.pos += 1;
616    }
617    #[inline(always)]
618    fn write_ascii(&mut self, ascii: u8) {
619        debug_assert!(ascii < 0x80);
620        self.write_code_unit(u16::from(ascii));
621    }
622    #[inline(always)]
623    fn write_bmp(&mut self, bmp: u16) {
624        self.write_code_unit(bmp);
625    }
626    #[inline(always)]
627    fn write_bmp_excl_ascii(&mut self, bmp: u16) {
628        debug_assert!(bmp >= 0x80);
629        self.write_code_unit(bmp);
630    }
631    #[inline(always)]
632    fn write_mid_bmp(&mut self, bmp: u16) {
633        debug_assert!(bmp >= 0x80); // XXX
634        self.write_code_unit(bmp);
635    }
636    #[inline(always)]
637    fn write_upper_bmp(&mut self, bmp: u16) {
638        debug_assert!(bmp >= 0x80);
639        self.write_code_unit(bmp);
640    }
641    #[inline(always)]
642    fn write_astral(&mut self, astral: u32) {
643        debug_assert!(astral > 0xFFFF);
644        debug_assert!(astral <= 0x10_FFFF);
645        self.write_code_unit((0xD7C0 + (astral >> 10)) as u16);
646        self.write_code_unit((0xDC00 + (astral & 0x3FF)) as u16);
647    }
648    #[inline(always)]
649    fn write_surrogate_pair(&mut self, high: u16, low: u16) {
650        self.write_code_unit(high);
651        self.write_code_unit(low);
652    }
653    #[inline(always)]
654    fn write_big5_combination(&mut self, combined: u16, combining: u16) {
655        self.write_bmp_excl_ascii(combined);
656        self.write_bmp_excl_ascii(combining);
657    }
658    // Safety-usable invariant: CopyAsciiResult::GoOn will only contain bytes >=0x80
659    #[inline(always)]
660    pub fn copy_ascii_from_check_space_bmp<'b>(
661        &'b mut self,
662        source: &mut ByteSource,
663    ) -> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf16BmpHandle<'b, 'a>)> {
664        let non_ascii_ret = {
665            let src_remaining = &source.slice[source.pos..];
666            let dst_remaining = &mut self.slice[self.pos..];
667            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
668                (DecoderResult::OutputFull, dst_remaining.len())
669            } else {
670                (DecoderResult::InputEmpty, src_remaining.len())
671            };
672            // Safety: This function is documented as needing valid pointers for src/dest and len, which
673            // is true since we've passed the minumum length of the two
674            match ascii_to_basic_latin(src_remaining, dst_remaining) {
675                None => {
676                    source.pos += length;
677                    self.pos += length;
678                    return CopyAsciiResult::Stop((pending, source.pos, self.pos));
679                }
680                // Safety: the function is documented as returning bytes >=0x80 in the Some
681                Some((non_ascii, consumed)) => {
682                    source.pos += consumed;
683                    self.pos += consumed;
684                    source.pos += 1; // +1 for non_ascii
685                    // Safety: non-ascii bubbled out here
686                    non_ascii
687                }
688            }
689        };
690        // Safety: non-ascii returned here
691        CopyAsciiResult::GoOn((non_ascii_ret, Utf16BmpHandle::new(self)))
692    }
693    // Safety-usable invariant: CopyAsciiResult::GoOn will only contain bytes >=0x80
694    #[inline(always)]
695    pub fn copy_ascii_from_check_space_astral<'b>(
696        &'b mut self,
697        source: &mut ByteSource,
698    ) -> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf16AstralHandle<'b, 'a>)> {
699        let non_ascii_ret = {
700            let dst_len = self.slice.len();
701            let src_remaining = &source.slice[source.pos..];
702            let dst_remaining = &mut self.slice[self.pos..];
703            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
704                (DecoderResult::OutputFull, dst_remaining.len())
705            } else {
706                (DecoderResult::InputEmpty, src_remaining.len())
707            };
708            // Safety: This function is documented as needing valid pointers for src/dest and len, which
709            // is true since we've passed the minumum length of the two
710            match ascii_to_basic_latin(src_remaining, dst_remaining) {
711                None => {
712                    source.pos += length;
713                    self.pos += length;
714                    return CopyAsciiResult::Stop((pending, source.pos, self.pos));
715                }
716                // Safety: the function is documented as returning bytes >=0x80 in the Some
717                Some((non_ascii, consumed)) => {
718                    source.pos += consumed;
719                    self.pos += consumed;
720                    if self.pos + 1 < dst_len {
721                        source.pos += 1; // +1 for non_ascii
722                        // Safety: non-ascii bubbled out here
723                        non_ascii
724                    } else {
725                        return CopyAsciiResult::Stop((
726                            DecoderResult::OutputFull,
727                            source.pos,
728                            self.pos,
729                        ));
730                    }
731                }
732            }
733        };
734        // Safety: non-ascii returned here
735        CopyAsciiResult::GoOn((non_ascii_ret, Utf16AstralHandle::new(self)))
736    }
737    #[inline(always)]
738    pub fn copy_utf8_up_to_invalid_from(&mut self, source: &mut ByteSource) {
739        let src_remaining = &source.slice[source.pos..];
740        let dst_remaining = &mut self.slice[self.pos..];
741        let (read, written) = convert_utf8_to_utf16_up_to_invalid(src_remaining, dst_remaining);
742        source.pos += read;
743        self.pos += written;
744    }
745    #[inline(always)]
746    pub fn copy_utf16_from<E: Endian>(
747        &mut self,
748        source: &mut ByteSource,
749    ) -> Option<(usize, usize)> {
750        let src_remaining = &source.slice[source.pos..];
751        let dst_remaining = &mut self.slice[self.pos..];
752
753        let mut src_unaligned = unsafe {
754            UnalignedU16Slice::new(
755                src_remaining.as_ptr(),
756                ::core::cmp::min(src_remaining.len() / 2, dst_remaining.len()),
757            )
758        };
759        if src_unaligned.len() == 0 {
760            return None;
761        }
762        let last_unit = swap_if_opposite_endian::<E>(src_unaligned.at(src_unaligned.len() - 1));
763        if super::in_range16(last_unit, 0xD800, 0xDC00) {
764            // Last code unit is a high surrogate. It might
765            // legitimately form a pair later, so let's not
766            // include it.
767            src_unaligned.trim_last();
768        }
769        let mut offset = 0usize;
770        loop {
771            if let Some((surrogate, bmp_len)) = {
772                let src_left = src_unaligned.tail(offset);
773                let dst_left = &mut dst_remaining[offset..src_unaligned.len()];
774                src_left.copy_bmp_to::<E>(dst_left)
775            } {
776                offset += bmp_len; // surrogate has not been consumed yet
777                let second_pos = offset + 1;
778                if surrogate > 0xDBFF || second_pos == src_unaligned.len() {
779                    // Unpaired surrogate
780                    source.pos += second_pos * 2;
781                    self.pos += offset;
782                    return Some((source.pos, self.pos));
783                }
784                let second = swap_if_opposite_endian::<E>(src_unaligned.at(second_pos));
785                if !super::in_range16(second, 0xDC00, 0xE000) {
786                    // Unpaired surrogate
787                    source.pos += second_pos * 2;
788                    self.pos += offset;
789                    return Some((source.pos, self.pos));
790                }
791                // `surrogate` was already speculatively written
792                dst_remaining[second_pos] = second;
793                offset += 2;
794                continue;
795            } else {
796                source.pos += src_unaligned.len() * 2;
797                self.pos += src_unaligned.len();
798                return None;
799            }
800        }
801    }
802}
803
804// UTF-8 destination
805
806pub struct Utf8BmpHandle<'a, 'b>
807where
808    'b: 'a,
809{
810    dest: &'a mut Utf8Destination<'b>,
811}
812
813impl<'a, 'b> Utf8BmpHandle<'a, 'b>
814where
815    'b: 'a,
816{
817    #[inline(always)]
818    fn new(dst: &'a mut Utf8Destination<'b>) -> Utf8BmpHandle<'a, 'b> {
819        Utf8BmpHandle { dest: dst }
820    }
821    #[inline(always)]
822    pub fn written(&self) -> usize {
823        self.dest.written()
824    }
825    #[inline(always)]
826    pub fn write_ascii(self, ascii: u8) -> &'a mut Utf8Destination<'b> {
827        self.dest.write_ascii(ascii);
828        self.dest
829    }
830    #[inline(always)]
831    pub fn write_bmp(self, bmp: u16) -> &'a mut Utf8Destination<'b> {
832        self.dest.write_bmp(bmp);
833        self.dest
834    }
835    #[inline(always)]
836    pub fn write_bmp_excl_ascii(self, bmp: u16) -> &'a mut Utf8Destination<'b> {
837        self.dest.write_bmp_excl_ascii(bmp);
838        self.dest
839    }
840    #[inline(always)]
841    pub fn write_mid_bmp(self, bmp: u16) -> &'a mut Utf8Destination<'b> {
842        self.dest.write_mid_bmp(bmp);
843        self.dest
844    }
845    #[inline(always)]
846    pub fn write_upper_bmp(self, bmp: u16) -> &'a mut Utf8Destination<'b> {
847        self.dest.write_upper_bmp(bmp);
848        self.dest
849    }
850    #[inline(always)]
851    pub fn commit(self) -> &'a mut Utf8Destination<'b> {
852        self.dest
853    }
854}
855
856pub struct Utf8AstralHandle<'a, 'b>
857where
858    'b: 'a,
859{
860    dest: &'a mut Utf8Destination<'b>,
861}
862
863impl<'a, 'b> Utf8AstralHandle<'a, 'b>
864where
865    'b: 'a,
866{
867    #[inline(always)]
868    fn new(dst: &'a mut Utf8Destination<'b>) -> Utf8AstralHandle<'a, 'b> {
869        Utf8AstralHandle { dest: dst }
870    }
871    #[inline(always)]
872    pub fn written(&self) -> usize {
873        self.dest.written()
874    }
875    #[inline(always)]
876    pub fn write_ascii(self, ascii: u8) -> &'a mut Utf8Destination<'b> {
877        self.dest.write_ascii(ascii);
878        self.dest
879    }
880    #[inline(always)]
881    pub fn write_bmp(self, bmp: u16) -> &'a mut Utf8Destination<'b> {
882        self.dest.write_bmp(bmp);
883        self.dest
884    }
885    #[inline(always)]
886    pub fn write_bmp_excl_ascii(self, bmp: u16) -> &'a mut Utf8Destination<'b> {
887        self.dest.write_bmp_excl_ascii(bmp);
888        self.dest
889    }
890    #[inline(always)]
891    pub fn write_upper_bmp(self, bmp: u16) -> &'a mut Utf8Destination<'b> {
892        self.dest.write_upper_bmp(bmp);
893        self.dest
894    }
895    #[inline(always)]
896    pub fn write_astral(self, astral: u32) -> &'a mut Utf8Destination<'b> {
897        self.dest.write_astral(astral);
898        self.dest
899    }
900    #[inline(always)]
901    pub fn write_surrogate_pair(self, high: u16, low: u16) -> &'a mut Utf8Destination<'b> {
902        self.dest.write_surrogate_pair(high, low);
903        self.dest
904    }
905    #[inline(always)]
906    pub fn write_big5_combination(
907        self,
908        combined: u16,
909        combining: u16,
910    ) -> &'a mut Utf8Destination<'b> {
911        self.dest.write_big5_combination(combined, combining);
912        self.dest
913    }
914    #[inline(always)]
915    pub fn commit(self) -> &'a mut Utf8Destination<'b> {
916        self.dest
917    }
918}
919
920pub struct Utf8Destination<'a> {
921    slice: &'a mut [u8],
922    pos: usize,
923}
924
925impl<'a> Utf8Destination<'a> {
926    #[inline(always)]
927    pub fn new(dst: &mut [u8]) -> Utf8Destination<'_> {
928        Utf8Destination { slice: dst, pos: 0 }
929    }
930    #[inline(always)]
931    pub fn check_space_bmp<'b>(&'b mut self) -> Space<Utf8BmpHandle<'b, 'a>> {
932        if self.pos + 2 < self.slice.len() {
933            Space::Available(Utf8BmpHandle::new(self))
934        } else {
935            Space::Full(self.written())
936        }
937    }
938    #[inline(always)]
939    pub fn check_space_astral<'b>(&'b mut self) -> Space<Utf8AstralHandle<'b, 'a>> {
940        if self.pos + 3 < self.slice.len() {
941            Space::Available(Utf8AstralHandle::new(self))
942        } else {
943            Space::Full(self.written())
944        }
945    }
946    #[inline(always)]
947    pub fn written(&self) -> usize {
948        self.pos
949    }
950    #[inline(always)]
951    fn write_code_unit(&mut self, u: u8) {
952        unsafe {
953            // OK, because we checked before handing out a handle.
954            *(self.slice.get_unchecked_mut(self.pos)) = u;
955        }
956        self.pos += 1;
957    }
958    #[inline(always)]
959    fn write_ascii(&mut self, ascii: u8) {
960        debug_assert!(ascii < 0x80);
961        self.write_code_unit(ascii);
962    }
963    #[inline(always)]
964    fn write_bmp(&mut self, bmp: u16) {
965        if bmp < 0x80u16 {
966            self.write_ascii(bmp as u8);
967        } else if bmp < 0x800u16 {
968            self.write_mid_bmp(bmp);
969        } else {
970            self.write_upper_bmp(bmp);
971        }
972    }
973    #[inline(always)]
974    fn write_mid_bmp(&mut self, mid_bmp: u16) {
975        debug_assert!(mid_bmp >= 0x80);
976        debug_assert!(mid_bmp < 0x800);
977        self.write_code_unit(((mid_bmp >> 6) | 0xC0) as u8);
978        self.write_code_unit(((mid_bmp & 0x3F) | 0x80) as u8);
979    }
980    #[inline(always)]
981    fn write_upper_bmp(&mut self, upper_bmp: u16) {
982        debug_assert!(upper_bmp >= 0x800);
983        self.write_code_unit(((upper_bmp >> 12) | 0xE0) as u8);
984        self.write_code_unit((((upper_bmp & 0xFC0) >> 6) | 0x80) as u8);
985        self.write_code_unit(((upper_bmp & 0x3F) | 0x80) as u8);
986    }
987    #[inline(always)]
988    fn write_bmp_excl_ascii(&mut self, bmp: u16) {
989        if bmp < 0x800u16 {
990            self.write_mid_bmp(bmp);
991        } else {
992            self.write_upper_bmp(bmp);
993        }
994    }
995    #[inline(always)]
996    fn write_astral(&mut self, astral: u32) {
997        debug_assert!(astral > 0xFFFF);
998        debug_assert!(astral <= 0x10_FFFF);
999        self.write_code_unit(((astral >> 18) | 0xF0) as u8);
1000        self.write_code_unit((((astral & 0x3F000) >> 12) | 0x80) as u8);
1001        self.write_code_unit((((astral & 0xFC0) >> 6) | 0x80) as u8);
1002        self.write_code_unit(((astral & 0x3F) | 0x80) as u8);
1003    }
1004    #[inline(always)]
1005    pub fn write_surrogate_pair(&mut self, high: u16, low: u16) {
1006        self.write_astral(
1007            (u32::from(high) << 10) + u32::from(low)
1008                - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32),
1009        );
1010    }
1011    #[inline(always)]
1012    fn write_big5_combination(&mut self, combined: u16, combining: u16) {
1013        self.write_mid_bmp(combined);
1014        self.write_mid_bmp(combining);
1015    }
1016    #[inline(always)]
1017    pub fn copy_ascii_from_check_space_bmp<'b>(
1018        &'b mut self,
1019        source: &mut ByteSource,
1020    ) -> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf8BmpHandle<'b, 'a>)> {
1021        let non_ascii_ret = {
1022            let dst_len = self.slice.len();
1023            let src_remaining = &source.slice[source.pos..];
1024            let dst_remaining = &mut self.slice[self.pos..];
1025            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
1026                (DecoderResult::OutputFull, dst_remaining.len())
1027            } else {
1028                (DecoderResult::InputEmpty, src_remaining.len())
1029            };
1030            match ascii_to_ascii(src_remaining, dst_remaining) {
1031                None => {
1032                    source.pos += length;
1033                    self.pos += length;
1034                    return CopyAsciiResult::Stop((pending, source.pos, self.pos));
1035                }
1036                Some((non_ascii, consumed)) => {
1037                    source.pos += consumed;
1038                    self.pos += consumed;
1039                    if self.pos + 2 < dst_len {
1040                        source.pos += 1; // +1 for non_ascii
1041                        non_ascii
1042                    } else {
1043                        return CopyAsciiResult::Stop((
1044                            DecoderResult::OutputFull,
1045                            source.pos,
1046                            self.pos,
1047                        ));
1048                    }
1049                }
1050            }
1051        };
1052        CopyAsciiResult::GoOn((non_ascii_ret, Utf8BmpHandle::new(self)))
1053    }
1054    #[inline(always)]
1055    pub fn copy_ascii_from_check_space_astral<'b>(
1056        &'b mut self,
1057        source: &mut ByteSource,
1058    ) -> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf8AstralHandle<'b, 'a>)> {
1059        let non_ascii_ret = {
1060            let dst_len = self.slice.len();
1061            let src_remaining = &source.slice[source.pos..];
1062            let dst_remaining = &mut self.slice[self.pos..];
1063            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
1064                (DecoderResult::OutputFull, dst_remaining.len())
1065            } else {
1066                (DecoderResult::InputEmpty, src_remaining.len())
1067            };
1068            match ascii_to_ascii(src_remaining, dst_remaining) {
1069                None => {
1070                    source.pos += length;
1071                    self.pos += length;
1072                    return CopyAsciiResult::Stop((pending, source.pos, self.pos));
1073                }
1074                Some((non_ascii, consumed)) => {
1075                    source.pos += consumed;
1076                    self.pos += consumed;
1077                    if self.pos + 3 < dst_len {
1078                        source.pos += 1; // +1 for non_ascii
1079                        non_ascii
1080                    } else {
1081                        return CopyAsciiResult::Stop((
1082                            DecoderResult::OutputFull,
1083                            source.pos,
1084                            self.pos,
1085                        ));
1086                    }
1087                }
1088            }
1089        };
1090        CopyAsciiResult::GoOn((non_ascii_ret, Utf8AstralHandle::new(self)))
1091    }
1092    #[inline(always)]
1093    pub fn copy_utf8_up_to_invalid_from(&mut self, source: &mut ByteSource) {
1094        let src_remaining = &source.slice[source.pos..];
1095        let dst_remaining = &mut self.slice[self.pos..];
1096        let min_len = ::core::cmp::min(src_remaining.len(), dst_remaining.len());
1097        // Validate first, then memcpy to let memcpy do its thing even for
1098        // non-ASCII. (And potentially do something better than SSE2 for ASCII.)
1099        let valid_len = utf8_valid_up_to(&src_remaining[..min_len]);
1100        dst_remaining[..valid_len].copy_from_slice(&src_remaining[..valid_len]);
1101        source.pos += valid_len;
1102        self.pos += valid_len;
1103    }
1104    #[inline(always)]
1105    pub fn copy_utf16_from<E: Endian>(
1106        &mut self,
1107        source: &mut ByteSource,
1108    ) -> Option<(usize, usize)> {
1109        let src_remaining = &source.slice[source.pos..];
1110        let dst_remaining = &mut self.slice[self.pos..];
1111
1112        let mut src_unaligned =
1113            unsafe { UnalignedU16Slice::new(src_remaining.as_ptr(), src_remaining.len() / 2) };
1114        if src_unaligned.len() == 0 {
1115            return None;
1116        }
1117        let mut last_unit = src_unaligned.at(src_unaligned.len() - 1);
1118        if E::OPPOSITE_ENDIAN {
1119            last_unit = last_unit.swap_bytes();
1120        }
1121        if super::in_range16(last_unit, 0xD800, 0xDC00) {
1122            // Last code unit is a high surrogate. It might
1123            // legitimately form a pair later, so let's not
1124            // include it.
1125            src_unaligned.trim_last();
1126        }
1127        let (read, written, had_error) =
1128            convert_unaligned_utf16_to_utf8::<E>(src_unaligned, dst_remaining);
1129        source.pos += read * 2;
1130        self.pos += written;
1131        if had_error {
1132            Some((source.pos, self.pos))
1133        } else {
1134            None
1135        }
1136    }
1137}
1138
1139// UTF-16 source
1140
1141pub struct Utf16Source<'a> {
1142    slice: &'a [u16],
1143    pos: usize,
1144}
1145
1146impl<'a> Utf16Source<'a> {
1147    #[inline(always)]
1148    pub fn new(src: &'a [u16]) -> Utf16Source<'a> {
1149        Utf16Source { slice: src, pos: 0 }
1150    }
1151    #[inline(always)]
1152    pub fn check_available<'b>(&'b mut self) -> Space<Utf16ReadHandle<'b, 'a>> {
1153        if self.pos < self.slice.len() {
1154            Space::Available(Utf16ReadHandle::new(self))
1155        } else {
1156            Space::Full(self.consumed())
1157        }
1158    }
1159    #[allow(clippy::collapsible_if)]
1160    #[inline(always)]
1161    fn read(&mut self) -> char {
1162        let unit = self.slice[self.pos];
1163        self.pos += 1;
1164        let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
1165        if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
1166            return unsafe { ::core::char::from_u32_unchecked(u32::from(unit)) };
1167        }
1168        if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
1169            // high surrogate
1170            if self.pos < self.slice.len() {
1171                let second = self.slice[self.pos];
1172                let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
1173                if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
1174                    // The next code unit is a low surrogate. Advance position.
1175                    self.pos += 1;
1176                    return unsafe {
1177                        ::core::char::from_u32_unchecked(
1178                            (u32::from(unit) << 10) + u32::from(second)
1179                                - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32),
1180                        )
1181                    };
1182                }
1183                // The next code unit is not a low surrogate. Don't advance
1184                // position and treat the high surrogate as unpaired.
1185                // fall through
1186            }
1187            // Unpaired surrogate at the end of buffer, fall through
1188        }
1189        // Unpaired low surrogate
1190        '\u{FFFD}'
1191    }
1192    #[allow(clippy::collapsible_if)]
1193    #[inline(always)]
1194    fn read_enum(&mut self) -> Unicode {
1195        let unit = self.slice[self.pos];
1196        self.pos += 1;
1197        if unit < 0x80 {
1198            return Unicode::Ascii(unit as u8);
1199        }
1200        let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
1201        if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
1202            return Unicode::NonAscii(NonAscii::BmpExclAscii(unit));
1203        }
1204        if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
1205            // high surrogate
1206            if self.pos < self.slice.len() {
1207                let second = self.slice[self.pos];
1208                let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
1209                if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
1210                    // The next code unit is a low surrogate. Advance position.
1211                    self.pos += 1;
1212                    return Unicode::NonAscii(NonAscii::Astral(unsafe {
1213                        ::core::char::from_u32_unchecked(
1214                            (u32::from(unit) << 10) + u32::from(second)
1215                                - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32),
1216                        )
1217                    }));
1218                }
1219                // The next code unit is not a low surrogate. Don't advance
1220                // position and treat the high surrogate as unpaired.
1221                // fall through
1222            }
1223            // Unpaired surrogate at the end of buffer, fall through
1224        }
1225        // Unpaired low surrogate
1226        Unicode::NonAscii(NonAscii::BmpExclAscii(0xFFFDu16))
1227    }
1228    #[inline(always)]
1229    pub fn consumed(&self) -> usize {
1230        self.pos
1231    }
1232    #[inline(always)]
1233    pub fn copy_ascii_to_check_space_two<'b>(
1234        &mut self,
1235        dest: &'b mut ByteDestination<'a>,
1236    ) -> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteTwoHandle<'b, 'a>)> {
1237        let non_ascii_ret = {
1238            let src_remaining = &self.slice[self.pos..];
1239            let dst_remaining = dest.remaining();
1240            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
1241                (EncoderResult::OutputFull, dst_remaining.len())
1242            } else {
1243                (EncoderResult::InputEmpty, src_remaining.len())
1244            };
1245            match basic_latin_to_ascii(src_remaining, dst_remaining) {
1246                None => {
1247                    self.pos += length;
1248                    dest.advance(length);
1249                    return CopyAsciiResult::Stop((pending, self.pos, dest.written()));
1250                }
1251                Some((non_ascii, consumed)) => {
1252                    self.pos += consumed;
1253                    dest.advance(consumed);
1254                    if dest.remaining().len() >= 2 {
1255                        self.pos += 1; // commit to reading `non_ascii`
1256                        let unit = non_ascii;
1257                        let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
1258                        if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
1259                            NonAscii::BmpExclAscii(unit)
1260                        } else if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
1261                            // high surrogate
1262                            if self.pos < self.slice.len() {
1263                                let second = self.slice[self.pos];
1264                                let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
1265                                if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
1266                                    // The next code unit is a low surrogate. Advance position.
1267                                    self.pos += 1;
1268                                    NonAscii::Astral(unsafe {
1269                                        ::core::char::from_u32_unchecked(
1270                                            (u32::from(unit) << 10) + u32::from(second)
1271                                                - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32),
1272                                        )
1273                                    })
1274                                } else {
1275                                    // The next code unit is not a low surrogate. Don't advance
1276                                    // position and treat the high surrogate as unpaired.
1277                                    NonAscii::BmpExclAscii(0xFFFDu16)
1278                                }
1279                            } else {
1280                                // Unpaired surrogate at the end of the buffer.
1281                                NonAscii::BmpExclAscii(0xFFFDu16)
1282                            }
1283                        } else {
1284                            // Unpaired low surrogate
1285                            NonAscii::BmpExclAscii(0xFFFDu16)
1286                        }
1287                    } else {
1288                        return CopyAsciiResult::Stop((
1289                            EncoderResult::OutputFull,
1290                            self.pos,
1291                            dest.written(),
1292                        ));
1293                    }
1294                }
1295            }
1296        };
1297        CopyAsciiResult::GoOn((non_ascii_ret, ByteTwoHandle::new(dest)))
1298    }
1299    #[inline(always)]
1300    pub fn copy_ascii_to_check_space_four<'b>(
1301        &mut self,
1302        dest: &'b mut ByteDestination<'a>,
1303    ) -> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteFourHandle<'b, 'a>)> {
1304        let non_ascii_ret = {
1305            let src_remaining = &self.slice[self.pos..];
1306            let dst_remaining = dest.remaining();
1307            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
1308                (EncoderResult::OutputFull, dst_remaining.len())
1309            } else {
1310                (EncoderResult::InputEmpty, src_remaining.len())
1311            };
1312            match basic_latin_to_ascii(src_remaining, dst_remaining) {
1313                None => {
1314                    self.pos += length;
1315                    dest.advance(length);
1316                    return CopyAsciiResult::Stop((pending, self.pos, dest.written()));
1317                }
1318                Some((non_ascii, consumed)) => {
1319                    self.pos += consumed;
1320                    dest.advance(consumed);
1321                    if dest.remaining().len() >= 4 {
1322                        self.pos += 1; // commit to reading `non_ascii`
1323                        let unit = non_ascii;
1324                        let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
1325                        if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
1326                            NonAscii::BmpExclAscii(unit)
1327                        } else if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
1328                            // high surrogate
1329                            if self.pos == self.slice.len() {
1330                                // Unpaired surrogate at the end of the buffer.
1331                                NonAscii::BmpExclAscii(0xFFFDu16)
1332                            } else {
1333                                let second = self.slice[self.pos];
1334                                let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
1335                                if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
1336                                    // The next code unit is a low surrogate. Advance position.
1337                                    self.pos += 1;
1338                                    NonAscii::Astral(unsafe {
1339                                        ::core::char::from_u32_unchecked(
1340                                            (u32::from(unit) << 10) + u32::from(second)
1341                                                - (((0xD800u32 << 10) - 0x1_0000u32) + 0xDC00u32),
1342                                        )
1343                                    })
1344                                } else {
1345                                    // The next code unit is not a low surrogate. Don't advance
1346                                    // position and treat the high surrogate as unpaired.
1347                                    NonAscii::BmpExclAscii(0xFFFDu16)
1348                                }
1349                            }
1350                        } else {
1351                            // Unpaired low surrogate
1352                            NonAscii::BmpExclAscii(0xFFFDu16)
1353                        }
1354                    } else {
1355                        return CopyAsciiResult::Stop((
1356                            EncoderResult::OutputFull,
1357                            self.pos,
1358                            dest.written(),
1359                        ));
1360                    }
1361                }
1362            }
1363        };
1364        CopyAsciiResult::GoOn((non_ascii_ret, ByteFourHandle::new(dest)))
1365    }
1366}
1367
1368pub struct Utf16ReadHandle<'a, 'b>
1369where
1370    'b: 'a,
1371{
1372    source: &'a mut Utf16Source<'b>,
1373}
1374
1375impl<'a, 'b> Utf16ReadHandle<'a, 'b>
1376where
1377    'b: 'a,
1378{
1379    #[inline(always)]
1380    fn new(src: &'a mut Utf16Source<'b>) -> Utf16ReadHandle<'a, 'b> {
1381        Utf16ReadHandle { source: src }
1382    }
1383    #[inline(always)]
1384    pub fn read(self) -> (char, Utf16UnreadHandle<'a, 'b>) {
1385        Utf16UnreadHandle::new_char(self.source)
1386    }
1387    #[inline(always)]
1388    pub fn read_enum(self) -> (Unicode, Utf16UnreadHandle<'a, 'b>) {
1389        Utf16UnreadHandle::new_enum(self.source)
1390    }
1391    #[inline(always)]
1392    pub fn consumed(&self) -> usize {
1393        self.source.consumed()
1394    }
1395}
1396
1397pub struct Utf16UnreadHandle<'a, 'b>
1398where
1399    'b: 'a,
1400{
1401    source: &'a mut Utf16Source<'b>,
1402    old_pos: usize,
1403}
1404
1405impl<'a, 'b> Utf16UnreadHandle<'a, 'b>
1406where
1407    'b: 'a,
1408{
1409    #[inline(always)]
1410    fn new_char(source: &'a mut Utf16Source<'b>) -> (char, Self) {
1411        let old_pos = source.pos;
1412        let character = source.read();
1413        (character, Self { source, old_pos })
1414    }
1415    #[inline(always)]
1416    fn new_enum(source: &'a mut Utf16Source<'b>) -> (Unicode, Self) {
1417        let old_pos = source.pos;
1418        let character = source.read_enum();
1419        (character, Self { source, old_pos })
1420    }
1421
1422    #[inline(always)]
1423    pub fn unread(self) -> usize {
1424        self.source.pos = self.old_pos;
1425        self.old_pos
1426    }
1427    #[inline(always)]
1428    pub fn consumed(&self) -> usize {
1429        self.source.consumed()
1430    }
1431    #[inline(always)]
1432    pub fn commit(self) -> &'a mut Utf16Source<'b> {
1433        self.source
1434    }
1435}
1436
1437// UTF-8 source
1438
1439pub struct Utf8Source<'a> {
1440    slice: &'a [u8],
1441    pos: usize,
1442}
1443
1444impl<'a> Utf8Source<'a> {
1445    #[inline(always)]
1446    pub fn new(src: &'a str) -> Utf8Source<'a> {
1447        Utf8Source {
1448            slice: src.as_bytes(),
1449            pos: 0,
1450        }
1451    }
1452    #[inline(always)]
1453    pub fn check_available<'b>(&'b mut self) -> Space<Utf8ReadHandle<'b, 'a>> {
1454        if self.pos < self.slice.len() {
1455            Space::Available(Utf8ReadHandle::new(self))
1456        } else {
1457            Space::Full(self.consumed())
1458        }
1459    }
1460    #[inline(always)]
1461    fn read(&mut self) -> char {
1462        let unit = self.slice[self.pos];
1463        if unit < 0x80 {
1464            self.pos += 1;
1465            return char::from(unit);
1466        }
1467        if unit < 0xE0 {
1468            let point =
1469                ((u32::from(unit) & 0x1F) << 6) | (u32::from(self.slice[self.pos + 1]) & 0x3F);
1470            self.pos += 2;
1471            return unsafe { ::core::char::from_u32_unchecked(point) };
1472        }
1473        if unit < 0xF0 {
1474            let point = ((u32::from(unit) & 0xF) << 12)
1475                | ((u32::from(self.slice[self.pos + 1]) & 0x3F) << 6)
1476                | (u32::from(self.slice[self.pos + 2]) & 0x3F);
1477            self.pos += 3;
1478            return unsafe { ::core::char::from_u32_unchecked(point) };
1479        }
1480        let point = ((u32::from(unit) & 0x7) << 18)
1481            | ((u32::from(self.slice[self.pos + 1]) & 0x3F) << 12)
1482            | ((u32::from(self.slice[self.pos + 2]) & 0x3F) << 6)
1483            | (u32::from(self.slice[self.pos + 3]) & 0x3F);
1484        self.pos += 4;
1485        unsafe { ::core::char::from_u32_unchecked(point) }
1486    }
1487    #[inline(always)]
1488    fn read_enum(&mut self) -> Unicode {
1489        let unit = self.slice[self.pos];
1490        if unit < 0x80 {
1491            self.pos += 1;
1492            return Unicode::Ascii(unit);
1493        }
1494        if unit < 0xE0 {
1495            let point =
1496                ((u16::from(unit) & 0x1F) << 6) | (u16::from(self.slice[self.pos + 1]) & 0x3F);
1497            self.pos += 2;
1498            return Unicode::NonAscii(NonAscii::BmpExclAscii(point));
1499        }
1500        if unit < 0xF0 {
1501            let point = ((u16::from(unit) & 0xF) << 12)
1502                | ((u16::from(self.slice[self.pos + 1]) & 0x3F) << 6)
1503                | (u16::from(self.slice[self.pos + 2]) & 0x3F);
1504            self.pos += 3;
1505            return Unicode::NonAscii(NonAscii::BmpExclAscii(point));
1506        }
1507        let point = ((u32::from(unit) & 0x7) << 18)
1508            | ((u32::from(self.slice[self.pos + 1]) & 0x3F) << 12)
1509            | ((u32::from(self.slice[self.pos + 2]) & 0x3F) << 6)
1510            | (u32::from(self.slice[self.pos + 3]) & 0x3F);
1511        self.pos += 4;
1512        Unicode::NonAscii(NonAscii::Astral(unsafe {
1513            ::core::char::from_u32_unchecked(point)
1514        }))
1515    }
1516    #[inline(always)]
1517    pub fn consumed(&self) -> usize {
1518        self.pos
1519    }
1520    #[inline(always)]
1521    pub fn copy_ascii_to_check_space_one<'b>(
1522        &mut self,
1523        dest: &'b mut ByteDestination<'a>,
1524    ) -> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteOneHandle<'b, 'a>)> {
1525        let non_ascii_ret = {
1526            let src_remaining = &self.slice[self.pos..];
1527            let dst_remaining = dest.remaining();
1528            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
1529                (EncoderResult::OutputFull, dst_remaining.len())
1530            } else {
1531                (EncoderResult::InputEmpty, src_remaining.len())
1532            };
1533            match ascii_to_ascii(src_remaining, dst_remaining) {
1534                None => {
1535                    self.pos += length;
1536                    dest.advance(length);
1537                    return CopyAsciiResult::Stop((pending, self.pos, dest.written()));
1538                }
1539                Some((non_ascii, consumed)) => {
1540                    self.pos += consumed;
1541                    dest.advance(consumed);
1542                    // We don't need to check space in destination, because
1543                    // `ascii_to_ascii()` already did.
1544                    if non_ascii < 0xE0 {
1545                        let point = ((u16::from(non_ascii) & 0x1F) << 6)
1546                            | (u16::from(self.slice[self.pos + 1]) & 0x3F);
1547                        self.pos += 2;
1548                        NonAscii::BmpExclAscii(point)
1549                    } else if non_ascii < 0xF0 {
1550                        let point = ((u16::from(non_ascii) & 0xF) << 12)
1551                            | ((u16::from(self.slice[self.pos + 1]) & 0x3F) << 6)
1552                            | (u16::from(self.slice[self.pos + 2]) & 0x3F);
1553                        self.pos += 3;
1554                        NonAscii::BmpExclAscii(point)
1555                    } else {
1556                        let point = ((u32::from(non_ascii) & 0x7) << 18)
1557                            | ((u32::from(self.slice[self.pos + 1]) & 0x3F) << 12)
1558                            | ((u32::from(self.slice[self.pos + 2]) & 0x3F) << 6)
1559                            | (u32::from(self.slice[self.pos + 3]) & 0x3F);
1560                        self.pos += 4;
1561                        NonAscii::Astral(unsafe { ::core::char::from_u32_unchecked(point) })
1562                    }
1563                }
1564            }
1565        };
1566        CopyAsciiResult::GoOn((non_ascii_ret, ByteOneHandle::new(dest)))
1567    }
1568    #[inline(always)]
1569    pub fn copy_ascii_to_check_space_two<'b>(
1570        &mut self,
1571        dest: &'b mut ByteDestination<'a>,
1572    ) -> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteTwoHandle<'b, 'a>)> {
1573        let non_ascii_ret = {
1574            let src_remaining = &self.slice[self.pos..];
1575            let dst_remaining = dest.remaining();
1576            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
1577                (EncoderResult::OutputFull, dst_remaining.len())
1578            } else {
1579                (EncoderResult::InputEmpty, src_remaining.len())
1580            };
1581            match ascii_to_ascii(src_remaining, dst_remaining) {
1582                None => {
1583                    self.pos += length;
1584                    dest.advance(length);
1585                    return CopyAsciiResult::Stop((pending, self.pos, dest.written()));
1586                }
1587                Some((non_ascii, consumed)) => {
1588                    self.pos += consumed;
1589                    dest.advance(consumed);
1590                    if dest.remaining().len() >= 2 {
1591                        if non_ascii < 0xE0 {
1592                            let point = ((u16::from(non_ascii) & 0x1F) << 6)
1593                                | (u16::from(self.slice[self.pos + 1]) & 0x3F);
1594                            self.pos += 2;
1595                            NonAscii::BmpExclAscii(point)
1596                        } else if non_ascii < 0xF0 {
1597                            let point = ((u16::from(non_ascii) & 0xF) << 12)
1598                                | ((u16::from(self.slice[self.pos + 1]) & 0x3F) << 6)
1599                                | (u16::from(self.slice[self.pos + 2]) & 0x3F);
1600                            self.pos += 3;
1601                            NonAscii::BmpExclAscii(point)
1602                        } else {
1603                            let point = ((u32::from(non_ascii) & 0x7) << 18)
1604                                | ((u32::from(self.slice[self.pos + 1]) & 0x3F) << 12)
1605                                | ((u32::from(self.slice[self.pos + 2]) & 0x3F) << 6)
1606                                | (u32::from(self.slice[self.pos + 3]) & 0x3F);
1607                            self.pos += 4;
1608                            NonAscii::Astral(unsafe { ::core::char::from_u32_unchecked(point) })
1609                        }
1610                    } else {
1611                        return CopyAsciiResult::Stop((
1612                            EncoderResult::OutputFull,
1613                            self.pos,
1614                            dest.written(),
1615                        ));
1616                    }
1617                }
1618            }
1619        };
1620        CopyAsciiResult::GoOn((non_ascii_ret, ByteTwoHandle::new(dest)))
1621    }
1622    #[inline(always)]
1623    pub fn copy_ascii_to_check_space_four<'b>(
1624        &mut self,
1625        dest: &'b mut ByteDestination<'a>,
1626    ) -> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteFourHandle<'b, 'a>)> {
1627        let non_ascii_ret = {
1628            let src_remaining = &self.slice[self.pos..];
1629            let dst_remaining = dest.remaining();
1630            let (pending, length) = if dst_remaining.len() < src_remaining.len() {
1631                (EncoderResult::OutputFull, dst_remaining.len())
1632            } else {
1633                (EncoderResult::InputEmpty, src_remaining.len())
1634            };
1635            match ascii_to_ascii(src_remaining, dst_remaining) {
1636                None => {
1637                    self.pos += length;
1638                    dest.advance(length);
1639                    return CopyAsciiResult::Stop((pending, self.pos, dest.written()));
1640                }
1641                Some((non_ascii, consumed)) => {
1642                    self.pos += consumed;
1643                    dest.advance(consumed);
1644                    if dest.remaining().len() >= 4 {
1645                        if non_ascii < 0xE0 {
1646                            let point = ((u16::from(non_ascii) & 0x1F) << 6)
1647                                | (u16::from(self.slice[self.pos + 1]) & 0x3F);
1648                            self.pos += 2;
1649                            NonAscii::BmpExclAscii(point)
1650                        } else if non_ascii < 0xF0 {
1651                            let point = ((u16::from(non_ascii) & 0xF) << 12)
1652                                | ((u16::from(self.slice[self.pos + 1]) & 0x3F) << 6)
1653                                | (u16::from(self.slice[self.pos + 2]) & 0x3F);
1654                            self.pos += 3;
1655                            NonAscii::BmpExclAscii(point)
1656                        } else {
1657                            let point = ((u32::from(non_ascii) & 0x7) << 18)
1658                                | ((u32::from(self.slice[self.pos + 1]) & 0x3F) << 12)
1659                                | ((u32::from(self.slice[self.pos + 2]) & 0x3F) << 6)
1660                                | (u32::from(self.slice[self.pos + 3]) & 0x3F);
1661                            self.pos += 4;
1662                            NonAscii::Astral(unsafe { ::core::char::from_u32_unchecked(point) })
1663                        }
1664                    } else {
1665                        return CopyAsciiResult::Stop((
1666                            EncoderResult::OutputFull,
1667                            self.pos,
1668                            dest.written(),
1669                        ));
1670                    }
1671                }
1672            }
1673        };
1674        CopyAsciiResult::GoOn((non_ascii_ret, ByteFourHandle::new(dest)))
1675    }
1676}
1677
1678pub struct Utf8ReadHandle<'a, 'b>
1679where
1680    'b: 'a,
1681{
1682    source: &'a mut Utf8Source<'b>,
1683}
1684
1685impl<'a, 'b> Utf8ReadHandle<'a, 'b>
1686where
1687    'b: 'a,
1688{
1689    #[inline(always)]
1690    fn new(source: &'a mut Utf8Source<'b>) -> Utf8ReadHandle<'a, 'b> {
1691        Utf8ReadHandle { source }
1692    }
1693    #[inline(always)]
1694    pub fn read(self) -> (char, Utf8UnreadHandle<'a, 'b>) {
1695        Utf8UnreadHandle::new_char(self.source)
1696    }
1697    #[inline(always)]
1698    pub fn read_enum(self) -> (Unicode, Utf8UnreadHandle<'a, 'b>) {
1699        Utf8UnreadHandle::new_enum(self.source)
1700    }
1701    #[inline(always)]
1702    pub fn consumed(&self) -> usize {
1703        self.source.consumed()
1704    }
1705}
1706
1707pub struct Utf8UnreadHandle<'a, 'b>
1708where
1709    'b: 'a,
1710{
1711    source: &'a mut Utf8Source<'b>,
1712    old_pos: usize,
1713}
1714
1715impl<'a, 'b> Utf8UnreadHandle<'a, 'b>
1716where
1717    'b: 'a,
1718{
1719    #[inline(always)]
1720    fn new_char(source: &'a mut Utf8Source<'b>) -> (char, Self) {
1721        let old_pos = source.pos;
1722        let character = source.read();
1723        (character, Self { source, old_pos })
1724    }
1725    #[inline(always)]
1726    fn new_enum(source: &'a mut Utf8Source<'b>) -> (Unicode, Self) {
1727        let old_pos = source.pos;
1728        let character = source.read_enum();
1729        (character, Self { source, old_pos })
1730    }
1731    #[inline(always)]
1732    pub fn unread(self) -> usize {
1733        self.source.pos = self.old_pos;
1734        self.old_pos
1735    }
1736    #[inline(always)]
1737    pub fn consumed(&self) -> usize {
1738        self.source.consumed()
1739    }
1740    #[inline(always)]
1741    pub fn commit(self) -> &'a mut Utf8Source<'b> {
1742        self.source
1743    }
1744}
1745
1746// Byte destination
1747
1748pub struct ByteOneHandle<'a, 'b>
1749where
1750    'b: 'a,
1751{
1752    dest: &'a mut ByteDestination<'b>,
1753}
1754
1755impl<'a, 'b> ByteOneHandle<'a, 'b>
1756where
1757    'b: 'a,
1758{
1759    #[inline(always)]
1760    fn new(dst: &'a mut ByteDestination<'b>) -> ByteOneHandle<'a, 'b> {
1761        ByteOneHandle { dest: dst }
1762    }
1763    #[inline(always)]
1764    pub fn written(&self) -> usize {
1765        self.dest.written()
1766    }
1767    #[inline(always)]
1768    pub fn write_one(self, first: u8) -> &'a mut ByteDestination<'b> {
1769        self.dest.write_one(first);
1770        self.dest
1771    }
1772}
1773
1774pub struct ByteTwoHandle<'a, 'b>
1775where
1776    'b: 'a,
1777{
1778    dest: &'a mut ByteDestination<'b>,
1779}
1780
1781impl<'a, 'b> ByteTwoHandle<'a, 'b>
1782where
1783    'b: 'a,
1784{
1785    #[inline(always)]
1786    fn new(dst: &'a mut ByteDestination<'b>) -> ByteTwoHandle<'a, 'b> {
1787        ByteTwoHandle { dest: dst }
1788    }
1789    #[inline(always)]
1790    pub fn written(&self) -> usize {
1791        self.dest.written()
1792    }
1793    #[inline(always)]
1794    pub fn write_one(self, first: u8) -> &'a mut ByteDestination<'b> {
1795        self.dest.write_one(first);
1796        self.dest
1797    }
1798    #[inline(always)]
1799    pub fn write_two(self, first: u8, second: u8) -> &'a mut ByteDestination<'b> {
1800        self.dest.write_two(first, second);
1801        self.dest
1802    }
1803}
1804
1805pub struct ByteThreeHandle<'a, 'b>
1806where
1807    'b: 'a,
1808{
1809    dest: &'a mut ByteDestination<'b>,
1810}
1811
1812impl<'a, 'b> ByteThreeHandle<'a, 'b>
1813where
1814    'b: 'a,
1815{
1816    #[inline(always)]
1817    fn new(dst: &'a mut ByteDestination<'b>) -> ByteThreeHandle<'a, 'b> {
1818        ByteThreeHandle { dest: dst }
1819    }
1820    #[inline(always)]
1821    pub fn written(&self) -> usize {
1822        self.dest.written()
1823    }
1824    #[inline(always)]
1825    pub fn write_one(self, first: u8) -> &'a mut ByteDestination<'b> {
1826        self.dest.write_one(first);
1827        self.dest
1828    }
1829    #[inline(always)]
1830    pub fn write_two(self, first: u8, second: u8) -> &'a mut ByteDestination<'b> {
1831        self.dest.write_two(first, second);
1832        self.dest
1833    }
1834    #[inline(always)]
1835    pub fn write_three(self, first: u8, second: u8, third: u8) -> &'a mut ByteDestination<'b> {
1836        self.dest.write_three(first, second, third);
1837        self.dest
1838    }
1839    #[inline(always)]
1840    pub fn write_three_return_written(self, first: u8, second: u8, third: u8) -> usize {
1841        self.dest.write_three(first, second, third);
1842        self.dest.written()
1843    }
1844}
1845
1846pub struct ByteFourHandle<'a, 'b>
1847where
1848    'b: 'a,
1849{
1850    dest: &'a mut ByteDestination<'b>,
1851}
1852
1853impl<'a, 'b> ByteFourHandle<'a, 'b>
1854where
1855    'b: 'a,
1856{
1857    #[inline(always)]
1858    fn new(dst: &'a mut ByteDestination<'b>) -> ByteFourHandle<'a, 'b> {
1859        ByteFourHandle { dest: dst }
1860    }
1861    #[inline(always)]
1862    pub fn written(&self) -> usize {
1863        self.dest.written()
1864    }
1865    #[inline(always)]
1866    pub fn write_one(self, first: u8) -> &'a mut ByteDestination<'b> {
1867        self.dest.write_one(first);
1868        self.dest
1869    }
1870    #[inline(always)]
1871    pub fn write_two(self, first: u8, second: u8) -> &'a mut ByteDestination<'b> {
1872        self.dest.write_two(first, second);
1873        self.dest
1874    }
1875    #[inline(always)]
1876    pub fn write_four(
1877        self,
1878        first: u8,
1879        second: u8,
1880        third: u8,
1881        fourth: u8,
1882    ) -> &'a mut ByteDestination<'b> {
1883        self.dest.write_four(first, second, third, fourth);
1884        self.dest
1885    }
1886}
1887
1888pub struct ByteDestination<'a> {
1889    slice: &'a mut [u8],
1890    /// Pointer to the original start of the slice. It's never dereferenced.
1891    start: *const u8,
1892}
1893
1894impl<'a> ByteDestination<'a> {
1895    #[inline(always)]
1896    pub fn new(dst: &mut [u8]) -> ByteDestination<'_> {
1897        ByteDestination {
1898            start: dst.as_ptr(),
1899            slice: dst,
1900        }
1901    }
1902    #[inline(always)]
1903    pub fn remaining(&mut self) -> &mut [u8] {
1904        self.slice
1905    }
1906    #[inline(always)]
1907    pub fn check_space_one<'b>(&'b mut self) -> Space<ByteOneHandle<'b, 'a>> {
1908        if !self.slice.is_empty() {
1909            Space::Available(ByteOneHandle::new(self))
1910        } else {
1911            Space::Full(self.written())
1912        }
1913    }
1914    #[inline(always)]
1915    pub fn check_space_two<'b>(&'b mut self) -> Space<ByteTwoHandle<'b, 'a>> {
1916        if self.slice.len() >= 2 {
1917            Space::Available(ByteTwoHandle::new(self))
1918        } else {
1919            Space::Full(self.written())
1920        }
1921    }
1922    #[inline(always)]
1923    pub fn check_space_three<'b>(&'b mut self) -> Space<ByteThreeHandle<'b, 'a>> {
1924        if self.slice.len() >= 3 {
1925            Space::Available(ByteThreeHandle::new(self))
1926        } else {
1927            Space::Full(self.written())
1928        }
1929    }
1930    #[inline(always)]
1931    pub fn check_space_four<'b>(&'b mut self) -> Space<ByteFourHandle<'b, 'a>> {
1932        if self.slice.len() >= 4 {
1933            Space::Available(ByteFourHandle::new(self))
1934        } else {
1935            Space::Full(self.written())
1936        }
1937    }
1938    #[inline(always)]
1939    pub fn written(&self) -> usize {
1940        // ptr::byte_offset_from(), but safe
1941        self.slice.as_ptr() as usize - self.start as usize
1942    }
1943    #[inline(always)]
1944    fn write_one(&mut self, first: u8) {
1945        // take() is necessary to use the slice's full lifetime, rather than a shorter reborrow via self
1946        let (dst, rest) = core::mem::take(&mut self.slice).split_first_mut().unwrap();
1947        self.slice = rest;
1948
1949        *dst = first;
1950    }
1951    #[inline(always)]
1952    fn write_two(&mut self, first: u8, second: u8) {
1953        let (dst, rest) = core::mem::take(&mut self.slice).split_at_mut(2);
1954        self.slice = rest;
1955
1956        dst[0] = first;
1957        dst[1] = second;
1958    }
1959    #[inline(always)]
1960    fn write_three(&mut self, first: u8, second: u8, third: u8) {
1961        let (dst, rest) = core::mem::take(&mut self.slice).split_at_mut(3);
1962        self.slice = rest;
1963
1964        dst[0] = first;
1965        dst[1] = second;
1966        dst[2] = third;
1967    }
1968    #[inline(always)]
1969    fn write_four(&mut self, first: u8, second: u8, third: u8, fourth: u8) {
1970        // consecutive assignments to self.slice[pos+n] would have four bounds checks
1971        let (dst, rest) = core::mem::take(&mut self.slice).split_at_mut(4);
1972        self.slice = rest;
1973
1974        dst[0] = first;
1975        dst[1] = second;
1976        dst[2] = third;
1977        dst[3] = fourth;
1978    }
1979    /// Assume this many bytes have been written
1980    #[inline(always)]
1981    pub fn advance(&mut self, length: usize) {
1982        self.slice = &mut core::mem::take(&mut self.slice)[length..];
1983    }
1984}
encoding_rs/handles.rs

encoding_rs/
handles.rs
