encoding_rs/
utf_8.rs

1// Copyright Mozilla Foundation. See the COPYRIGHT
2// file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use super::*;
11use crate::ascii::ascii_to_basic_latin;
12use crate::ascii::basic_latin_to_ascii;
13use crate::ascii::validate_ascii;
14use crate::handles::*;
15use crate::mem::convert_utf16_to_utf8_partial;
16use crate::variant::*;
17
18cfg_if! {
19    if #[cfg(feature = "simd-accel")] {
20        use ::core::intrinsics::unlikely;
21        use ::core::intrinsics::likely;
22    } else {
23        #[inline(always)]
24        fn unlikely(b: bool) -> bool {
25            b
26        }
27        #[inline(always)]
28        fn likely(b: bool) -> bool {
29            b
30        }
31    }
32}
33
34#[repr(align(64))] // Align to cache lines
35pub struct Utf8Data {
36    pub table: [u8; 384],
37}
38
39// BEGIN GENERATED CODE. PLEASE DO NOT EDIT.
40// Instead, please regenerate using generate-encoding-data.py
41
42pub static UTF8_DATA: Utf8Data = Utf8Data {
43    table: [
44        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
45        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
46        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
47        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
48        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
49        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
50        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
51        252, 252, 84, 84, 84, 84, 84, 84, 84, 84, 84, 84, 84, 84, 84, 84, 84, 84, 148, 148, 148,
52        148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 164, 164, 164, 164, 164,
53        164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
54        164, 164, 164, 164, 164, 164, 164, 164, 164, 252, 252, 252, 252, 252, 252, 252, 252, 252,
55        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
56        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
57        252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252,
58        252, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
59        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
60        4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
61        8, 8, 8, 8, 8, 8, 8, 16, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 32, 8, 8, 64, 8, 8, 8, 128, 4,
62        4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
63    ],
64};
65
66// END GENERATED CODE
67
68// The UTF-8 validation code provided by this crate is faster than the UTF-8 validation code
69// provided by the standard library. When SIMD is used, the simdutf8 crate is much faster
70// than the code here. However, when SIMD isn't used, simdutf8 delegates to the standard
71// library and not here. To use the code here when SSE 4.2 isn't available for simdutf8
72// to use, let's implement custom dispatch here. As a bonus, the `core_detect` crate
73// works without `std`.
74
75cfg_if! {
76    if #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] {
77        #[inline(always)]
78        fn fast_utf8_valid_up_to(src: &[u8]) -> Option<usize> {
79            if src.len() >= 64 {
80                // SAFETY: The cfg check above ensures that the precondition, NEON availability on aarch64, is satisfied.
81                Some(match unsafe { simdutf8::compat::imp::aarch64::neon::validate_utf8(src) } {
82                    Ok(_) => src.len(),
83                    Err(e) => e.valid_up_to(),
84                })
85            } else {
86                None
87            }
88        }
89    } else if #[cfg(target_feature = "avx2")] {
90        #[inline(always)]
91        fn fast_utf8_valid_up_to(src: &[u8]) -> Option<usize> {
92            if src.len() >= 64 {
93                // SAFETY: The cfg check above ensures that the precondition, AVX2 availability, is satisfied.
94                Some(match unsafe { simdutf8::compat::imp::x86::avx2::validate_utf8(src) } {
95                    Ok(_) => src.len(),
96                    Err(e) => e.valid_up_to(),
97                })
98            } else {
99                None
100            }
101        }
102    } else if #[cfg(target_feature = "sse4.2")] {
103        #[inline(always)]
104        fn fast_utf8_valid_up_to(src: &[u8]) -> Option<usize> {
105            if src.len() >= 64 {
106                if core_detect::is_x86_feature_detected!("avx2") {
107                    // SAFETY: The dynamic check above ensures that the precondition, AVX2 availability, is satisfied.
108                    Some(match unsafe { simdutf8::compat::imp::x86::avx2::validate_utf8(src) } {
109                        Ok(_) => src.len(),
110                        Err(e) => e.valid_up_to(),
111                    })
112                } else {
113                    // SAFETY: The cfg check above ensures that the precondition, SSE 4.2 availability, is satisfied.
114                    Some(match unsafe { simdutf8::compat::imp::x86::sse42::validate_utf8(src) } {
115                        Ok(_) => src.len(),
116                        Err(e) => e.valid_up_to(),
117                    })
118                }
119            } else {
120                None
121            }
122        }
123    } else if #[cfg(target_feature = "sse")] { // "sse" stands in for cpuid availability
124        #[inline(always)]
125        fn fast_utf8_valid_up_to(src: &[u8]) -> Option<usize> {
126            if src.len() >= 64 {
127                if core_detect::is_x86_feature_detected!("avx2") {
128                    // SAFETY: The dynamic check above ensures that the precondition, AVX2 availability, is satisfied.
129                    Some(match unsafe { simdutf8::compat::imp::x86::avx2::validate_utf8(src) } {
130                        Ok(_) => src.len(),
131                        Err(e) => e.valid_up_to(),
132                    })
133                } else if core_detect::is_x86_feature_detected!("sse4.2") {
134                    // SAFETY: The dynamic check above ensures that the precondition, SSE 4.2 availability, is satisfied.
135                    Some(match unsafe { simdutf8::compat::imp::x86::sse42::validate_utf8(src) } {
136                        Ok(_) => src.len(),
137                        Err(e) => e.valid_up_to(),
138                    })
139                } else {
140                    None
141                }
142            } else {
143                None
144            }
145        }
146    } else if #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] {
147        #[inline(always)]
148        fn fast_utf8_valid_up_to(src: &[u8]) -> Option<usize> {
149            if src.len() >= 64 {
150                // SAFETY: The cfg check above ensures that the precondition, simd128 availability on wasm32, is satisfied.
151                Some(match unsafe { simdutf8::compat::imp::wasm32::simd128::validate_utf8(src) } {
152                    Ok(_) => src.len(),
153                    Err(e) => e.valid_up_to(),
154                })
155            } else {
156                None
157            }
158        }
159    } else {
160        #[inline(always)]
161        fn fast_utf8_valid_up_to(_src: &[u8]) -> Option<usize> {
162            None
163        }
164    }
165}
166
167pub fn utf8_valid_up_to(src: &[u8]) -> usize {
168    if let Some(up_to) = fast_utf8_valid_up_to(src) {
169        return up_to;
170    }
171
172    let mut read = 0;
173    'outer: loop {
174        let mut byte = {
175            let src_remaining = &src[read..];
176            match validate_ascii(src_remaining) {
177                None => {
178                    return src.len();
179                }
180                Some((non_ascii, consumed)) => {
181                    read += consumed;
182                    non_ascii
183                }
184            }
185        };
186        // Check for the longest sequence to avoid checking twice for the
187        // multi-byte sequences. This can't overflow with 64-bit address space,
188        // because full 64 bits aren't in use. In the 32-bit PAE case, for this
189        // to overflow would mean that the source slice would be so large that
190        // the address space of the process would not have space for any code.
191        // Therefore, the slice cannot be so long that this would overflow.
192        if likely(read + 4 <= src.len()) {
193            'inner: loop {
194                // At this point, `byte` is not included in `read`, because we
195                // don't yet know that a) the UTF-8 sequence is valid and b) that there
196                // is output space if it is an astral sequence.
197                // Inspecting the lead byte directly is faster than what the
198                // std lib does!
199                if likely(in_inclusive_range8(byte, 0xC2, 0xDF)) {
200                    // Two-byte
201                    let second = unsafe { *(src.get_unchecked(read + 1)) };
202                    if !in_inclusive_range8(second, 0x80, 0xBF) {
203                        break 'outer;
204                    }
205                    read += 2;
206
207                    // Next lead (manually inlined)
208                    if likely(read + 4 <= src.len()) {
209                        byte = unsafe { *(src.get_unchecked(read)) };
210                        if byte < 0x80 {
211                            read += 1;
212                            continue 'outer;
213                        }
214                        continue 'inner;
215                    }
216                    break 'inner;
217                }
218                if likely(byte < 0xF0) {
219                    'three: loop {
220                        // Three-byte
221                        let second = unsafe { *(src.get_unchecked(read + 1)) };
222                        let third = unsafe { *(src.get_unchecked(read + 2)) };
223                        if ((UTF8_DATA.table[usize::from(second)]
224                            & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
225                            | (third >> 6))
226                            != 2
227                        {
228                            break 'outer;
229                        }
230                        read += 3;
231
232                        // Next lead (manually inlined)
233                        if likely(read + 4 <= src.len()) {
234                            byte = unsafe { *(src.get_unchecked(read)) };
235                            if in_inclusive_range8(byte, 0xE0, 0xEF) {
236                                continue 'three;
237                            }
238                            if likely(byte < 0x80) {
239                                read += 1;
240                                continue 'outer;
241                            }
242                            continue 'inner;
243                        }
244                        break 'inner;
245                    }
246                }
247                // Four-byte
248                let second = unsafe { *(src.get_unchecked(read + 1)) };
249                let third = unsafe { *(src.get_unchecked(read + 2)) };
250                let fourth = unsafe { *(src.get_unchecked(read + 3)) };
251                if (u16::from(
252                    UTF8_DATA.table[usize::from(second)]
253                        & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) },
254                ) | u16::from(third >> 6)
255                    | (u16::from(fourth & 0xC0) << 2))
256                    != 0x202
257                {
258                    break 'outer;
259                }
260                read += 4;
261
262                // Next lead
263                if likely(read + 4 <= src.len()) {
264                    byte = unsafe { *(src.get_unchecked(read)) };
265                    if byte < 0x80 {
266                        read += 1;
267                        continue 'outer;
268                    }
269                    continue 'inner;
270                }
271                break 'inner;
272            }
273        }
274        // We can't have a complete 4-byte sequence, but we could still have
275        // one to three shorter sequences.
276        'tail: loop {
277            // >= is better for bound check elision than ==
278            if read >= src.len() {
279                break 'outer;
280            }
281            byte = src[read];
282            // At this point, `byte` is not included in `read`, because we
283            // don't yet know that a) the UTF-8 sequence is valid and b) that there
284            // is output space if it is an astral sequence.
285            // Inspecting the lead byte directly is faster than what the
286            // std lib does!
287            if byte < 0x80 {
288                read += 1;
289                continue 'tail;
290            }
291            if in_inclusive_range8(byte, 0xC2, 0xDF) {
292                // Two-byte
293                let new_read = read + 2;
294                if new_read > src.len() {
295                    break 'outer;
296                }
297                let second = src[read + 1];
298                if !in_inclusive_range8(second, 0x80, 0xBF) {
299                    break 'outer;
300                }
301                read += 2;
302                continue 'tail;
303            }
304            // We need to exclude valid four byte lead bytes, because
305            // `UTF8_DATA.second_mask` covers
306            if byte < 0xF0 {
307                // Three-byte
308                let new_read = read + 3;
309                if new_read > src.len() {
310                    break 'outer;
311                }
312                let second = src[read + 1];
313                let third = src[read + 2];
314                if ((UTF8_DATA.table[usize::from(second)]
315                    & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
316                    | (third >> 6))
317                    != 2
318                {
319                    break 'outer;
320                }
321                read += 3;
322                // `'tail` handles sequences shorter than 4, so
323                // there can't be another sequence after this one.
324                break 'outer;
325            }
326            break 'outer;
327        }
328    }
329    read
330}
331
332#[inline(always)]
333#[allow(clippy::never_loop, clippy::cognitive_complexity)]
334pub fn convert_utf8_to_utf16_up_to_invalid(src: &[u8], dst: &mut [u16]) -> (usize, usize) {
335    let mut read = 0;
336    let mut written = 0;
337    'outer: loop {
338        let mut byte = {
339            let src_remaining = &src[read..];
340            let dst_remaining = &mut dst[written..];
341            let length = ::core::cmp::min(src_remaining.len(), dst_remaining.len());
342            match ascii_to_basic_latin(src_remaining, dst_remaining) {
343                None => {
344                    read += length;
345                    written += length;
346                    break 'outer;
347                }
348                Some((non_ascii, consumed)) => {
349                    read += consumed;
350                    written += consumed;
351                    non_ascii
352                }
353            }
354        };
355        // Check for the longest sequence to avoid checking twice for the
356        // multi-byte sequences. This can't overflow with 64-bit address space,
357        // because full 64 bits aren't in use. In the 32-bit PAE case, for this
358        // to overflow would mean that the source slice would be so large that
359        // the address space of the process would not have space for any code.
360        // Therefore, the slice cannot be so long that this would overflow.
361        if likely(read + 4 <= src.len()) {
362            'inner: loop {
363                // At this point, `byte` is not included in `read`, because we
364                // don't yet know that a) the UTF-8 sequence is valid and b) that there
365                // is output space if it is an astral sequence.
366                // We know, thanks to `ascii_to_basic_latin` that there is output
367                // space for at least one UTF-16 code unit, so no need to check
368                // for output space in the BMP cases.
369                // Inspecting the lead byte directly is faster than what the
370                // std lib does!
371                if likely(in_inclusive_range8(byte, 0xC2, 0xDF)) {
372                    // Two-byte
373                    let second = unsafe { *(src.get_unchecked(read + 1)) };
374                    if !in_inclusive_range8(second, 0x80, 0xBF) {
375                        break 'outer;
376                    }
377                    unsafe {
378                        *(dst.get_unchecked_mut(written)) =
379                            ((u16::from(byte) & 0x1F) << 6) | (u16::from(second) & 0x3F)
380                    };
381                    read += 2;
382                    written += 1;
383
384                    // Next lead (manually inlined)
385                    if written == dst.len() {
386                        break 'outer;
387                    }
388                    if likely(read + 4 <= src.len()) {
389                        byte = unsafe { *(src.get_unchecked(read)) };
390                        if byte < 0x80 {
391                            unsafe { *(dst.get_unchecked_mut(written)) = u16::from(byte) };
392                            read += 1;
393                            written += 1;
394                            continue 'outer;
395                        }
396                        continue 'inner;
397                    }
398                    break 'inner;
399                }
400                if likely(byte < 0xF0) {
401                    'three: loop {
402                        // Three-byte
403                        let second = unsafe { *(src.get_unchecked(read + 1)) };
404                        let third = unsafe { *(src.get_unchecked(read + 2)) };
405                        if ((UTF8_DATA.table[usize::from(second)]
406                            & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
407                            | (third >> 6))
408                            != 2
409                        {
410                            break 'outer;
411                        }
412                        let point = ((u16::from(byte) & 0xF) << 12)
413                            | ((u16::from(second) & 0x3F) << 6)
414                            | (u16::from(third) & 0x3F);
415                        unsafe { *(dst.get_unchecked_mut(written)) = point };
416                        read += 3;
417                        written += 1;
418
419                        // Next lead (manually inlined)
420                        if written == dst.len() {
421                            break 'outer;
422                        }
423                        if likely(read + 4 <= src.len()) {
424                            byte = unsafe { *(src.get_unchecked(read)) };
425                            if in_inclusive_range8(byte, 0xE0, 0xEF) {
426                                continue 'three;
427                            }
428                            if likely(byte < 0x80) {
429                                unsafe { *(dst.get_unchecked_mut(written)) = u16::from(byte) };
430                                read += 1;
431                                written += 1;
432                                continue 'outer;
433                            }
434                            continue 'inner;
435                        }
436                        break 'inner;
437                    }
438                }
439                // Four-byte
440                if written + 1 == dst.len() {
441                    break 'outer;
442                }
443                let second = unsafe { *(src.get_unchecked(read + 1)) };
444                let third = unsafe { *(src.get_unchecked(read + 2)) };
445                let fourth = unsafe { *(src.get_unchecked(read + 3)) };
446                if (u16::from(
447                    UTF8_DATA.table[usize::from(second)]
448                        & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) },
449                ) | u16::from(third >> 6)
450                    | (u16::from(fourth & 0xC0) << 2))
451                    != 0x202
452                {
453                    break 'outer;
454                }
455                let point = ((u32::from(byte) & 0x7) << 18)
456                    | ((u32::from(second) & 0x3F) << 12)
457                    | ((u32::from(third) & 0x3F) << 6)
458                    | (u32::from(fourth) & 0x3F);
459                unsafe { *(dst.get_unchecked_mut(written)) = (0xD7C0 + (point >> 10)) as u16 };
460                unsafe {
461                    *(dst.get_unchecked_mut(written + 1)) = (0xDC00 + (point & 0x3FF)) as u16
462                };
463                read += 4;
464                written += 2;
465
466                // Next lead
467                if written == dst.len() {
468                    break 'outer;
469                }
470                if likely(read + 4 <= src.len()) {
471                    byte = unsafe { *(src.get_unchecked(read)) };
472                    if byte < 0x80 {
473                        unsafe { *(dst.get_unchecked_mut(written)) = u16::from(byte) };
474                        read += 1;
475                        written += 1;
476                        // See `u8u16punct` branch for punctuation loop here.
477                        continue 'outer;
478                    }
479                    continue 'inner;
480                }
481                break 'inner;
482            }
483        }
484        // We can't have a complete 4-byte sequence, but we could still have
485        // one to three shorter sequences.
486        'tail: loop {
487            // >= is better for bound check elision than ==
488            if read >= src.len() || written >= dst.len() {
489                break 'outer;
490            }
491            byte = src[read];
492            // At this point, `byte` is not included in `read`, because we
493            // don't yet know that a) the UTF-8 sequence is valid and b) that there
494            // is output space if it is an astral sequence.
495            // Inspecting the lead byte directly is faster than what the
496            // std lib does!
497            if byte < 0x80 {
498                dst[written] = u16::from(byte);
499                read += 1;
500                written += 1;
501                continue 'tail;
502            }
503            if in_inclusive_range8(byte, 0xC2, 0xDF) {
504                // Two-byte
505                let new_read = read + 2;
506                if new_read > src.len() {
507                    break 'outer;
508                }
509                let second = src[read + 1];
510                if !in_inclusive_range8(second, 0x80, 0xBF) {
511                    break 'outer;
512                }
513                dst[written] = ((u16::from(byte) & 0x1F) << 6) | (u16::from(second) & 0x3F);
514                read += 2;
515                written += 1;
516                continue 'tail;
517            }
518            // We need to exclude valid four byte lead bytes, because
519            // `UTF8_DATA.second_mask` covers
520            if byte < 0xF0 {
521                // Three-byte
522                let new_read = read + 3;
523                if new_read > src.len() {
524                    break 'outer;
525                }
526                let second = src[read + 1];
527                let third = src[read + 2];
528                if ((UTF8_DATA.table[usize::from(second)]
529                    & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
530                    | (third >> 6))
531                    != 2
532                {
533                    break 'outer;
534                }
535                let point = ((u16::from(byte) & 0xF) << 12)
536                    | ((u16::from(second) & 0x3F) << 6)
537                    | (u16::from(third) & 0x3F);
538                dst[written] = point;
539                read += 3;
540                written += 1;
541                // `'tail` handles sequences shorter than 4, so
542                // there can't be another sequence after this one.
543                break 'outer;
544            }
545            break 'outer;
546        }
547    }
548    (read, written)
549}
550
551pub struct Utf8Decoder {
552    code_point: u32,
553    bytes_seen: usize,   // 1, 2 or 3: counts continuations only
554    bytes_needed: usize, // 1, 2 or 3: counts continuations only
555    lower_boundary: u8,
556    upper_boundary: u8,
557}
558
559impl Utf8Decoder {
560    pub fn new_inner() -> Utf8Decoder {
561        Utf8Decoder {
562            code_point: 0,
563            bytes_seen: 0,
564            bytes_needed: 0,
565            lower_boundary: 0x80u8,
566            upper_boundary: 0xBFu8,
567        }
568    }
569
570    pub fn new() -> VariantDecoder {
571        VariantDecoder::Utf8(Utf8Decoder::new_inner())
572    }
573
574    pub fn in_neutral_state(&self) -> bool {
575        self.bytes_needed == 0
576    }
577
578    fn extra_from_state(&self) -> usize {
579        if self.bytes_needed == 0 {
580            0
581        } else {
582            self.bytes_seen + 1
583        }
584    }
585
586    pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
587        byte_length.checked_add(1 + self.extra_from_state())
588    }
589
590    pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> {
591        byte_length.checked_add(3 + self.extra_from_state())
592    }
593
594    pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> {
595        checked_add(
596            3,
597            checked_mul(3, byte_length.checked_add(self.extra_from_state())),
598        )
599    }
600
601    decoder_functions!(
602        preamble = {},
603        loop_preamble = {
604            // This is the fast path. The rest runs only at the
605            // start and end for partial sequences.
606            if self.bytes_needed == 0 {
607                dest.copy_utf8_up_to_invalid_from(&mut source);
608            }
609        },
610        eof = {
611            if self.bytes_needed != 0 {
612                let bad_bytes = (self.bytes_seen + 1) as u8;
613                self.code_point = 0;
614                self.bytes_needed = 0;
615                self.bytes_seen = 0;
616                return (
617                    DecoderResult::Malformed(bad_bytes, 0),
618                    src_consumed,
619                    dest.written(),
620                );
621            }
622        },
623        body = {
624            if self.bytes_needed == 0 {
625                if b < 0x80u8 {
626                    destination_handle.write_ascii(b);
627                    continue;
628                }
629                if b < 0xC2u8 {
630                    return (
631                        DecoderResult::Malformed(1, 0),
632                        unread_handle.consumed(),
633                        destination_handle.written(),
634                    );
635                }
636                if b < 0xE0u8 {
637                    self.bytes_needed = 1;
638                    self.code_point = u32::from(b) & 0x1F;
639                    continue;
640                }
641                if b < 0xF0u8 {
642                    if b == 0xE0u8 {
643                        self.lower_boundary = 0xA0u8;
644                    } else if b == 0xEDu8 {
645                        self.upper_boundary = 0x9Fu8;
646                    }
647                    self.bytes_needed = 2;
648                    self.code_point = u32::from(b) & 0xF;
649                    continue;
650                }
651                if b < 0xF5u8 {
652                    if b == 0xF0u8 {
653                        self.lower_boundary = 0x90u8;
654                    } else if b == 0xF4u8 {
655                        self.upper_boundary = 0x8Fu8;
656                    }
657                    self.bytes_needed = 3;
658                    self.code_point = u32::from(b) & 0x7;
659                    continue;
660                }
661                return (
662                    DecoderResult::Malformed(1, 0),
663                    unread_handle.consumed(),
664                    destination_handle.written(),
665                );
666            }
667            // self.bytes_needed != 0
668            if !(b >= self.lower_boundary && b <= self.upper_boundary) {
669                let bad_bytes = (self.bytes_seen + 1) as u8;
670                self.code_point = 0;
671                self.bytes_needed = 0;
672                self.bytes_seen = 0;
673                self.lower_boundary = 0x80u8;
674                self.upper_boundary = 0xBFu8;
675                return (
676                    DecoderResult::Malformed(bad_bytes, 0),
677                    unread_handle.unread(),
678                    destination_handle.written(),
679                );
680            }
681            self.lower_boundary = 0x80u8;
682            self.upper_boundary = 0xBFu8;
683            self.code_point = (self.code_point << 6) | (u32::from(b) & 0x3F);
684            self.bytes_seen += 1;
685            if self.bytes_seen != self.bytes_needed {
686                continue;
687            }
688            if self.bytes_needed == 3 {
689                destination_handle.write_astral(self.code_point);
690            } else {
691                destination_handle.write_bmp_excl_ascii(self.code_point as u16);
692            }
693            self.code_point = 0;
694            self.bytes_needed = 0;
695            self.bytes_seen = 0;
696            continue;
697        },
698        self = self,
699        src_consumed = src_consumed,
700        dest = dest,
701        source = source,
702        byte = b,
703        destination_handle = destination_handle,
704        unread_handle = unread_handle,
705        destination_check = check_space_astral
706    );
707}
708
709#[allow(clippy::never_loop)]
710#[inline(never)]
711#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
712pub fn convert_utf16_to_utf8_partial_inner(src: &[u16], dst: &mut [u8]) -> (usize, usize) {
713    let mut read = 0;
714    let mut written = 0;
715    'outer: loop {
716        let mut unit = {
717            let src_remaining = &src[read..];
718            let dst_remaining = &mut dst[written..];
719            let length = if dst_remaining.len() < src_remaining.len() {
720                dst_remaining.len()
721            } else {
722                src_remaining.len()
723            };
724            match basic_latin_to_ascii(src_remaining, dst_remaining) {
725                None => {
726                    read += length;
727                    written += length;
728                    return (read, written);
729                }
730                Some((non_ascii, consumed)) => {
731                    read += consumed;
732                    written += consumed;
733                    non_ascii
734                }
735            }
736        };
737        'inner: loop {
738            // The following loop is only broken out of as a goto forward.
739            loop {
740                // Unfortunately, this check isn't enough for the compiler to elide
741                // the bound checks on writes to dst, which is why they are manually
742                // elided, which makes a measurable difference.
743                if written.checked_add(4).unwrap() > dst.len() {
744                    return (read, written);
745                }
746                read += 1;
747                if unit < 0x800 {
748                    unsafe {
749                        *(dst.get_unchecked_mut(written)) = (unit >> 6) as u8 | 0xC0u8;
750                        written += 1;
751                        *(dst.get_unchecked_mut(written)) = (unit & 0x3F) as u8 | 0x80u8;
752                        written += 1;
753                    }
754                    break;
755                }
756                let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
757                if likely(unit_minus_surrogate_start > (0xDFFF - 0xD800)) {
758                    unsafe {
759                        *(dst.get_unchecked_mut(written)) = (unit >> 12) as u8 | 0xE0u8;
760                        written += 1;
761                        *(dst.get_unchecked_mut(written)) = ((unit & 0xFC0) >> 6) as u8 | 0x80u8;
762                        written += 1;
763                        *(dst.get_unchecked_mut(written)) = (unit & 0x3F) as u8 | 0x80u8;
764                        written += 1;
765                    }
766                    break;
767                }
768                if likely(unit_minus_surrogate_start <= (0xDBFF - 0xD800)) {
769                    // high surrogate
770                    // read > src.len() is impossible, but using
771                    // >= instead of == allows the compiler to elide a bound check.
772                    if read >= src.len() {
773                        debug_assert_eq!(read, src.len());
774                        // Unpaired surrogate at the end of the buffer.
775                        unsafe {
776                            *(dst.get_unchecked_mut(written)) = 0xEFu8;
777                            written += 1;
778                            *(dst.get_unchecked_mut(written)) = 0xBFu8;
779                            written += 1;
780                            *(dst.get_unchecked_mut(written)) = 0xBDu8;
781                            written += 1;
782                        }
783                        return (read, written);
784                    }
785                    let second = src[read];
786                    let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
787                    if likely(second_minus_low_surrogate_start <= (0xDFFF - 0xDC00)) {
788                        // The next code unit is a low surrogate. Advance position.
789                        read += 1;
790                        let astral = (u32::from(unit) << 10) + u32::from(second)
791                            - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32);
792                        unsafe {
793                            *(dst.get_unchecked_mut(written)) = (astral >> 18) as u8 | 0xF0u8;
794                            written += 1;
795                            *(dst.get_unchecked_mut(written)) =
796                                ((astral & 0x3F000u32) >> 12) as u8 | 0x80u8;
797                            written += 1;
798                            *(dst.get_unchecked_mut(written)) =
799                                ((astral & 0xFC0u32) >> 6) as u8 | 0x80u8;
800                            written += 1;
801                            *(dst.get_unchecked_mut(written)) = (astral & 0x3F) as u8 | 0x80u8;
802                            written += 1;
803                        }
804                        break;
805                    }
806                    // The next code unit is not a low surrogate. Don't advance
807                    // position and treat the high surrogate as unpaired.
808                    // Fall through
809                }
810                // Unpaired low surrogate
811                unsafe {
812                    *(dst.get_unchecked_mut(written)) = 0xEFu8;
813                    written += 1;
814                    *(dst.get_unchecked_mut(written)) = 0xBFu8;
815                    written += 1;
816                    *(dst.get_unchecked_mut(written)) = 0xBDu8;
817                    written += 1;
818                }
819                break;
820            }
821            'punctuation: loop {
822                // Now see if the next unit is Basic Latin
823                // read > src.len() is impossible, but using
824                // >= instead of == allows the compiler to elide a bound check.
825                if read >= src.len() {
826                    debug_assert_eq!(read, src.len());
827                    return (read, written);
828                }
829                unit = src[read];
830                if unlikely(unit < 0x80) {
831                    // written > dst.len() is impossible, but using
832                    // >= instead of == allows the compiler to elide a bound check.
833                    if written >= dst.len() {
834                        debug_assert_eq!(written, dst.len());
835                        return (read, written);
836                    }
837                    dst[written] = unit as u8;
838                    read += 1;
839                    written += 1;
840                    // A punctuation check was slower in 2018 but makes sense in 2026.
841                    if unit < 0x3C {
842                        continue 'punctuation;
843                    }
844                    continue 'outer;
845                }
846                continue 'inner;
847            }
848        }
849    }
850}
851
852#[inline(never)]
853pub fn convert_utf16_to_utf8_partial_tail(src: &[u16], dst: &mut [u8]) -> (usize, usize) {
854    // Everything below is cold code!
855    let mut read = 0;
856    let mut written = 0;
857    let mut unit = src[read];
858    // We now have up to 3 output slots, so an astral character
859    // will not fit.
860    if unit < 0x800 {
861        loop {
862            if unit < 0x80 {
863                if written >= dst.len() {
864                    return (read, written);
865                }
866                read += 1;
867                dst[written] = unit as u8;
868                written += 1;
869            } else if unit < 0x800 {
870                if written + 2 > dst.len() {
871                    return (read, written);
872                }
873                read += 1;
874                dst[written] = (unit >> 6) as u8 | 0xC0u8;
875                written += 1;
876                dst[written] = (unit & 0x3F) as u8 | 0x80u8;
877                written += 1;
878            } else {
879                return (read, written);
880            }
881            // read > src.len() is impossible, but using
882            // >= instead of == allows the compiler to elide a bound check.
883            if read >= src.len() {
884                debug_assert_eq!(read, src.len());
885                return (read, written);
886            }
887            unit = src[read];
888        }
889    }
890    // Could be an unpaired surrogate, but we'll need 3 output
891    // slots in any case.
892    if written + 3 > dst.len() {
893        return (read, written);
894    }
895    read += 1;
896    let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
897    if unit_minus_surrogate_start <= (0xDFFF - 0xD800) {
898        // Got surrogate
899        if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
900            // Got high surrogate
901            #[allow(clippy::branches_sharing_code)]
902            if read >= src.len() {
903                // Unpaired high surrogate
904                unit = 0xFFFD;
905            } else {
906                let second = src[read];
907                if in_inclusive_range16(second, 0xDC00, 0xDFFF) {
908                    // Valid surrogate pair, but we know it won't fit.
909                    read -= 1;
910                    return (read, written);
911                }
912                // Unpaired high
913                unit = 0xFFFD;
914            }
915        } else {
916            // Unpaired low
917            unit = 0xFFFD;
918        }
919    }
920    dst[written] = (unit >> 12) as u8 | 0xE0u8;
921    written += 1;
922    dst[written] = ((unit & 0xFC0) >> 6) as u8 | 0x80u8;
923    written += 1;
924    dst[written] = (unit & 0x3F) as u8 | 0x80u8;
925    written += 1;
926    debug_assert_eq!(written, dst.len());
927    (read, written)
928}
929
930pub struct Utf8Encoder;
931
932impl Utf8Encoder {
933    pub fn new(encoding: &'static Encoding) -> Encoder {
934        Encoder::new(encoding, VariantEncoder::Utf8(Utf8Encoder))
935    }
936
937    pub fn max_buffer_length_from_utf16_without_replacement(
938        &self,
939        u16_length: usize,
940    ) -> Option<usize> {
941        u16_length.checked_mul(3)
942    }
943
944    pub fn max_buffer_length_from_utf8_without_replacement(
945        &self,
946        byte_length: usize,
947    ) -> Option<usize> {
948        Some(byte_length)
949    }
950
951    pub fn encode_from_utf16_raw(
952        &mut self,
953        src: &[u16],
954        dst: &mut [u8],
955        _last: bool,
956    ) -> (EncoderResult, usize, usize) {
957        let (read, written) = convert_utf16_to_utf8_partial(src, dst);
958        (
959            if read == src.len() {
960                EncoderResult::InputEmpty
961            } else {
962                EncoderResult::OutputFull
963            },
964            read,
965            written,
966        )
967    }
968
969    pub fn encode_from_utf8_raw(
970        &mut self,
971        src: &str,
972        dst: &mut [u8],
973        _last: bool,
974    ) -> (EncoderResult, usize, usize) {
975        let bytes = src.as_bytes();
976        let mut to_write = bytes.len();
977        if to_write <= dst.len() {
978            dst[..to_write].copy_from_slice(bytes);
979            return (EncoderResult::InputEmpty, to_write, to_write);
980        }
981        to_write = dst.len();
982        // Move back until we find a UTF-8 sequence boundary.
983        while (bytes[to_write] & 0xC0) == 0x80 {
984            to_write -= 1;
985        }
986        dst[..to_write].copy_from_slice(&bytes[..to_write]);
987        (EncoderResult::OutputFull, to_write, to_write)
988    }
989}
990
991// Any copyright to the test code below this comment is dedicated to the
992// Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
993
994#[cfg(all(test, feature = "alloc"))]
995mod tests {
996    use super::super::testing::*;
997    use super::super::*;
998
999    //    fn decode_utf8_to_utf16(bytes: &[u8], expect: &[u16]) {
1000    //        decode_to_utf16_without_replacement(UTF_8, bytes, expect);
1001    //    }
1002
1003    fn decode_utf8_to_utf8(bytes: &[u8], expect: &str) {
1004        decode_to_utf8(UTF_8, bytes, expect);
1005    }
1006
1007    fn decode_valid_utf8(string: &str) {
1008        decode_utf8_to_utf8(string.as_bytes(), string);
1009    }
1010
1011    fn encode_utf8_from_utf16(string: &[u16], expect: &[u8]) {
1012        encode_from_utf16(UTF_8, string, expect);
1013    }
1014
1015    fn encode_utf8_from_utf8(string: &str, expect: &[u8]) {
1016        encode_from_utf8(UTF_8, string, expect);
1017    }
1018
1019    fn encode_utf8_from_utf16_with_output_limit(
1020        string: &[u16],
1021        expect: &str,
1022        limit: usize,
1023        expect_result: EncoderResult,
1024    ) {
1025        let mut dst = Vec::new();
1026        {
1027            dst.resize(limit, 0u8);
1028            let mut encoder = UTF_8.new_encoder();
1029            let (result, read, written) =
1030                encoder.encode_from_utf16_without_replacement(string, &mut dst, false);
1031            assert_eq!(result, expect_result);
1032            if expect_result == EncoderResult::InputEmpty {
1033                assert_eq!(read, string.len());
1034            }
1035            assert_eq!(&dst[..written], expect.as_bytes());
1036        }
1037        {
1038            dst.resize(64, 0u8);
1039            for (i, elem) in dst.iter_mut().enumerate() {
1040                *elem = i as u8;
1041            }
1042            let mut encoder = UTF_8.new_encoder();
1043            let (_, _, mut j) =
1044                encoder.encode_from_utf16_without_replacement(string, &mut dst, false);
1045            while j < dst.len() {
1046                assert_eq!(usize::from(dst[j]), j);
1047                j += 1;
1048            }
1049        }
1050    }
1051
1052    #[test]
1053    fn test_utf8_decode() {
1054        // Empty
1055        decode_valid_utf8("");
1056        // ASCII
1057        decode_valid_utf8("ab");
1058        // Low BMP
1059        decode_valid_utf8("a\u{E4}Z");
1060        // High BMP
1061        decode_valid_utf8("a\u{2603}Z");
1062        // Astral
1063        decode_valid_utf8("a\u{1F4A9}Z");
1064        // Low BMP with last byte missing
1065        decode_utf8_to_utf8(b"a\xC3Z", "a\u{FFFD}Z");
1066        decode_utf8_to_utf8(b"a\xC3", "a\u{FFFD}");
1067        // High BMP with last byte missing
1068        decode_utf8_to_utf8(b"a\xE2\x98Z", "a\u{FFFD}Z");
1069        decode_utf8_to_utf8(b"a\xE2\x98", "a\u{FFFD}");
1070        // Astral with last byte missing
1071        decode_utf8_to_utf8(b"a\xF0\x9F\x92Z", "a\u{FFFD}Z");
1072        decode_utf8_to_utf8(b"a\xF0\x9F\x92", "a\u{FFFD}");
1073        // Lone highest continuation
1074        decode_utf8_to_utf8(b"a\xBFZ", "a\u{FFFD}Z");
1075        decode_utf8_to_utf8(b"a\xBF", "a\u{FFFD}");
1076        // Two lone highest continuations
1077        decode_utf8_to_utf8(b"a\xBF\xBFZ", "a\u{FFFD}\u{FFFD}Z");
1078        decode_utf8_to_utf8(b"a\xBF\xBF", "a\u{FFFD}\u{FFFD}");
1079        // Low BMP followed by lowest lone continuation
1080        decode_utf8_to_utf8(b"a\xC3\xA4\x80Z", "a\u{E4}\u{FFFD}Z");
1081        decode_utf8_to_utf8(b"a\xC3\xA4\x80", "a\u{E4}\u{FFFD}");
1082        // Low BMP followed by highest lone continuation
1083        decode_utf8_to_utf8(b"a\xC3\xA4\xBFZ", "a\u{E4}\u{FFFD}Z");
1084        decode_utf8_to_utf8(b"a\xC3\xA4\xBF", "a\u{E4}\u{FFFD}");
1085        // High BMP followed by lowest lone continuation
1086        decode_utf8_to_utf8(b"a\xE2\x98\x83\x80Z", "a\u{2603}\u{FFFD}Z");
1087        decode_utf8_to_utf8(b"a\xE2\x98\x83\x80", "a\u{2603}\u{FFFD}");
1088        // High BMP followed by highest lone continuation
1089        decode_utf8_to_utf8(b"a\xE2\x98\x83\xBFZ", "a\u{2603}\u{FFFD}Z");
1090        decode_utf8_to_utf8(b"a\xE2\x98\x83\xBF", "a\u{2603}\u{FFFD}");
1091        // Astral followed by lowest lone continuation
1092        decode_utf8_to_utf8(b"a\xF0\x9F\x92\xA9\x80Z", "a\u{1F4A9}\u{FFFD}Z");
1093        decode_utf8_to_utf8(b"a\xF0\x9F\x92\xA9\x80", "a\u{1F4A9}\u{FFFD}");
1094        // Astral followed by highest lone continuation
1095        decode_utf8_to_utf8(b"a\xF0\x9F\x92\xA9\xBFZ", "a\u{1F4A9}\u{FFFD}Z");
1096        decode_utf8_to_utf8(b"a\xF0\x9F\x92\xA9\xBF", "a\u{1F4A9}\u{FFFD}");
1097
1098        // Boundary conditions
1099        // Lowest single-byte
1100        decode_valid_utf8("Z\x00");
1101        decode_valid_utf8("Z\x00Z");
1102        // Lowest single-byte as two-byte overlong sequence
1103        decode_utf8_to_utf8(b"a\xC0\x80", "a\u{FFFD}\u{FFFD}");
1104        decode_utf8_to_utf8(b"a\xC0\x80Z", "a\u{FFFD}\u{FFFD}Z");
1105        // Lowest single-byte as three-byte overlong sequence
1106        decode_utf8_to_utf8(b"a\xE0\x80\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}");
1107        decode_utf8_to_utf8(b"a\xE0\x80\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}Z");
1108        // Lowest single-byte as four-byte overlong sequence
1109        decode_utf8_to_utf8(b"a\xF0\x80\x80\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1110        decode_utf8_to_utf8(b"a\xF0\x80\x80\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1111        // One below lowest single-byte
1112        decode_utf8_to_utf8(b"a\xFF", "a\u{FFFD}");
1113        decode_utf8_to_utf8(b"a\xFFZ", "a\u{FFFD}Z");
1114        // Highest single-byte
1115        decode_valid_utf8("a\x7F");
1116        decode_valid_utf8("a\x7FZ");
1117        // Highest single-byte as two-byte overlong sequence
1118        decode_utf8_to_utf8(b"a\xC1\xBF", "a\u{FFFD}\u{FFFD}");
1119        decode_utf8_to_utf8(b"a\xC1\xBFZ", "a\u{FFFD}\u{FFFD}Z");
1120        // Highest single-byte as three-byte overlong sequence
1121        decode_utf8_to_utf8(b"a\xE0\x81\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}");
1122        decode_utf8_to_utf8(b"a\xE0\x81\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}Z");
1123        // Highest single-byte as four-byte overlong sequence
1124        decode_utf8_to_utf8(b"a\xF0\x80\x81\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1125        decode_utf8_to_utf8(b"a\xF0\x80\x81\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1126        // One past highest single byte (also lone continuation)
1127        decode_utf8_to_utf8(b"a\x80Z", "a\u{FFFD}Z");
1128        decode_utf8_to_utf8(b"a\x80", "a\u{FFFD}");
1129        // Two lone continuations
1130        decode_utf8_to_utf8(b"a\x80\x80Z", "a\u{FFFD}\u{FFFD}Z");
1131        decode_utf8_to_utf8(b"a\x80\x80", "a\u{FFFD}\u{FFFD}");
1132        // Three lone continuations
1133        decode_utf8_to_utf8(b"a\x80\x80\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}Z");
1134        decode_utf8_to_utf8(b"a\x80\x80\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}");
1135        // Four lone continuations
1136        decode_utf8_to_utf8(b"a\x80\x80\x80\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1137        decode_utf8_to_utf8(b"a\x80\x80\x80\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1138        // Lowest two-byte
1139        decode_utf8_to_utf8(b"a\xC2\x80", "a\u{0080}");
1140        decode_utf8_to_utf8(b"a\xC2\x80Z", "a\u{0080}Z");
1141        // Lowest two-byte as three-byte overlong sequence
1142        decode_utf8_to_utf8(b"a\xE0\x82\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}");
1143        decode_utf8_to_utf8(b"a\xE0\x82\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}Z");
1144        // Lowest two-byte as four-byte overlong sequence
1145        decode_utf8_to_utf8(b"a\xF0\x80\x82\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1146        decode_utf8_to_utf8(b"a\xF0\x80\x82\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1147        // Lead one below lowest two-byte
1148        decode_utf8_to_utf8(b"a\xC1\x80", "a\u{FFFD}\u{FFFD}");
1149        decode_utf8_to_utf8(b"a\xC1\x80Z", "a\u{FFFD}\u{FFFD}Z");
1150        // Trail one below lowest two-byte
1151        decode_utf8_to_utf8(b"a\xC2\x7F", "a\u{FFFD}\u{007F}");
1152        decode_utf8_to_utf8(b"a\xC2\x7FZ", "a\u{FFFD}\u{007F}Z");
1153        // Highest two-byte
1154        decode_utf8_to_utf8(b"a\xDF\xBF", "a\u{07FF}");
1155        decode_utf8_to_utf8(b"a\xDF\xBFZ", "a\u{07FF}Z");
1156        // Highest two-byte as three-byte overlong sequence
1157        decode_utf8_to_utf8(b"a\xE0\x9F\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}");
1158        decode_utf8_to_utf8(b"a\xE0\x9F\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}Z");
1159        // Highest two-byte as four-byte overlong sequence
1160        decode_utf8_to_utf8(b"a\xF0\x80\x9F\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1161        decode_utf8_to_utf8(b"a\xF0\x80\x9F\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1162        // Lowest three-byte
1163        decode_utf8_to_utf8(b"a\xE0\xA0\x80", "a\u{0800}");
1164        decode_utf8_to_utf8(b"a\xE0\xA0\x80Z", "a\u{0800}Z");
1165        // Lowest three-byte as four-byte overlong sequence
1166        decode_utf8_to_utf8(b"a\xF0\x80\xA0\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1167        decode_utf8_to_utf8(b"a\xF0\x80\xA0\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1168        // Highest below surrogates
1169        decode_utf8_to_utf8(b"a\xED\x9F\xBF", "a\u{D7FF}");
1170        decode_utf8_to_utf8(b"a\xED\x9F\xBFZ", "a\u{D7FF}Z");
1171        // Highest below surrogates as four-byte overlong sequence
1172        decode_utf8_to_utf8(b"a\xF0\x8D\x9F\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1173        decode_utf8_to_utf8(b"a\xF0\x8D\x9F\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1174        // First surrogate
1175        decode_utf8_to_utf8(b"a\xED\xA0\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}");
1176        decode_utf8_to_utf8(b"a\xED\xA0\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}Z");
1177        // First surrogate as four-byte overlong sequence
1178        decode_utf8_to_utf8(b"a\xF0\x8D\xA0\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1179        decode_utf8_to_utf8(b"a\xF0\x8D\xA0\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1180        // Last surrogate
1181        decode_utf8_to_utf8(b"a\xED\xBF\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}");
1182        decode_utf8_to_utf8(b"a\xED\xBF\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}Z");
1183        // Last surrogate as four-byte overlong sequence
1184        decode_utf8_to_utf8(b"a\xF0\x8D\xBF\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1185        decode_utf8_to_utf8(b"a\xF0\x8D\xBF\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1186        // Lowest above surrogates
1187        decode_utf8_to_utf8(b"a\xEE\x80\x80", "a\u{E000}");
1188        decode_utf8_to_utf8(b"a\xEE\x80\x80Z", "a\u{E000}Z");
1189        // Lowest above surrogates as four-byte overlong sequence
1190        decode_utf8_to_utf8(b"a\xF0\x8E\x80\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1191        decode_utf8_to_utf8(b"a\xF0\x8E\x80\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1192        // Highest three-byte
1193        decode_utf8_to_utf8(b"a\xEF\xBF\xBF", "a\u{FFFF}");
1194        decode_utf8_to_utf8(b"a\xEF\xBF\xBFZ", "a\u{FFFF}Z");
1195        // Highest three-byte as four-byte overlong sequence
1196        decode_utf8_to_utf8(b"a\xF0\x8F\xBF\xBF", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1197        decode_utf8_to_utf8(b"a\xF0\x8F\xBF\xBFZ", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1198        // Lowest four-byte
1199        decode_utf8_to_utf8(b"a\xF0\x90\x80\x80", "a\u{10000}");
1200        decode_utf8_to_utf8(b"a\xF0\x90\x80\x80Z", "a\u{10000}Z");
1201        // Highest four-byte
1202        decode_utf8_to_utf8(b"a\xF4\x8F\xBF\xBF", "a\u{10FFFF}");
1203        decode_utf8_to_utf8(b"a\xF4\x8F\xBF\xBFZ", "a\u{10FFFF}Z");
1204        // One past highest four-byte
1205        decode_utf8_to_utf8(b"a\xF4\x90\x80\x80", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}");
1206        decode_utf8_to_utf8(b"a\xF4\x90\x80\x80Z", "a\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}Z");
1207
1208        // Highest four-byte with last byte replaced with 0xFF
1209        decode_utf8_to_utf8(b"a\xF4\x8F\xBF\xFF", "a\u{FFFD}\u{FFFD}");
1210        decode_utf8_to_utf8(b"a\xF4\x8F\xBF\xFFZ", "a\u{FFFD}\u{FFFD}Z");
1211    }
1212
1213    #[test]
1214    fn test_utf8_encode() {
1215        // Empty
1216        encode_utf8_from_utf16(&[], b"");
1217        encode_utf8_from_utf8("", b"");
1218
1219        encode_utf8_from_utf16(&[0x0000], "\u{0000}".as_bytes());
1220        encode_utf8_from_utf16(&[0x007F], "\u{007F}".as_bytes());
1221        encode_utf8_from_utf16(&[0x0080], "\u{0080}".as_bytes());
1222        encode_utf8_from_utf16(&[0x07FF], "\u{07FF}".as_bytes());
1223        encode_utf8_from_utf16(&[0x0800], "\u{0800}".as_bytes());
1224        encode_utf8_from_utf16(&[0xD7FF], "\u{D7FF}".as_bytes());
1225        encode_utf8_from_utf16(&[0xD800], "\u{FFFD}".as_bytes());
1226        encode_utf8_from_utf16(&[0xD800, 0x0062], "\u{FFFD}\u{0062}".as_bytes());
1227        encode_utf8_from_utf16(&[0xDFFF], "\u{FFFD}".as_bytes());
1228        encode_utf8_from_utf16(&[0xDFFF, 0x0062], "\u{FFFD}\u{0062}".as_bytes());
1229        encode_utf8_from_utf16(&[0xE000], "\u{E000}".as_bytes());
1230        encode_utf8_from_utf16(&[0xFFFF], "\u{FFFF}".as_bytes());
1231        encode_utf8_from_utf16(&[0xD800, 0xDC00], "\u{10000}".as_bytes());
1232        encode_utf8_from_utf16(&[0xDBFF, 0xDFFF], "\u{10FFFF}".as_bytes());
1233        encode_utf8_from_utf16(&[0xDC00, 0xDEDE], "\u{FFFD}\u{FFFD}".as_bytes());
1234    }
1235
1236    #[test]
1237    fn test_encode_utf8_from_utf16_with_output_limit() {
1238        encode_utf8_from_utf16_with_output_limit(&[0x0062], "\u{62}", 1, EncoderResult::InputEmpty);
1239        encode_utf8_from_utf16_with_output_limit(&[0x00A7], "\u{A7}", 2, EncoderResult::InputEmpty);
1240        encode_utf8_from_utf16_with_output_limit(
1241            &[0x2603],
1242            "\u{2603}",
1243            3,
1244            EncoderResult::InputEmpty,
1245        );
1246        encode_utf8_from_utf16_with_output_limit(
1247            &[0xD83D, 0xDCA9],
1248            "\u{1F4A9}",
1249            4,
1250            EncoderResult::InputEmpty,
1251        );
1252
1253        encode_utf8_from_utf16_with_output_limit(&[0x00A7], "", 1, EncoderResult::OutputFull);
1254        encode_utf8_from_utf16_with_output_limit(&[0x2603], "", 2, EncoderResult::OutputFull);
1255        encode_utf8_from_utf16_with_output_limit(
1256            &[0xD83D, 0xDCA9],
1257            "",
1258            3,
1259            EncoderResult::OutputFull,
1260        );
1261
1262        encode_utf8_from_utf16_with_output_limit(
1263            &[0x0063, 0x0062],
1264            "\u{63}\u{62}",
1265            2,
1266            EncoderResult::InputEmpty,
1267        );
1268        encode_utf8_from_utf16_with_output_limit(
1269            &[0x0063, 0x00A7],
1270            "\u{63}\u{A7}",
1271            3,
1272            EncoderResult::InputEmpty,
1273        );
1274        encode_utf8_from_utf16_with_output_limit(
1275            &[0x0063, 0x2603],
1276            "\u{63}\u{2603}",
1277            4,
1278            EncoderResult::InputEmpty,
1279        );
1280        encode_utf8_from_utf16_with_output_limit(
1281            &[0x0063, 0xD83D, 0xDCA9],
1282            "\u{63}\u{1F4A9}",
1283            5,
1284            EncoderResult::InputEmpty,
1285        );
1286
1287        encode_utf8_from_utf16_with_output_limit(
1288            &[0x0063, 0x00A7],
1289            "\u{63}",
1290            2,
1291            EncoderResult::OutputFull,
1292        );
1293        encode_utf8_from_utf16_with_output_limit(
1294            &[0x0063, 0x2603],
1295            "\u{63}",
1296            3,
1297            EncoderResult::OutputFull,
1298        );
1299        encode_utf8_from_utf16_with_output_limit(
1300            &[0x0063, 0xD83D, 0xDCA9],
1301            "\u{63}",
1302            4,
1303            EncoderResult::OutputFull,
1304        );
1305
1306        encode_utf8_from_utf16_with_output_limit(
1307            &[0x00B6, 0x0062],
1308            "\u{B6}\u{62}",
1309            3,
1310            EncoderResult::InputEmpty,
1311        );
1312        encode_utf8_from_utf16_with_output_limit(
1313            &[0x00B6, 0x00A7],
1314            "\u{B6}\u{A7}",
1315            4,
1316            EncoderResult::InputEmpty,
1317        );
1318        encode_utf8_from_utf16_with_output_limit(
1319            &[0x00B6, 0x2603],
1320            "\u{B6}\u{2603}",
1321            5,
1322            EncoderResult::InputEmpty,
1323        );
1324        encode_utf8_from_utf16_with_output_limit(
1325            &[0x00B6, 0xD83D, 0xDCA9],
1326            "\u{B6}\u{1F4A9}",
1327            6,
1328            EncoderResult::InputEmpty,
1329        );
1330
1331        encode_utf8_from_utf16_with_output_limit(
1332            &[0x00B6, 0x00A7],
1333            "\u{B6}",
1334            3,
1335            EncoderResult::OutputFull,
1336        );
1337        encode_utf8_from_utf16_with_output_limit(
1338            &[0x00B6, 0x2603],
1339            "\u{B6}",
1340            4,
1341            EncoderResult::OutputFull,
1342        );
1343        encode_utf8_from_utf16_with_output_limit(
1344            &[0x00B6, 0xD83D, 0xDCA9],
1345            "\u{B6}",
1346            5,
1347            EncoderResult::OutputFull,
1348        );
1349
1350        encode_utf8_from_utf16_with_output_limit(
1351            &[0x263A, 0x0062],
1352            "\u{263A}\u{62}",
1353            4,
1354            EncoderResult::InputEmpty,
1355        );
1356        encode_utf8_from_utf16_with_output_limit(
1357            &[0x263A, 0x00A7],
1358            "\u{263A}\u{A7}",
1359            5,
1360            EncoderResult::InputEmpty,
1361        );
1362        encode_utf8_from_utf16_with_output_limit(
1363            &[0x263A, 0x2603],
1364            "\u{263A}\u{2603}",
1365            6,
1366            EncoderResult::InputEmpty,
1367        );
1368        encode_utf8_from_utf16_with_output_limit(
1369            &[0x263A, 0xD83D, 0xDCA9],
1370            "\u{263A}\u{1F4A9}",
1371            7,
1372            EncoderResult::InputEmpty,
1373        );
1374
1375        encode_utf8_from_utf16_with_output_limit(
1376            &[0x263A, 0x00A7],
1377            "\u{263A}",
1378            4,
1379            EncoderResult::OutputFull,
1380        );
1381        encode_utf8_from_utf16_with_output_limit(
1382            &[0x263A, 0x2603],
1383            "\u{263A}",
1384            5,
1385            EncoderResult::OutputFull,
1386        );
1387        encode_utf8_from_utf16_with_output_limit(
1388            &[0x263A, 0xD83D, 0xDCA9],
1389            "\u{263A}",
1390            6,
1391            EncoderResult::OutputFull,
1392        );
1393
1394        encode_utf8_from_utf16_with_output_limit(
1395            &[0xD83D, 0xDE0E, 0x0062],
1396            "\u{1F60E}\u{62}",
1397            5,
1398            EncoderResult::InputEmpty,
1399        );
1400        encode_utf8_from_utf16_with_output_limit(
1401            &[0xD83D, 0xDE0E, 0x00A7],
1402            "\u{1F60E}\u{A7}",
1403            6,
1404            EncoderResult::InputEmpty,
1405        );
1406        encode_utf8_from_utf16_with_output_limit(
1407            &[0xD83D, 0xDE0E, 0x2603],
1408            "\u{1F60E}\u{2603}",
1409            7,
1410            EncoderResult::InputEmpty,
1411        );
1412        encode_utf8_from_utf16_with_output_limit(
1413            &[0xD83D, 0xDE0E, 0xD83D, 0xDCA9],
1414            "\u{1F60E}\u{1F4A9}",
1415            8,
1416            EncoderResult::InputEmpty,
1417        );
1418
1419        encode_utf8_from_utf16_with_output_limit(
1420            &[0xD83D, 0xDE0E, 0x00A7],
1421            "\u{1F60E}",
1422            5,
1423            EncoderResult::OutputFull,
1424        );
1425        encode_utf8_from_utf16_with_output_limit(
1426            &[0xD83D, 0xDE0E, 0x2603],
1427            "\u{1F60E}",
1428            6,
1429            EncoderResult::OutputFull,
1430        );
1431        encode_utf8_from_utf16_with_output_limit(
1432            &[0xD83D, 0xDE0E, 0xD83D, 0xDCA9],
1433            "\u{1F60E}",
1434            7,
1435            EncoderResult::OutputFull,
1436        );
1437
1438        encode_utf8_from_utf16_with_output_limit(
1439            &[0x0063, 0x00B6, 0x0062, 0x0062],
1440            "\u{63}\u{B6}\u{62}\u{62}",
1441            5,
1442            EncoderResult::InputEmpty,
1443        );
1444        encode_utf8_from_utf16_with_output_limit(
1445            &[0x0063, 0x00B6, 0x0062, 0x0062],
1446            "\u{63}\u{B6}\u{62}",
1447            4,
1448            EncoderResult::OutputFull,
1449        );
1450
1451        encode_utf8_from_utf16_with_output_limit(
1452            &[0x0063, 0x00B6, 0x0062, 0x0062, 0x0062],
1453            "\u{63}\u{B6}\u{62}\u{62}\u{62}",
1454            6,
1455            EncoderResult::InputEmpty,
1456        );
1457        encode_utf8_from_utf16_with_output_limit(
1458            &[0x0063, 0x00B6, 0x0062, 0x0062, 0x0062],
1459            "\u{63}\u{B6}\u{62}\u{62}",
1460            5,
1461            EncoderResult::OutputFull,
1462        );
1463
1464        encode_utf8_from_utf16_with_output_limit(
1465            &[0x263A, 0x0062, 0x0062],
1466            "\u{263A}\u{62}\u{62}",
1467            5,
1468            EncoderResult::InputEmpty,
1469        );
1470        encode_utf8_from_utf16_with_output_limit(
1471            &[0x263A, 0x0062, 0x0062],
1472            "\u{263A}\u{62}",
1473            4,
1474            EncoderResult::OutputFull,
1475        );
1476
1477        encode_utf8_from_utf16_with_output_limit(
1478            &[0x263A, 0x0062, 0x0062, 0x0062],
1479            "\u{263A}\u{62}\u{62}\u{62}",
1480            6,
1481            EncoderResult::InputEmpty,
1482        );
1483        encode_utf8_from_utf16_with_output_limit(
1484            &[0x263A, 0x0062, 0x0062, 0x0062],
1485            "\u{263A}\u{62}\u{62}",
1486            5,
1487            EncoderResult::OutputFull,
1488        );
1489
1490        encode_utf8_from_utf16_with_output_limit(
1491            &[0x0063, 0x00B6, 0x00A7],
1492            "\u{63}\u{B6}\u{A7}",
1493            5,
1494            EncoderResult::InputEmpty,
1495        );
1496        encode_utf8_from_utf16_with_output_limit(
1497            &[0x0063, 0x00B6, 0x00A7],
1498            "\u{63}\u{B6}",
1499            4,
1500            EncoderResult::OutputFull,
1501        );
1502
1503        encode_utf8_from_utf16_with_output_limit(
1504            &[0x0063, 0x00B6, 0x00A7, 0x0062],
1505            "\u{63}\u{B6}\u{A7}\u{62}",
1506            6,
1507            EncoderResult::InputEmpty,
1508        );
1509        encode_utf8_from_utf16_with_output_limit(
1510            &[0x0063, 0x00B6, 0x00A7, 0x0062],
1511            "\u{63}\u{B6}\u{A7}",
1512            5,
1513            EncoderResult::OutputFull,
1514        );
1515
1516        encode_utf8_from_utf16_with_output_limit(
1517            &[0x263A, 0x00A7, 0x0062],
1518            "\u{263A}\u{A7}\u{62}",
1519            6,
1520            EncoderResult::InputEmpty,
1521        );
1522        encode_utf8_from_utf16_with_output_limit(
1523            &[0x263A, 0x00A7, 0x0062],
1524            "\u{263A}\u{A7}",
1525            5,
1526            EncoderResult::OutputFull,
1527        );
1528
1529        encode_utf8_from_utf16_with_output_limit(
1530            &[0x0063, 0x00B6, 0x0062, 0x00A7],
1531            "\u{63}\u{B6}\u{62}\u{A7}",
1532            6,
1533            EncoderResult::InputEmpty,
1534        );
1535        encode_utf8_from_utf16_with_output_limit(
1536            &[0x0063, 0x00B6, 0x0062, 0x00A7],
1537            "\u{63}\u{B6}\u{62}",
1538            5,
1539            EncoderResult::OutputFull,
1540        );
1541
1542        encode_utf8_from_utf16_with_output_limit(
1543            &[0x263A, 0x0062, 0x00A7],
1544            "\u{263A}\u{62}\u{A7}",
1545            6,
1546            EncoderResult::InputEmpty,
1547        );
1548        encode_utf8_from_utf16_with_output_limit(
1549            &[0x263A, 0x0062, 0x00A7],
1550            "\u{263A}\u{62}",
1551            5,
1552            EncoderResult::OutputFull,
1553        );
1554
1555        encode_utf8_from_utf16_with_output_limit(
1556            &[0x0063, 0x00B6, 0x2603],
1557            "\u{63}\u{B6}\u{2603}",
1558            6,
1559            EncoderResult::InputEmpty,
1560        );
1561        encode_utf8_from_utf16_with_output_limit(
1562            &[0x0063, 0x00B6, 0x2603],
1563            "\u{63}\u{B6}",
1564            5,
1565            EncoderResult::OutputFull,
1566        );
1567
1568        encode_utf8_from_utf16_with_output_limit(
1569            &[0x263A, 0x2603],
1570            "\u{263A}\u{2603}",
1571            6,
1572            EncoderResult::InputEmpty,
1573        );
1574        encode_utf8_from_utf16_with_output_limit(
1575            &[0x263A, 0x2603],
1576            "\u{263A}",
1577            5,
1578            EncoderResult::OutputFull,
1579        );
1580
1581        encode_utf8_from_utf16_with_output_limit(
1582            &[0x0063, 0x00B6, 0xD83D],
1583            "\u{63}\u{B6}\u{FFFD}",
1584            6,
1585            EncoderResult::InputEmpty,
1586        );
1587        encode_utf8_from_utf16_with_output_limit(
1588            &[0x0063, 0x00B6, 0xD83D],
1589            "\u{63}\u{B6}",
1590            5,
1591            EncoderResult::OutputFull,
1592        );
1593
1594        encode_utf8_from_utf16_with_output_limit(
1595            &[0x263A, 0xD83D],
1596            "\u{263A}\u{FFFD}",
1597            6,
1598            EncoderResult::InputEmpty,
1599        );
1600        encode_utf8_from_utf16_with_output_limit(
1601            &[0x263A, 0xD83D],
1602            "\u{263A}",
1603            5,
1604            EncoderResult::OutputFull,
1605        );
1606
1607        encode_utf8_from_utf16_with_output_limit(
1608            &[0x0063, 0x00B6, 0xDCA9],
1609            "\u{63}\u{B6}\u{FFFD}",
1610            6,
1611            EncoderResult::InputEmpty,
1612        );
1613        encode_utf8_from_utf16_with_output_limit(
1614            &[0x0063, 0x00B6, 0xDCA9],
1615            "\u{63}\u{B6}",
1616            5,
1617            EncoderResult::OutputFull,
1618        );
1619
1620        encode_utf8_from_utf16_with_output_limit(
1621            &[0x263A, 0xDCA9],
1622            "\u{263A}\u{FFFD}",
1623            6,
1624            EncoderResult::InputEmpty,
1625        );
1626        encode_utf8_from_utf16_with_output_limit(
1627            &[0x263A, 0xDCA9],
1628            "\u{263A}",
1629            5,
1630            EncoderResult::OutputFull,
1631        );
1632    }
1633
1634    #[test]
1635    fn test_utf8_max_length_from_utf16() {
1636        let mut encoder = UTF_8.new_encoder();
1637        let mut output = [0u8; 13];
1638        let input = &[0x2C9Fu16, 0x2CA9u16, 0x2CA3u16, 0x2C9Fu16];
1639        let needed = encoder
1640            .max_buffer_length_from_utf16_without_replacement(input.len())
1641            .unwrap();
1642        let (result, _, _) =
1643            encoder.encode_from_utf16_without_replacement(input, &mut output[..needed], true);
1644        assert_eq!(result, EncoderResult::InputEmpty);
1645    }
1646
1647    #[test]
1648    fn test_decode_bom_prefixed_split_byte_triple() {
1649        let mut output = [0u16; 20];
1650        let mut decoder = UTF_8.new_decoder();
1651        {
1652            let needed = decoder.max_utf16_buffer_length(1).unwrap();
1653            let (result, read, written, had_errors) =
1654                decoder.decode_to_utf16(b"\xEF", &mut output[..needed], false);
1655            assert_eq!(result, CoderResult::InputEmpty);
1656            assert_eq!(read, 1);
1657            assert_eq!(written, 0);
1658            assert!(!had_errors);
1659        }
1660        {
1661            let needed = decoder.max_utf16_buffer_length(1).unwrap();
1662            let (result, read, written, had_errors) =
1663                decoder.decode_to_utf16(b"\xBF", &mut output[..needed], false);
1664            assert_eq!(result, CoderResult::InputEmpty);
1665            assert_eq!(read, 1);
1666            assert_eq!(written, 0);
1667            assert!(!had_errors);
1668        }
1669        {
1670            let needed = decoder.max_utf16_buffer_length(1).unwrap();
1671            let (result, read, written, had_errors) =
1672                decoder.decode_to_utf16(b"\xBE", &mut output[..needed], true);
1673            assert_eq!(result, CoderResult::InputEmpty);
1674            assert_eq!(read, 1);
1675            assert_eq!(written, 1);
1676            assert!(!had_errors);
1677            assert_eq!(output[0], 0xFFFE);
1678        }
1679    }
1680
1681    #[test]
1682    fn test_decode_bom_prefixed_split_byte_pair() {
1683        let mut output = [0u16; 20];
1684        let mut decoder = UTF_8.new_decoder();
1685        {
1686            let needed = decoder.max_utf16_buffer_length(1).unwrap();
1687            let (result, read, written, had_errors) =
1688                decoder.decode_to_utf16(b"\xEF", &mut output[..needed], false);
1689            assert_eq!(result, CoderResult::InputEmpty);
1690            assert_eq!(read, 1);
1691            assert_eq!(written, 0);
1692            assert!(!had_errors);
1693        }
1694        {
1695            let needed = decoder.max_utf16_buffer_length(1).unwrap();
1696            let (result, read, written, had_errors) =
1697                decoder.decode_to_utf16(b"\xBC", &mut output[..needed], true);
1698            assert_eq!(result, CoderResult::InputEmpty);
1699            assert_eq!(read, 1);
1700            assert_eq!(written, 1);
1701            assert!(had_errors);
1702            assert_eq!(output[0], 0xFFFD);
1703        }
1704    }
1705
1706    #[test]
1707    fn test_decode_bom_prefix() {
1708        let mut output = [0u16; 20];
1709        let mut decoder = UTF_8.new_decoder();
1710        {
1711            let needed = decoder.max_utf16_buffer_length(1).unwrap();
1712            let (result, read, written, had_errors) =
1713                decoder.decode_to_utf16(b"\xEF", &mut output[..needed], true);
1714            assert_eq!(result, CoderResult::InputEmpty);
1715            assert_eq!(read, 1);
1716            assert_eq!(written, 1);
1717            assert!(had_errors);
1718            assert_eq!(output[0], 0xFFFD);
1719        }
1720    }
1721
1722    #[test]
1723    fn test_tail() {
1724        let mut output = [0u16; 1];
1725        let mut decoder = UTF_8.new_decoder_without_bom_handling();
1726        {
1727            let (result, read, written, had_errors) =
1728                decoder.decode_to_utf16("\u{E4}a".as_bytes(), &mut output[..], false);
1729            assert_eq!(result, CoderResult::OutputFull);
1730            assert_eq!(read, 2);
1731            assert_eq!(written, 1);
1732            assert!(!had_errors);
1733            assert_eq!(output[0], 0x00E4);
1734        }
1735    }
1736}
encoding_rs/utf_8.rs

encoding_rs/
utf_8.rs
