encoding_rs/
x_user_defined.rs

1// Copyright Mozilla Foundation. See the COPYRIGHT
2// file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use super::*;
11use crate::handles::*;
12use crate::variant::*;
13
14cfg_if! {
15    if #[cfg(all(
16        feature = "simd-accel",
17        target_endian = "little",
18    ))] {
19        use crate::ascii::STRIDE;
20        use simd_funcs::*;
21        use core::simd::u8x16;
22        use core::simd::u16x16;
23        use core::simd::cmp::SimdPartialOrd;
24        #[rustversion::since(1.95)]
25        use core::simd::Select;
26
27        #[inline(always)]
28        fn shift_upper(unpacked: u16x16) -> u16x16 {
29            let highest_ascii = u16x16::splat(0x7F);
30            unpacked + unpacked.simd_gt(highest_ascii).select(u16x16::splat(0xF700), u16x16::splat(0))        }
31    } else {
32    }
33}
34
35pub struct UserDefinedDecoder;
36
37impl UserDefinedDecoder {
38    pub fn new() -> VariantDecoder {
39        VariantDecoder::UserDefined(UserDefinedDecoder)
40    }
41
42    pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
43        Some(byte_length)
44    }
45
46    pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> {
47        byte_length.checked_mul(3)
48    }
49
50    pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> {
51        byte_length.checked_mul(3)
52    }
53
54    decoder_function!(
55        {},
56        {},
57        {},
58        {
59            unread_handle.commit();
60
61            if b < 0x80 {
62                // ASCII run not optimized, because binary data expected
63                destination_handle.write_ascii(b);
64                continue;
65            }
66            destination_handle.write_upper_bmp(u16::from(b) + 0xF700);
67            continue;
68        },
69        self,
70        src_consumed,
71        dest,
72        source,
73        b,
74        destination_handle,
75        unread_handle,
76        check_space_bmp,
77        decode_to_utf8_raw,
78        u8,
79        Utf8Destination
80    );
81
82    #[cfg(not(all(feature = "simd-accel", target_endian = "little")))]
83    pub fn decode_to_utf16_raw(
84        &mut self,
85        src: &[u8],
86        dst: &mut [u16],
87        _last: bool,
88    ) -> (DecoderResult, usize, usize) {
89        let (pending, length) = if dst.len() < src.len() {
90            (DecoderResult::OutputFull, dst.len())
91        } else {
92            (DecoderResult::InputEmpty, src.len())
93        };
94        let src_trim = &src[..length];
95        let dst_trim = &mut dst[..length];
96        src_trim
97            .iter()
98            .zip(dst_trim.iter_mut())
99            .for_each(|(from, to)| {
100                *to = {
101                    let unit = *from;
102                    if unit < 0x80 {
103                        u16::from(unit)
104                    } else {
105                        u16::from(unit) + 0xF700
106                    }
107                }
108            });
109        (pending, length, length)
110    }
111
112    #[cfg(all(feature = "simd-accel", target_endian = "little"))]
113    #[inline(always)]
114    pub fn decode_to_utf16_raw(
115        &mut self,
116        src: &[u8],
117        dst: &mut [u16],
118        _last: bool,
119    ) -> (DecoderResult, usize, usize) {
120        let (pending, length) = if dst.len() < src.len() {
121            (DecoderResult::OutputFull, dst.len())
122        } else {
123            (DecoderResult::InputEmpty, src.len())
124        };
125        let (src_strides, src_tail) = src[..length].as_chunks::<STRIDE>();
126        let (dst_strides, dst_tail) = dst[..length].as_chunks_mut::<STRIDE>();
127        for (src_stride, dst_stride) in src_strides.iter().zip(dst_strides.iter_mut()) {
128            let src_simd: u8x16 = (*src_stride).into();
129            let unpacked = simd_unpack(src_simd);
130            let shifted = shift_upper(unpacked);
131            *dst_stride = shifted.to_array();
132        }
133        src_tail
134            .iter()
135            .zip(dst_tail.iter_mut())
136            .for_each(|(from, to)| {
137                *to = {
138                    let unit = *from;
139                    if unit < 0x80 {
140                        u16::from(unit)
141                    } else {
142                        u16::from(unit) + 0xF700
143                    }
144                }
145            });
146        (pending, length, length)
147    }
148}
149
150pub struct UserDefinedEncoder;
151
152impl UserDefinedEncoder {
153    pub fn new(encoding: &'static Encoding) -> Encoder {
154        Encoder::new(encoding, VariantEncoder::UserDefined(UserDefinedEncoder))
155    }
156
157    pub fn max_buffer_length_from_utf16_without_replacement(
158        &self,
159        u16_length: usize,
160    ) -> Option<usize> {
161        Some(u16_length)
162    }
163
164    pub fn max_buffer_length_from_utf8_without_replacement(
165        &self,
166        byte_length: usize,
167    ) -> Option<usize> {
168        Some(byte_length)
169    }
170
171    encoder_functions!(
172        eof = {},
173        body = {
174            if c <= '\u{7F}' {
175                // TODO optimize ASCII run
176                destination_handle.write_one(c as u8);
177                continue;
178            }
179            if !('\u{F780}'..='\u{F7FF}').contains(&c) {
180                return (
181                    EncoderResult::Unmappable(c),
182                    unread_handle.consumed(),
183                    destination_handle.written(),
184                );
185            }
186            destination_handle.write_one((u32::from(c) - 0xF700) as u8);
187            continue;
188        },
189        self = self,
190        src_consumed = src_consumed,
191        source = source,
192        dest = dest,
193        c = c,
194        destination_handle = destination_handle,
195        unread_handle = unread_handle,
196        destination_check = check_space_one
197    );
198}
199
200// Any copyright to the test code below this comment is dedicated to the
201// Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
202
203#[cfg(all(test, feature = "alloc"))]
204mod tests {
205    use super::super::testing::*;
206    use super::super::*;
207
208    fn decode_x_user_defined(bytes: &[u8], expect: &str) {
209        decode(X_USER_DEFINED, bytes, expect);
210    }
211
212    fn encode_x_user_defined(string: &str, expect: &[u8]) {
213        encode(X_USER_DEFINED, string, expect);
214    }
215
216    #[test]
217    fn test_x_user_defined_decode() {
218        // Empty
219        decode_x_user_defined(b"", "");
220
221        // ASCII
222        decode_x_user_defined(b"\x61\x62", "\u{0061}\u{0062}");
223
224        decode_x_user_defined(b"\x80\xFF", "\u{F780}\u{F7FF}");
225        decode_x_user_defined(
226            b"\x80\xFF\x61\x62\x80\xFF\x61\x62\x80\xFF\x61\x62\x80\xFF\x61\x62\x80\xFF\x61\x62",
227            "\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}",
228        );
229    }
230
231    #[test]
232    fn test_x_user_defined_encode() {
233        // Empty
234        encode_x_user_defined("", b"");
235
236        // ASCII
237        encode_x_user_defined("\u{0061}\u{0062}", b"\x61\x62");
238
239        encode_x_user_defined("\u{F780}\u{F7FF}", b"\x80\xFF");
240        encode_x_user_defined("\u{F77F}\u{F800}", b"&#63359;&#63488;");
241    }
242
243    #[test]
244    fn test_x_user_defined_from_two_low_surrogates() {
245        let expectation = b"&#65533;&#65533;";
246        let mut output = [0u8; 40];
247        let mut encoder = X_USER_DEFINED.new_encoder();
248        let (result, read, written, had_errors) =
249            encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
250        assert_eq!(result, CoderResult::InputEmpty);
251        assert_eq!(read, 2);
252        assert_eq!(written, expectation.len());
253        assert!(had_errors);
254        assert_eq!(&output[..written], expectation);
255    }
256}
encoding_rs/x_user_defined.rs

encoding_rs/
x_user_defined.rs
