encoding_rs/
utf_16.rs

1// Copyright Mozilla Foundation. See the COPYRIGHT
2// file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use super::*;
11use crate::handles::*;
12use crate::variant::*;
13
14pub struct Utf16Decoder {
15    lead_surrogate: u16, // If non-zero and pending_bmp == false, a pending lead surrogate
16    lead_byte: Option<u8>,
17    be: bool,
18    pending_bmp: bool, // if true, lead_surrogate is actually pending BMP
19}
20
21impl Utf16Decoder {
22    pub fn new(big_endian: bool) -> VariantDecoder {
23        VariantDecoder::Utf16(Utf16Decoder {
24            lead_surrogate: 0,
25            lead_byte: None,
26            be: big_endian,
27            pending_bmp: false,
28        })
29    }
30
31    pub fn additional_from_state(&self) -> usize {
32        1 + if self.lead_byte.is_some() { 1 } else { 0 }
33            + if self.lead_surrogate == 0 { 0 } else { 2 }
34    }
35
36    pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
37        checked_add(
38            1,
39            checked_div(byte_length.checked_add(self.additional_from_state()), 2),
40        )
41    }
42
43    pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> {
44        checked_add(
45            1,
46            checked_mul(
47                3,
48                checked_div(byte_length.checked_add(self.additional_from_state()), 2),
49            ),
50        )
51    }
52
53    pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> {
54        checked_add(
55            1,
56            checked_mul(
57                3,
58                checked_div(byte_length.checked_add(self.additional_from_state()), 2),
59            ),
60        )
61    }
62
63    decoder_functions!(
64        preamble = {
65            if self.pending_bmp {
66                match dest.check_space_bmp() {
67                    Space::Full(_) => {
68                        return (DecoderResult::OutputFull, 0, 0);
69                    }
70                    Space::Available(destination_handle) => {
71                        destination_handle.write_bmp(self.lead_surrogate);
72                        self.pending_bmp = false;
73                        self.lead_surrogate = 0;
74                    }
75                }
76            }
77        },
78        loop_preamble = {
79            // This is the fast path. The rest runs only at the
80            // start and end for partial sequences.
81            if self.lead_byte.is_none()
82                && self.lead_surrogate == 0
83                && let Some((read, written)) = if self.be {
84                    dest.copy_utf16_from::<BigEndian>(&mut source)
85                } else {
86                    dest.copy_utf16_from::<LittleEndian>(&mut source)
87                }
88            {
89                return (DecoderResult::Malformed(2, 0), read, written);
90            }
91        },
92        eof = {
93            debug_assert!(!self.pending_bmp);
94            if self.lead_surrogate != 0 || self.lead_byte.is_some() {
95                // We need to check space without intent to write in order to
96                // make sure that there is space for the replacement character.
97                match dest.check_space_bmp() {
98                    Space::Full(_) => {
99                        return (DecoderResult::OutputFull, 0, 0);
100                    }
101                    Space::Available(_) => {
102                        if self.lead_surrogate != 0 {
103                            self.lead_surrogate = 0;
104                            match self.lead_byte {
105                                None => {
106                                    return (
107                                        DecoderResult::Malformed(2, 0),
108                                        src_consumed,
109                                        dest.written(),
110                                    );
111                                }
112                                Some(_) => {
113                                    self.lead_byte = None;
114                                    return (
115                                        DecoderResult::Malformed(3, 0),
116                                        src_consumed,
117                                        dest.written(),
118                                    );
119                                }
120                            }
121                        }
122                        debug_assert!(self.lead_byte.is_some());
123                        self.lead_byte = None;
124                        return (DecoderResult::Malformed(1, 0), src_consumed, dest.written());
125                    }
126                }
127            }
128        },
129        body = {
130            match self.lead_byte {
131                None => {
132                    self.lead_byte = Some(b);
133                    continue;
134                }
135                Some(lead) => {
136                    self.lead_byte = None;
137                    let code_unit = if self.be {
138                        u16::from(lead) << 8 | u16::from(b)
139                    } else {
140                        u16::from(b) << 8 | u16::from(lead)
141                    };
142                    let high_bits = code_unit & 0xFC00u16;
143                    if high_bits == 0xD800u16 {
144                        // high surrogate
145                        if self.lead_surrogate != 0 {
146                            // The previous high surrogate was in
147                            // error and this one becomes the new
148                            // pending one.
149                            self.lead_surrogate = code_unit;
150                            return (
151                                DecoderResult::Malformed(2, 2),
152                                unread_handle.consumed(),
153                                destination_handle.written(),
154                            );
155                        }
156                        self.lead_surrogate = code_unit;
157                        continue;
158                    }
159                    if high_bits == 0xDC00u16 {
160                        // low surrogate
161                        if self.lead_surrogate == 0 {
162                            return (
163                                DecoderResult::Malformed(2, 0),
164                                unread_handle.consumed(),
165                                destination_handle.written(),
166                            );
167                        }
168                        destination_handle.write_surrogate_pair(self.lead_surrogate, code_unit);
169                        self.lead_surrogate = 0;
170                        continue;
171                    }
172                    // bmp
173                    if self.lead_surrogate != 0 {
174                        // The previous high surrogate was in
175                        // error and this code unit becomes a
176                        // pending BMP character.
177                        self.lead_surrogate = code_unit;
178                        self.pending_bmp = true;
179                        return (
180                            DecoderResult::Malformed(2, 2),
181                            unread_handle.consumed(),
182                            destination_handle.written(),
183                        );
184                    }
185                    destination_handle.write_bmp(code_unit);
186                    continue;
187                }
188            }
189        },
190        self = self,
191        src_consumed = src_consumed,
192        dest = dest,
193        source = source,
194        byte = b,
195        destination_handle = destination_handle,
196        unread_handle = unread_handle,
197        destination_check = check_space_astral
198    );
199}
200
201// Any copyright to the test code below this comment is dedicated to the
202// Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
203
204#[cfg(all(test, feature = "alloc"))]
205mod tests {
206    use super::super::testing::*;
207    use super::super::*;
208
209    fn decode_utf_16le(bytes: &[u8], expect: &str) {
210        decode_without_padding(UTF_16LE, bytes, expect);
211    }
212
213    fn decode_utf_16be(bytes: &[u8], expect: &str) {
214        decode_without_padding(UTF_16BE, bytes, expect);
215    }
216
217    fn encode_utf_16le(string: &str, expect: &[u8]) {
218        encode(UTF_16LE, string, expect);
219    }
220
221    fn encode_utf_16be(string: &str, expect: &[u8]) {
222        encode(UTF_16BE, string, expect);
223    }
224
225    #[test]
226    fn test_utf_16_decode() {
227        decode_utf_16le(b"", "");
228        decode_utf_16be(b"", "");
229
230        decode_utf_16le(b"\x61\x00\x62\x00", "\u{0061}\u{0062}");
231        decode_utf_16be(b"\x00\x61\x00\x62", "\u{0061}\u{0062}");
232
233        decode_utf_16le(b"\xFE\xFF\x00\x61\x00\x62", "\u{0061}\u{0062}");
234        decode_utf_16be(b"\xFF\xFE\x61\x00\x62\x00", "\u{0061}\u{0062}");
235
236        decode_utf_16le(b"\x61\x00\x62", "\u{0061}\u{FFFD}");
237        decode_utf_16be(b"\x00\x61\x00", "\u{0061}\u{FFFD}");
238
239        decode_utf_16le(b"\x3D\xD8\xA9", "\u{FFFD}");
240        decode_utf_16be(b"\xD8\x3D\xDC", "\u{FFFD}");
241
242        decode_utf_16le(b"\x3D\xD8\xA9\xDC\x03\x26", "\u{1F4A9}\u{2603}");
243        decode_utf_16be(b"\xD8\x3D\xDC\xA9\x26\x03", "\u{1F4A9}\u{2603}");
244
245        decode_utf_16le(b"\xA9\xDC\x03\x26", "\u{FFFD}\u{2603}");
246        decode_utf_16be(b"\xDC\xA9\x26\x03", "\u{FFFD}\u{2603}");
247
248        decode_utf_16le(b"\x3D\xD8\x03\x26", "\u{FFFD}\u{2603}");
249        decode_utf_16be(b"\xD8\x3D\x26\x03", "\u{FFFD}\u{2603}");
250
251        // The \xFF makes sure that the parts before and after have different alignment
252        let long_le = b"\x00\x00\x00\x00\x00\x00\x00\x00\x3D\xD8\xA9\xDC\x00\x00\x00\x00\x00\x00\x00\x00\x3D\xD8\x00\x00\x00\x00\x00\x00\x00\x00\xA9\xDC\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3D\xD8\xFF\x00\x00\x00\x00\x00\x00\x00\x00\x3D\xD8\xA9\xDC\x00\x00\x00\x00\x00\x00\x00\x00\x3D\xD8\x00\x00\x00\x00\x00\x00\x00\x00\xA9\xDC\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3D\xD8";
253        let long_be = b"\x00\x00\x00\x00\x00\x00\x00\x00\xD8\x3D\xDC\xA9\x00\x00\x00\x00\x00\x00\x00\x00\xD8\x3D\x00\x00\x00\x00\x00\x00\x00\x00\xDC\xA9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xD8\x3D\xFF\x00\x00\x00\x00\x00\x00\x00\x00\xD8\x3D\xDC\xA9\x00\x00\x00\x00\x00\x00\x00\x00\xD8\x3D\x00\x00\x00\x00\x00\x00\x00\x00\xDC\xA9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xD8\x3D";
254        let long_expect = "\x00\x00\x00\x00\u{1F4A9}\x00\x00\x00\x00\u{FFFD}\x00\x00\x00\x00\u{FFFD}\x00\x00\x00\x00\x00\x00\x00\x00\u{FFFD}";
255        decode_utf_16le(&long_le[..long_le.len() / 2], long_expect);
256        decode_utf_16be(&long_be[..long_be.len() / 2], long_expect);
257        decode_utf_16le(&long_le[long_le.len() / 2 + 1..], long_expect);
258        decode_utf_16be(&long_be[long_be.len() / 2 + 1..], long_expect);
259    }
260
261    #[test]
262    fn test_utf_16_encode() {
263        // Empty
264        encode_utf_16be("", b"");
265        encode_utf_16le("", b"");
266
267        // Encodes as UTF-8
268        assert_eq!(UTF_16LE.new_encoder().encoding(), UTF_8);
269        assert_eq!(UTF_16BE.new_encoder().encoding(), UTF_8);
270        encode_utf_16le("\u{1F4A9}\u{2603}", "\u{1F4A9}\u{2603}".as_bytes());
271        encode_utf_16be("\u{1F4A9}\u{2603}", "\u{1F4A9}\u{2603}".as_bytes());
272    }
273
274    #[test]
275    fn test_utf_16be_decode_one_by_one() {
276        let input = b"\x00\x61\x00\xE4\x26\x03\xD8\x3D\xDC\xA9";
277        let mut output = [0u16; 20];
278        let mut decoder = UTF_16BE.new_decoder();
279        for b in input.chunks(1) {
280            assert_eq!(b.len(), 1);
281            let needed = decoder.max_utf16_buffer_length(b.len()).unwrap();
282            let (result, read, _, had_errors) =
283                decoder.decode_to_utf16(b, &mut output[..needed], false);
284            assert_eq!(result, CoderResult::InputEmpty);
285            assert_eq!(read, 1);
286            assert!(!had_errors);
287        }
288    }
289
290    #[test]
291    fn test_utf_16le_decode_one_by_one() {
292        let input = b"\x61\x00\xE4\x00\x03\x26\x3D\xD8\xA9\xDC";
293        let mut output = [0u16; 20];
294        let mut decoder = UTF_16LE.new_decoder();
295        for b in input.chunks(1) {
296            assert_eq!(b.len(), 1);
297            let needed = decoder.max_utf16_buffer_length(b.len()).unwrap();
298            let (result, read, _, had_errors) =
299                decoder.decode_to_utf16(b, &mut output[..needed], false);
300            assert_eq!(result, CoderResult::InputEmpty);
301            assert_eq!(read, 1);
302            assert!(!had_errors);
303        }
304    }
305
306    #[test]
307    fn test_utf_16be_decode_three_at_a_time() {
308        let input = b"\x00\xE4\x26\x03\xD8\x3D\xDC\xA9\x00\x61\x00\xE4";
309        let mut output = [0u16; 20];
310        let mut decoder = UTF_16BE.new_decoder();
311        for b in input.chunks(3) {
312            assert_eq!(b.len(), 3);
313            let needed = decoder.max_utf16_buffer_length(b.len()).unwrap();
314            let (result, read, _, had_errors) =
315                decoder.decode_to_utf16(b, &mut output[..needed], false);
316            assert_eq!(result, CoderResult::InputEmpty);
317            assert_eq!(read, b.len());
318            assert!(!had_errors);
319        }
320    }
321
322    #[test]
323    fn test_utf_16le_decode_three_at_a_time() {
324        let input = b"\xE4\x00\x03\x26\x3D\xD8\xA9\xDC\x61\x00\xE4\x00";
325        let mut output = [0u16; 20];
326        let mut decoder = UTF_16LE.new_decoder();
327        for b in input.chunks(3) {
328            assert_eq!(b.len(), 3);
329            let needed = decoder.max_utf16_buffer_length(b.len()).unwrap();
330            let (result, read, _, had_errors) =
331                decoder.decode_to_utf16(b, &mut output[..needed], false);
332            assert_eq!(result, CoderResult::InputEmpty);
333            assert_eq!(read, b.len());
334            assert!(!had_errors);
335        }
336    }
337
338    #[test]
339    fn test_utf_16le_decode_bom_prefixed_split_byte_pair() {
340        let mut output = [0u16; 20];
341        let mut decoder = UTF_16LE.new_decoder();
342        {
343            let needed = decoder.max_utf16_buffer_length(1).unwrap();
344            let (result, read, written, had_errors) =
345                decoder.decode_to_utf16(b"\xFF", &mut output[..needed], false);
346            assert_eq!(result, CoderResult::InputEmpty);
347            assert_eq!(read, 1);
348            assert_eq!(written, 0);
349            assert!(!had_errors);
350        }
351        {
352            let needed = decoder.max_utf16_buffer_length(1).unwrap();
353            let (result, read, written, had_errors) =
354                decoder.decode_to_utf16(b"\xFD", &mut output[..needed], true);
355            assert_eq!(result, CoderResult::InputEmpty);
356            assert_eq!(read, 1);
357            assert_eq!(written, 1);
358            assert!(!had_errors);
359            assert_eq!(output[0], 0xFDFF);
360        }
361    }
362
363    #[test]
364    fn test_utf_16be_decode_bom_prefixed_split_byte_pair() {
365        let mut output = [0u16; 20];
366        let mut decoder = UTF_16BE.new_decoder();
367        {
368            let needed = decoder.max_utf16_buffer_length(1).unwrap();
369            let (result, read, written, had_errors) =
370                decoder.decode_to_utf16(b"\xFE", &mut output[..needed], false);
371            assert_eq!(result, CoderResult::InputEmpty);
372            assert_eq!(read, 1);
373            assert_eq!(written, 0);
374            assert!(!had_errors);
375        }
376        {
377            let needed = decoder.max_utf16_buffer_length(1).unwrap();
378            let (result, read, written, had_errors) =
379                decoder.decode_to_utf16(b"\xFD", &mut output[..needed], true);
380            assert_eq!(result, CoderResult::InputEmpty);
381            assert_eq!(read, 1);
382            assert_eq!(written, 1);
383            assert!(!had_errors);
384            assert_eq!(output[0], 0xFEFD);
385        }
386    }
387
388    #[test]
389    fn test_utf_16le_decode_bom_prefix() {
390        let mut output = [0u16; 20];
391        let mut decoder = UTF_16LE.new_decoder();
392        {
393            let needed = decoder.max_utf16_buffer_length(1).unwrap();
394            let (result, read, written, had_errors) =
395                decoder.decode_to_utf16(b"\xFF", &mut output[..needed], true);
396            assert_eq!(result, CoderResult::InputEmpty);
397            assert_eq!(read, 1);
398            assert_eq!(written, 1);
399            assert!(had_errors);
400            assert_eq!(output[0], 0xFFFD);
401        }
402    }
403
404    #[test]
405    fn test_utf_16be_decode_bom_prefix() {
406        let mut output = [0u16; 20];
407        let mut decoder = UTF_16BE.new_decoder();
408        {
409            let needed = decoder.max_utf16_buffer_length(1).unwrap();
410            let (result, read, written, had_errors) =
411                decoder.decode_to_utf16(b"\xFE", &mut output[..needed], true);
412            assert_eq!(result, CoderResult::InputEmpty);
413            assert_eq!(read, 1);
414            assert_eq!(written, 1);
415            assert!(had_errors);
416            assert_eq!(output[0], 0xFFFD);
417        }
418    }
419
420    #[test]
421    fn test_utf_16le_decode_near_end() {
422        let mut output = [0u8; 4];
423        let mut decoder = UTF_16LE.new_decoder();
424        {
425            let (result, read, written, had_errors) =
426                decoder.decode_to_utf8(&[0x03], &mut output[..], false);
427            assert_eq!(result, CoderResult::InputEmpty);
428            assert_eq!(read, 1);
429            assert_eq!(written, 0);
430            assert!(!had_errors);
431            assert_eq!(output[0], 0x0);
432        }
433        {
434            let (result, read, written, had_errors) =
435                decoder.decode_to_utf8(&[0x26, 0x03, 0x26], &mut output[..], false);
436            assert_eq!(result, CoderResult::OutputFull);
437            assert_eq!(read, 1);
438            assert_eq!(written, 3);
439            assert!(!had_errors);
440            assert_eq!(output[0], 0xE2);
441            assert_eq!(output[1], 0x98);
442            assert_eq!(output[2], 0x83);
443            assert_eq!(output[3], 0x00);
444        }
445    }
446
447    #[test]
448    fn test_utf_16be_decode_near_end() {
449        let mut output = [0u8; 4];
450        let mut decoder = UTF_16BE.new_decoder();
451        {
452            let (result, read, written, had_errors) =
453                decoder.decode_to_utf8(&[0x26], &mut output[..], false);
454            assert_eq!(result, CoderResult::InputEmpty);
455            assert_eq!(read, 1);
456            assert_eq!(written, 0);
457            assert!(!had_errors);
458            assert_eq!(output[0], 0x0);
459        }
460        {
461            let (result, read, written, had_errors) =
462                decoder.decode_to_utf8(&[0x03, 0x26, 0x03], &mut output[..], false);
463            assert_eq!(result, CoderResult::OutputFull);
464            assert_eq!(read, 1);
465            assert_eq!(written, 3);
466            assert!(!had_errors);
467            assert_eq!(output[0], 0xE2);
468            assert_eq!(output[1], 0x98);
469            assert_eq!(output[2], 0x83);
470            assert_eq!(output[3], 0x00);
471        }
472    }
473}
encoding_rs/utf_16.rs

encoding_rs/
utf_16.rs
