encoding_rs/single_byte.rs
1// Copyright Mozilla Foundation. See the COPYRIGHT
2// file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use super::*;
11use crate::ascii::*;
12use crate::data::position;
13use crate::handles::*;
14use crate::variant::*;
15
16pub struct SingleByteDecoder {
17 table: &'static [u16; 128],
18}
19
20impl SingleByteDecoder {
21 pub fn new(data: &'static [u16; 128]) -> VariantDecoder {
22 VariantDecoder::SingleByte(SingleByteDecoder { table: data })
23 }
24
25 pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
26 Some(byte_length)
27 }
28
29 pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> {
30 byte_length.checked_mul(3)
31 }
32
33 pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> {
34 byte_length.checked_mul(3)
35 }
36
37 pub fn decode_to_utf8_raw(
38 &mut self,
39 src: &[u8],
40 dst: &mut [u8],
41 _last: bool,
42 ) -> (DecoderResult, usize, usize) {
43 let mut source = ByteSource::new(src);
44 let mut dest = Utf8Destination::new(dst);
45 'outermost: loop {
46 match dest.copy_ascii_from_check_space_bmp(&mut source) {
47 CopyAsciiResult::Stop(ret) => return ret,
48 CopyAsciiResult::GoOn((mut non_ascii, mut handle)) => 'middle: loop {
49 // Start non-boilerplate
50 //
51 // Since the non-ASCIIness of `non_ascii` is hidden from
52 // the optimizer, it can't figure out that it's OK to
53 // statically omit the bound check when accessing
54 // `[u16; 128]` with an index
55 // `non_ascii as usize - 0x80usize`.
56 //
57 // Safety: `non_ascii` is a u8 byte >=0x80, from the invariants
58 // on Utf8Destination::copy_ascii_from_check_space_bmp()
59 let mapped =
60 unsafe { *(self.table.get_unchecked(non_ascii as usize - 0x80usize)) };
61 // let mapped = self.table[non_ascii as usize - 0x80usize];
62 if mapped == 0u16 {
63 return (
64 DecoderResult::Malformed(1, 0),
65 source.consumed(),
66 handle.written(),
67 );
68 }
69 let dest_again = handle.write_bmp_excl_ascii(mapped);
70 // End non-boilerplate
71 match source.check_available() {
72 Space::Full(src_consumed) => {
73 return (
74 DecoderResult::InputEmpty,
75 src_consumed,
76 dest_again.written(),
77 );
78 }
79 Space::Available(source_handle) => {
80 match dest_again.check_space_bmp() {
81 Space::Full(dst_written) => {
82 return (
83 DecoderResult::OutputFull,
84 source_handle.consumed(),
85 dst_written,
86 );
87 }
88 Space::Available(mut destination_handle) => {
89 let (mut b, unread_handle) = source_handle.read();
90 let source_again = unread_handle.commit();
91 'innermost: loop {
92 if b > 127 {
93 non_ascii = b;
94 handle = destination_handle;
95 continue 'middle;
96 }
97 // Testing on Haswell says that we should write the
98 // byte unconditionally instead of trying to unread it
99 // to make it part of the next SIMD stride.
100 let dest_again_again = destination_handle.write_ascii(b);
101 if b < 60 {
102 // We've got punctuation
103 match source_again.check_available() {
104 Space::Full(src_consumed_again) => {
105 return (
106 DecoderResult::InputEmpty,
107 src_consumed_again,
108 dest_again_again.written(),
109 );
110 }
111 Space::Available(source_handle_again) => {
112 match dest_again_again.check_space_bmp() {
113 Space::Full(dst_written_again) => {
114 return (
115 DecoderResult::OutputFull,
116 source_handle_again.consumed(),
117 dst_written_again,
118 );
119 }
120 Space::Available(
121 destination_handle_again,
122 ) => {
123 let (b_again, unread_handle_again) =
124 source_handle_again.read();
125 unread_handle_again.commit();
126 b = b_again;
127 destination_handle =
128 destination_handle_again;
129 continue 'innermost;
130 }
131 }
132 }
133 }
134 }
135 // We've got markup or ASCII text
136 continue 'outermost;
137 }
138 }
139 }
140 }
141 }
142 },
143 }
144 }
145 }
146
147 #[inline(always)]
148 pub fn decode_to_utf16_raw(
149 &mut self,
150 src: &[u8],
151 dst: &mut [u16],
152 _last: bool,
153 ) -> (DecoderResult, usize, usize) {
154 let (pending, length) = if dst.len() < src.len() {
155 (DecoderResult::OutputFull, dst.len())
156 } else {
157 (DecoderResult::InputEmpty, src.len())
158 };
159 // Safety invariant: converted <= length. Quite often we have `converted < length`
160 // which will be separately marked.
161 let mut converted = 0usize;
162 'outermost: loop {
163 // Safety: length is the minimum length, `src/dst + x` will always be valid for reads/writes of `len - x`
164 match ascii_to_basic_latin(&src[converted..], &mut dst[converted..]) {
165 None => {
166 return (pending, length, length);
167 }
168 Some((mut non_ascii, consumed)) => {
169 // Safety invariant: `converted <= length` upheld, since this can only consume
170 // up to `length - converted` bytes.
171 //
172 // Furthermore, in this context,
173 // we can assume `converted < length` since this branch is only ever hit when
174 // ascii_to_basic_latin fails to consume the entire slice
175 converted += consumed;
176 'middle: loop {
177 // `converted` doesn't count the reading of `non_ascii` yet.
178 // Since the non-ASCIIness of `non_ascii` is hidden from
179 // the optimizer, it can't figure out that it's OK to
180 // statically omit the bound check when accessing
181 // `[u16; 128]` with an index
182 // `non_ascii as usize - 0x80usize`.
183 //
184 // Safety: We can rely on `non_ascii` being between `0x80` and `0xFF` due to
185 // the invariants of `ascii_to_basic_latin()`, and our table has enough space for that.
186 let mapped =
187 unsafe { *(self.table.get_unchecked(non_ascii as usize - 0x80usize)) };
188 // let mapped = self.table[non_ascii as usize - 0x80usize];
189 if mapped == 0u16 {
190 return (
191 DecoderResult::Malformed(1, 0),
192 converted + 1, // +1 `for non_ascii`
193 converted,
194 );
195 }
196 unsafe {
197 // Safety: As mentioned above, `converted < length`
198 *(dst.get_unchecked_mut(converted)) = mapped;
199 }
200 // Safety: `converted <= length` upheld, since `converted < length` before this
201 converted += 1;
202 // Next, handle ASCII punctuation and non-ASCII without
203 // going back to ASCII acceleration. Non-ASCII scripts
204 // use ASCII punctuation, so this avoid going to
205 // acceleration just for punctuation/space and then
206 // failing. This is a significant boost to non-ASCII
207 // scripts.
208 // TODO: Split out Latin converters without this part
209 // this stuff makes Latin script-conversion slower.
210 if converted == length {
211 return (pending, length, length);
212 }
213 // Safety: We are back to `converted < length` because of the == above
214 // and can perform this check.
215 let mut b = unsafe { *(src.get_unchecked(converted)) };
216 // Safety: `converted < length` is upheld for this loop
217 'innermost: loop {
218 if b > 127 {
219 non_ascii = b;
220 continue 'middle;
221 }
222 // Testing on Haswell says that we should write the
223 // byte unconditionally instead of trying to unread it
224 // to make it part of the next SIMD stride.
225 unsafe {
226 // Safety: `converted < length` is true for this loop
227 *(dst.get_unchecked_mut(converted)) = u16::from(b);
228 }
229 // Safety: We are now at `converted <= length`. We should *not* `continue`
230 // the loop without reverifying
231 converted += 1;
232 if b < 60 {
233 // We've got punctuation
234 if converted == length {
235 return (pending, length, length);
236 }
237 // Safety: we're back to `converted <= length` because of the == above
238 b = unsafe { *(src.get_unchecked(converted)) };
239 // Safety: The loop continues as `converted < length`
240 continue 'innermost;
241 }
242 // We've got markup or ASCII text
243 continue 'outermost;
244 }
245 }
246 }
247 }
248 }
249 }
250
251 pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> usize {
252 let mut bytes = buffer;
253 let mut total = 0;
254 loop {
255 if let Some((non_ascii, offset)) = validate_ascii(bytes) {
256 total += offset;
257 // Safety: We can rely on `non_ascii` being between `0x80` and `0xFF` due to
258 // the invariants of `ascii_to_basic_latin()`, and our table has enough space for that.
259 let mapped = unsafe { *(self.table.get_unchecked(non_ascii as usize - 0x80usize)) };
260 if mapped != u16::from(non_ascii) {
261 return total;
262 }
263 total += 1;
264 bytes = &bytes[offset + 1..];
265 } else {
266 return total;
267 }
268 }
269 }
270}
271
272pub struct SingleByteEncoder {
273 table: &'static [u16; 128],
274 run_bmp_offset: usize,
275 run_byte_offset: usize,
276 run_length: usize,
277}
278
279impl SingleByteEncoder {
280 pub fn new(
281 encoding: &'static Encoding,
282 data: &'static [u16; 128],
283 run_bmp_offset: u16,
284 run_byte_offset: u8,
285 run_length: u8,
286 ) -> Encoder {
287 Encoder::new(
288 encoding,
289 VariantEncoder::SingleByte(SingleByteEncoder {
290 table: data,
291 run_bmp_offset: run_bmp_offset as usize,
292 run_byte_offset: run_byte_offset as usize,
293 run_length: run_length as usize,
294 }),
295 )
296 }
297
298 pub fn max_buffer_length_from_utf16_without_replacement(
299 &self,
300 u16_length: usize,
301 ) -> Option<usize> {
302 Some(u16_length)
303 }
304
305 pub fn max_buffer_length_from_utf8_without_replacement(
306 &self,
307 byte_length: usize,
308 ) -> Option<usize> {
309 Some(byte_length)
310 }
311
312 #[inline(always)]
313 fn encode_u16(&self, code_unit: u16) -> Option<u8> {
314 // First, we see if the code unit falls into a run of consecutive
315 // code units that can be mapped by offset. This is very efficient
316 // for most non-Latin encodings as well as Latin1-ish encodings.
317 //
318 // For encodings that don't fit this pattern, the run (which may
319 // have the length of just one) just establishes the starting point
320 // for the next rule.
321 //
322 // Next, we do a forward linear search in the part of the index
323 // after the run. Even in non-Latin1-ish Latin encodings (except
324 // macintosh), the lower case letters are here.
325 //
326 // Next, we search the third quadrant up to the start of the run
327 // (upper case letters in Latin encodings except macintosh, in
328 // Greek and in KOI encodings) and then the second quadrant,
329 // except if the run stared before the third quadrant, we search
330 // the second quadrant up to the run.
331 //
332 // Last, we search the first quadrant, which has unused controls
333 // or punctuation in most encodings. This is bad for macintosh
334 // and IBM866, but those are rare.
335
336 // Run of consecutive units
337 let unit_as_usize = code_unit as usize;
338 let offset = unit_as_usize.wrapping_sub(self.run_bmp_offset);
339 if offset < self.run_length {
340 return Some((128 + self.run_byte_offset + offset) as u8);
341 }
342
343 // Search after the run
344 let tail_start = self.run_byte_offset + self.run_length;
345 if let Some(pos) = position(&self.table[tail_start..], code_unit) {
346 return Some((128 + tail_start + pos) as u8);
347 }
348
349 if self.run_byte_offset >= 64 {
350 // Search third quadrant before the run
351 if let Some(pos) = position(&self.table[64..self.run_byte_offset], code_unit) {
352 return Some(((128 + 64) + pos) as u8);
353 }
354
355 // Search second quadrant
356 if let Some(pos) = position(&self.table[32..64], code_unit) {
357 return Some(((128 + 32) + pos) as u8);
358 }
359 } else if let Some(pos) = position(&self.table[32..self.run_byte_offset], code_unit) {
360 // windows-1252, windows-874, ISO-8859-15 and ISO-8859-5
361 // Search second quadrant before the run
362 return Some(((128 + 32) + pos) as u8);
363 }
364
365 // Search first quadrant
366 if let Some(pos) = position(&self.table[..32], code_unit) {
367 return Some((128 + pos) as u8);
368 }
369
370 None
371 }
372
373 ascii_compatible_bmp_encoder_function!(
374 {
375 match self.encode_u16(bmp) {
376 Some(byte) => handle.write_one(byte),
377 None => {
378 return (
379 EncoderResult::unmappable_from_bmp(bmp),
380 source.consumed(),
381 handle.written(),
382 );
383 }
384 }
385 },
386 bmp,
387 self,
388 source,
389 handle,
390 copy_ascii_to_check_space_one,
391 check_space_one,
392 encode_from_utf8_raw,
393 str,
394 Utf8Source,
395 true
396 );
397
398 pub fn encode_from_utf16_raw(
399 &mut self,
400 src: &[u16],
401 dst: &mut [u8],
402 _last: bool,
403 ) -> (EncoderResult, usize, usize) {
404 let (pending, length) = if dst.len() < src.len() {
405 (EncoderResult::OutputFull, dst.len())
406 } else {
407 (EncoderResult::InputEmpty, src.len())
408 };
409 // Safety invariant: converted <= length. Quite often we have `converted < length`
410 // which will be separately marked.
411 let mut converted = 0usize;
412 'outermost: loop {
413 // Safety: length is the minimum length, `src/dst + x` will always be valid for reads/writes of `len - x`
414 match basic_latin_to_ascii(&src[converted..], &mut dst[converted..]) {
415 None => {
416 return (pending, length, length);
417 }
418 Some((mut non_ascii, consumed)) => {
419 // Safety invariant: `converted <= length` upheld, since this can only consume
420 // up to `length - converted` bytes.
421 //
422 // Furthermore, in this context,
423 // we can assume `converted < length` since this branch is only ever hit when
424 // ascii_to_basic_latin fails to consume the entire slice
425 converted += consumed;
426 'middle: loop {
427 // `converted` doesn't count the reading of `non_ascii` yet.
428 match self.encode_u16(non_ascii) {
429 Some(byte) => {
430 unsafe {
431 // Safety: we're allowed this access since `converted < length`
432 *(dst.get_unchecked_mut(converted)) = byte;
433 }
434 converted += 1;
435 // `converted <= length` now
436 }
437 None => {
438 // At this point, we need to know if we
439 // have a surrogate.
440 let high_bits = non_ascii & 0xFC00u16;
441 if high_bits == 0xD800u16 {
442 // high surrogate
443 if converted + 1 == length {
444 // End of buffer. This surrogate is unpaired.
445 return (
446 EncoderResult::Unmappable('\u{FFFD}'),
447 converted + 1, // +1 `for non_ascii`
448 converted,
449 );
450 }
451 // Safety: convered < length from outside the match, and `converted + 1 != length`,
452 // So `converted + 1 < length` as well. We're in bounds
453 let second =
454 u32::from(unsafe { *src.get_unchecked(converted + 1) });
455 if second & 0xFC00u32 != 0xDC00u32 {
456 return (
457 EncoderResult::Unmappable('\u{FFFD}'),
458 converted + 1, // +1 `for non_ascii`
459 converted,
460 );
461 }
462 // The next code unit is a low surrogate.
463 let astral: char = unsafe {
464 // Safety: We can rely on non_ascii being 0xD800-0xDBFF since the high bits are 0xD800
465 // Then, (non_ascii << 10 - 0xD800 << 10) becomes between (0 to 0x3FF) << 10, which is between
466 // 0x400 to 0xffc00. Adding the 0x10000 gives a range of 0x10400 to 0x10fc00. Subtracting the 0xDC00
467 // gives 0x2800 to 0x102000
468 // The second term is between 0xDC00 and 0xDFFF from the check above. This gives a maximum
469 // possible range of (0x10400 + 0xDC00) to (0x102000 + 0xDFFF) which is 0x1E000 to 0x10ffff.
470 // This is in range.
471 //
472 // From a Unicode principles perspective this can also be verified as we have checked that `non_ascii` is a high surrogate
473 // (0xD800..=0xDBFF), and that `second` is a low surrogate (`0xDC00..=0xDFFF`), and we are applying reverse of the UTC16 transformation
474 // algorithm <https://en.wikipedia.org/wiki/UTF-16#Code_points_from_U+010000_to_U+10FFFF>, by applying the high surrogate - 0xD800 to the
475 // high ten bits, and the low surrogate - 0xDc00 to the low ten bits, and then adding 0x10000
476 ::core::char::from_u32_unchecked(
477 (u32::from(non_ascii) << 10) + second
478 - (((0xD800u32 << 10) - 0x1_0000u32) + 0xDC00u32),
479 )
480 };
481 return (
482 EncoderResult::Unmappable(astral),
483 converted + 2, // +2 `for non_ascii` and `second`
484 converted,
485 );
486 }
487 if high_bits == 0xDC00u16 {
488 // Unpaired low surrogate
489 return (
490 EncoderResult::Unmappable('\u{FFFD}'),
491 converted + 1, // +1 `for non_ascii`
492 converted,
493 );
494 }
495 return (
496 EncoderResult::unmappable_from_bmp(non_ascii),
497 converted + 1, // +1 `for non_ascii`
498 converted,
499 );
500 // Safety: This branch diverges, so no need to uphold invariants on `converted`
501 }
502 }
503 // Next, handle ASCII punctuation and non-ASCII without
504 // going back to ASCII acceleration. Non-ASCII scripts
505 // use ASCII punctuation, so this avoid going to
506 // acceleration just for punctuation/space and then
507 // failing. This is a significant boost to non-ASCII
508 // scripts.
509 // TODO: Split out Latin converters without this part
510 // this stuff makes Latin script-conversion slower.
511 if converted == length {
512 return (pending, length, length);
513 }
514 // Safety: we're back to `converted < length` due to the == above and can perform
515 // the unchecked read
516 let mut unit = unsafe { *(src.get_unchecked(converted)) };
517 'innermost: loop {
518 // Safety: This loop always begins with `converted < length`, see
519 // the invariant outside and the comment on the continue below
520 if unit > 127 {
521 non_ascii = unit;
522 continue 'middle;
523 }
524 // Testing on Haswell says that we should write the
525 // byte unconditionally instead of trying to unread it
526 // to make it part of the next SIMD stride.
527 unsafe {
528 // Safety: Can rely on converted < length
529 *(dst.get_unchecked_mut(converted)) = unit as u8;
530 }
531 converted += 1;
532 // `converted <= length` here
533 if unit < 60 {
534 // We've got punctuation
535 if converted == length {
536 return (pending, length, length);
537 }
538 // Safety: `converted < length` due to the == above. The read is safe.
539 unit = unsafe { *(src.get_unchecked(converted)) };
540 // Safety: This only happens if `converted < length`, maintaining it
541 continue 'innermost;
542 }
543 // We've got markup or ASCII text
544 continue 'outermost;
545 // Safety: All other routes to here diverge so the continue is the only
546 // way to run the innermost loop.
547 }
548 }
549 }
550 }
551 }
552 }
553}
554
555// Any copyright to the test code below this comment is dedicated to the
556// Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
557
558#[cfg(all(test, feature = "alloc"))]
559mod tests {
560 use super::super::testing::*;
561 use super::super::*;
562
563 #[test]
564 fn test_windows_1255_ca() {
565 decode(WINDOWS_1255, b"\xCA", "\u{05BA}");
566 encode(WINDOWS_1255, "\u{05BA}", b"\xCA");
567 }
568
569 #[test]
570 fn test_ascii_punctuation() {
571 let bytes = b"\xC1\xF5\xF4\xFC \xE5\xDF\xED\xE1\xE9 \xDD\xED\xE1 \xF4\xE5\xF3\xF4. \xC1\xF5\xF4\xFC \xE5\xDF\xED\xE1\xE9 \xDD\xED\xE1 \xF4\xE5\xF3\xF4.";
572 let characters = "\u{0391}\u{03C5}\u{03C4}\u{03CC} \
573 \u{03B5}\u{03AF}\u{03BD}\u{03B1}\u{03B9} \u{03AD}\u{03BD}\u{03B1} \
574 \u{03C4}\u{03B5}\u{03C3}\u{03C4}. \u{0391}\u{03C5}\u{03C4}\u{03CC} \
575 \u{03B5}\u{03AF}\u{03BD}\u{03B1}\u{03B9} \u{03AD}\u{03BD}\u{03B1} \
576 \u{03C4}\u{03B5}\u{03C3}\u{03C4}.";
577 decode(WINDOWS_1253, bytes, characters);
578 encode(WINDOWS_1253, characters, bytes);
579 }
580
581 #[test]
582 fn test_decode_malformed() {
583 decode(
584 WINDOWS_1253,
585 b"\xC1\xF5\xD2\xF4\xFC",
586 "\u{0391}\u{03C5}\u{FFFD}\u{03C4}\u{03CC}",
587 );
588 }
589
590 #[test]
591 fn test_encode_unmappables() {
592 encode(
593 WINDOWS_1253,
594 "\u{0391}\u{03C5}\u{2603}\u{03C4}\u{03CC}",
595 b"\xC1\xF5☃\xF4\xFC",
596 );
597 encode(
598 WINDOWS_1253,
599 "\u{0391}\u{03C5}\u{1F4A9}\u{03C4}\u{03CC}",
600 b"\xC1\xF5💩\xF4\xFC",
601 );
602 }
603
604 #[test]
605 fn test_encode_unpaired_surrogates() {
606 encode_from_utf16(
607 WINDOWS_1253,
608 &[0x0391u16, 0x03C5u16, 0xDCA9u16, 0x03C4u16, 0x03CCu16],
609 b"\xC1\xF5�\xF4\xFC",
610 );
611 encode_from_utf16(
612 WINDOWS_1253,
613 &[0x0391u16, 0x03C5u16, 0xD83Du16, 0x03C4u16, 0x03CCu16],
614 b"\xC1\xF5�\xF4\xFC",
615 );
616 encode_from_utf16(
617 WINDOWS_1253,
618 &[0x0391u16, 0x03C5u16, 0x03C4u16, 0x03CCu16, 0xD83Du16],
619 b"\xC1\xF5\xF4\xFC�",
620 );
621 }
622
623 pub const HIGH_BYTES: &'static [u8; 128] = &[
624 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E,
625 0x8F, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D,
626 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC,
627 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB,
628 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA,
629 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9,
630 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8,
631 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
632 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
633 ];
634
635 fn decode_single_byte(encoding: &'static Encoding, data: &'static [u16; 128]) {
636 let mut with_replacement = [0u16; 128];
637 let mut it = data.iter().enumerate();
638 loop {
639 match it.next() {
640 Some((i, code_point)) => {
641 if *code_point == 0 {
642 with_replacement[i] = 0xFFFD;
643 } else {
644 with_replacement[i] = *code_point;
645 }
646 }
647 None => {
648 break;
649 }
650 }
651 }
652
653 decode_to_utf16(encoding, HIGH_BYTES, &with_replacement[..]);
654 }
655
656 fn encode_single_byte(encoding: &'static Encoding, data: &'static [u16; 128]) {
657 let mut with_zeros = [0u8; 128];
658 let mut it = data.iter().enumerate();
659 loop {
660 match it.next() {
661 Some((i, code_point)) => {
662 if *code_point == 0 {
663 with_zeros[i] = 0;
664 } else {
665 with_zeros[i] = HIGH_BYTES[i];
666 }
667 }
668 None => {
669 break;
670 }
671 }
672 }
673
674 encode_from_utf16(encoding, data, &with_zeros[..]);
675 }
676
677 #[test]
678 fn test_single_byte_from_two_low_surrogates() {
679 let expectation = b"��";
680 let mut output = [0u8; 40];
681 let mut encoder = WINDOWS_1253.new_encoder();
682 let (result, read, written, had_errors) =
683 encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
684 assert_eq!(result, CoderResult::InputEmpty);
685 assert_eq!(read, 2);
686 assert_eq!(written, expectation.len());
687 assert!(had_errors);
688 assert_eq!(&output[..written], expectation);
689 }
690
691 // These tests are so self-referential that they are pretty useless.
692
693 // BEGIN GENERATED CODE. PLEASE DO NOT EDIT.
694 // Instead, please regenerate using generate-encoding-data.py
695
696 #[test]
697 fn test_single_byte_decode() {
698 decode_single_byte(IBM866, &data::SINGLE_BYTE_DATA.ibm866);
699 decode_single_byte(ISO_8859_10, &data::SINGLE_BYTE_DATA.iso_8859_10);
700 if cfg!(miri) {
701 // Miri is too slow
702 return;
703 }
704 decode_single_byte(ISO_8859_13, &data::SINGLE_BYTE_DATA.iso_8859_13);
705 decode_single_byte(ISO_8859_14, &data::SINGLE_BYTE_DATA.iso_8859_14);
706 decode_single_byte(ISO_8859_15, &data::SINGLE_BYTE_DATA.iso_8859_15);
707 decode_single_byte(ISO_8859_16, &data::SINGLE_BYTE_DATA.iso_8859_16);
708 decode_single_byte(ISO_8859_2, &data::SINGLE_BYTE_DATA.iso_8859_2);
709 decode_single_byte(ISO_8859_3, &data::SINGLE_BYTE_DATA.iso_8859_3);
710 decode_single_byte(ISO_8859_4, &data::SINGLE_BYTE_DATA.iso_8859_4);
711 decode_single_byte(ISO_8859_5, &data::SINGLE_BYTE_DATA.iso_8859_5);
712 decode_single_byte(ISO_8859_6, &data::SINGLE_BYTE_DATA.iso_8859_6);
713 decode_single_byte(ISO_8859_7, &data::SINGLE_BYTE_DATA.iso_8859_7);
714 decode_single_byte(ISO_8859_8, &data::SINGLE_BYTE_DATA.iso_8859_8);
715 decode_single_byte(KOI8_R, &data::SINGLE_BYTE_DATA.koi8_r);
716 decode_single_byte(KOI8_U, &data::SINGLE_BYTE_DATA.koi8_u);
717 decode_single_byte(MACINTOSH, &data::SINGLE_BYTE_DATA.macintosh);
718 decode_single_byte(WINDOWS_1250, &data::SINGLE_BYTE_DATA.windows_1250);
719 decode_single_byte(WINDOWS_1251, &data::SINGLE_BYTE_DATA.windows_1251);
720 decode_single_byte(WINDOWS_1252, &data::SINGLE_BYTE_DATA.windows_1252);
721 decode_single_byte(WINDOWS_1253, &data::SINGLE_BYTE_DATA.windows_1253);
722 decode_single_byte(WINDOWS_1254, &data::SINGLE_BYTE_DATA.windows_1254);
723 decode_single_byte(WINDOWS_1255, &data::SINGLE_BYTE_DATA.windows_1255);
724 decode_single_byte(WINDOWS_1256, &data::SINGLE_BYTE_DATA.windows_1256);
725 decode_single_byte(WINDOWS_1257, &data::SINGLE_BYTE_DATA.windows_1257);
726 decode_single_byte(WINDOWS_1258, &data::SINGLE_BYTE_DATA.windows_1258);
727 decode_single_byte(WINDOWS_874, &data::SINGLE_BYTE_DATA.windows_874);
728 decode_single_byte(X_MAC_CYRILLIC, &data::SINGLE_BYTE_DATA.x_mac_cyrillic);
729 }
730
731 #[test]
732 fn test_single_byte_encode() {
733 encode_single_byte(IBM866, &data::SINGLE_BYTE_DATA.ibm866);
734 encode_single_byte(ISO_8859_10, &data::SINGLE_BYTE_DATA.iso_8859_10);
735 if cfg!(miri) {
736 // Miri is too slow
737 return;
738 }
739 encode_single_byte(ISO_8859_13, &data::SINGLE_BYTE_DATA.iso_8859_13);
740 encode_single_byte(ISO_8859_14, &data::SINGLE_BYTE_DATA.iso_8859_14);
741 encode_single_byte(ISO_8859_15, &data::SINGLE_BYTE_DATA.iso_8859_15);
742 encode_single_byte(ISO_8859_16, &data::SINGLE_BYTE_DATA.iso_8859_16);
743 encode_single_byte(ISO_8859_2, &data::SINGLE_BYTE_DATA.iso_8859_2);
744 encode_single_byte(ISO_8859_3, &data::SINGLE_BYTE_DATA.iso_8859_3);
745 encode_single_byte(ISO_8859_4, &data::SINGLE_BYTE_DATA.iso_8859_4);
746 encode_single_byte(ISO_8859_5, &data::SINGLE_BYTE_DATA.iso_8859_5);
747 encode_single_byte(ISO_8859_6, &data::SINGLE_BYTE_DATA.iso_8859_6);
748 encode_single_byte(ISO_8859_7, &data::SINGLE_BYTE_DATA.iso_8859_7);
749 encode_single_byte(ISO_8859_8, &data::SINGLE_BYTE_DATA.iso_8859_8);
750 encode_single_byte(KOI8_R, &data::SINGLE_BYTE_DATA.koi8_r);
751 encode_single_byte(KOI8_U, &data::SINGLE_BYTE_DATA.koi8_u);
752 encode_single_byte(MACINTOSH, &data::SINGLE_BYTE_DATA.macintosh);
753 encode_single_byte(WINDOWS_1250, &data::SINGLE_BYTE_DATA.windows_1250);
754 encode_single_byte(WINDOWS_1251, &data::SINGLE_BYTE_DATA.windows_1251);
755 encode_single_byte(WINDOWS_1252, &data::SINGLE_BYTE_DATA.windows_1252);
756 encode_single_byte(WINDOWS_1253, &data::SINGLE_BYTE_DATA.windows_1253);
757 encode_single_byte(WINDOWS_1254, &data::SINGLE_BYTE_DATA.windows_1254);
758 encode_single_byte(WINDOWS_1255, &data::SINGLE_BYTE_DATA.windows_1255);
759 encode_single_byte(WINDOWS_1256, &data::SINGLE_BYTE_DATA.windows_1256);
760 encode_single_byte(WINDOWS_1257, &data::SINGLE_BYTE_DATA.windows_1257);
761 encode_single_byte(WINDOWS_1258, &data::SINGLE_BYTE_DATA.windows_1258);
762 encode_single_byte(WINDOWS_874, &data::SINGLE_BYTE_DATA.windows_874);
763 encode_single_byte(X_MAC_CYRILLIC, &data::SINGLE_BYTE_DATA.x_mac_cyrillic);
764 }
765 // END GENERATED CODE
766}