encoding_rs/
euc_kr.rs

1// Copyright Mozilla Foundation. See the COPYRIGHT
2// file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use super::*;
11use crate::data::*;
12use crate::handles::*;
13use crate::variant::*;
14// Rust 1.14.0 requires the following despite the asterisk above.
15use super::in_inclusive_range16;
16use super::in_range16;
17
18pub struct EucKrDecoder {
19    lead: Option<u8>,
20}
21
22impl EucKrDecoder {
23    pub fn new() -> VariantDecoder {
24        VariantDecoder::EucKr(EucKrDecoder { lead: None })
25    }
26
27    pub fn in_neutral_state(&self) -> bool {
28        self.lead.is_none()
29    }
30
31    fn plus_one_if_lead(&self, byte_length: usize) -> Option<usize> {
32        byte_length.checked_add(match self.lead {
33            None => 0,
34            Some(_) => 1,
35        })
36    }
37
38    pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
39        self.plus_one_if_lead(byte_length)
40    }
41
42    pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> {
43        // worst case: 2 to 3
44        let len = self.plus_one_if_lead(byte_length);
45        checked_add(2, checked_add_opt(len, checked_div(checked_add(1, len), 2)))
46    }
47
48    pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> {
49        checked_mul(3, self.plus_one_if_lead(byte_length))
50    }
51
52    ascii_compatible_two_byte_decoder_functions!(
53        lead = {
54            // If lead is between 0x81 and 0xFE, inclusive,
55            // subtract offset 0x81.
56            let non_ascii_minus_offset =
57                non_ascii.wrapping_sub(0x81);
58            if non_ascii_minus_offset > (0xFE - 0x81) {
59                return (DecoderResult::Malformed(1, 0),
60                        source.consumed(),
61                        handle.written());
62            }
63            non_ascii_minus_offset
64        },
65        trail = {
66            if lead_minus_offset >= 0x20 {
67                // Not the extension range above KS X 1001
68                let trail_minus_offset =
69                    byte.wrapping_sub(0xA1);
70                if trail_minus_offset <= (0xFE - 0xA1) {
71                    // KS X 1001
72                    let ksx_pointer = mul_94(lead_minus_offset - 0x20) + trail_minus_offset as usize;
73                    let hangul_pointer = ksx_pointer.wrapping_sub((0x2F - 0x20) * 94);
74                    if hangul_pointer < KSX1001_HANGUL.len() {
75                        let upper_bmp = KSX1001_HANGUL[hangul_pointer];
76                        handle.write_upper_bmp(upper_bmp)
77                    } else if ksx_pointer < KSX1001_SYMBOLS.len() {
78                        let bmp = KSX1001_SYMBOLS[ksx_pointer];
79                        handle.write_bmp_excl_ascii(bmp)
80                    } else {
81                        let hanja_pointer = ksx_pointer.wrapping_sub((0x49 - 0x20) * 94);
82                        if hanja_pointer < KSX1001_HANJA.len() {
83                            let upper_bmp = KSX1001_HANJA[hanja_pointer];
84                            handle.write_upper_bmp(upper_bmp)
85                        } else if (lead_minus_offset == 0x27) && ((trail_minus_offset as usize) < KSX1001_UPPERCASE.len()) {
86                            let mid_bmp = KSX1001_UPPERCASE[trail_minus_offset as usize];
87                            if mid_bmp == 0 {
88                                return (DecoderResult::Malformed(2, 0),
89                                        unread_handle_trail.consumed(),
90                                        handle.written());
91                            }
92                            handle.write_mid_bmp(mid_bmp)
93                        } else if (lead_minus_offset == 0x28) && ((trail_minus_offset as usize) < KSX1001_LOWERCASE.len()) {
94                            let mid_bmp = KSX1001_LOWERCASE[trail_minus_offset as usize];
95                            handle.write_mid_bmp(mid_bmp)
96                        } else if (lead_minus_offset == 0x25) && ((trail_minus_offset as usize) < KSX1001_BOX.len()) {
97                            let upper_bmp = KSX1001_BOX[trail_minus_offset as usize];
98                            handle.write_upper_bmp(upper_bmp)
99                        } else {
100                            let other_pointer = ksx_pointer.wrapping_sub(2 * 94);
101                            if other_pointer < 0x039F {
102                                let bmp = ksx1001_other_decode(other_pointer as u16);
103                                // ASCII range means unassigned
104                                if bmp < 0x80 {
105                                    return (DecoderResult::Malformed(2, 0),
106                                            unread_handle_trail.consumed(),
107                                            handle.written());
108                                }
109                                handle.write_bmp_excl_ascii(bmp)
110                            } else {
111                                return (DecoderResult::Malformed(2, 0),
112                                        unread_handle_trail.consumed(),
113                                        handle.written());
114                            }
115                        }
116                    }
117                } else {
118                    // Extension range to the left of
119                    // KS X 1001
120                    let left_lead = lead_minus_offset - 0x20;
121                    let left_trail = if byte.wrapping_sub(0x40 + 0x41) < (0x60 - 0x40) {
122                        byte - (12 + 0x41)
123                    } else if byte.wrapping_sub(0x20 + 0x41) < (0x3A - 0x20) {
124                        byte - (6 + 0x41)
125                    } else if byte.wrapping_sub(0x41) < 0x1A {
126                        byte - 0x41
127                    } else {
128                        if byte < 0x80 {
129                            return (DecoderResult::Malformed(1, 0),
130                                    unread_handle_trail.unread(),
131                                    handle.written());
132                        }
133                        return (DecoderResult::Malformed(2, 0),
134                                unread_handle_trail.consumed(),
135                                handle.written());
136                    };
137                    let left_pointer = ((left_lead as usize) * (190 - 94 - 12)) + left_trail as usize;
138                    if left_pointer < (0x45 - 0x20) * (190 - 94 - 12) + 0x12 {
139                        let upper_bmp = cp949_left_hangul_decode(left_pointer as u16);
140                        handle.write_upper_bmp(upper_bmp)
141                    } else {
142                        if byte < 0x80 {
143                            return (DecoderResult::Malformed(1, 0),
144                                    unread_handle_trail.unread(),
145                                    handle.written());
146                        }
147                        return (DecoderResult::Malformed(2, 0),
148                                unread_handle_trail.consumed(),
149                                handle.written());
150                    }
151                }
152            } else {
153                // Extension range above KS X 1001
154                let top_trail = if byte.wrapping_sub(0x40 + 0x41) < (0xBE - 0x40) {
155                    byte - (12 + 0x41)
156                } else if byte.wrapping_sub(0x20 + 0x41) < (0x3A - 0x20) {
157                    byte - (6 + 0x41)
158                } else if byte.wrapping_sub(0x41) < 0x1A {
159                    byte - 0x41
160                } else {
161                    if byte < 0x80 {
162                        return (DecoderResult::Malformed(1, 0),
163                                unread_handle_trail.unread(),
164                                handle.written());
165                    }
166                    return (DecoderResult::Malformed(2, 0),
167                            unread_handle_trail.consumed(),
168                            handle.written());
169                };
170                let top_pointer = ((lead_minus_offset as usize) * (190 - 12)) + top_trail as usize;
171                let upper_bmp = cp949_top_hangul_decode(top_pointer as u16);
172                handle.write_upper_bmp(upper_bmp)
173            }
174        },
175        self = self,
176        non_ascii = non_ascii,
177        byte = byte,
178        lead_minus_offset = lead_minus_offset,
179        unread_handle_trail = unread_handle_trail,
180        source = source,
181        handle = handle,
182        outermost = 'outermost,
183        copy_ascii = copy_ascii_from_check_space_bmp,
184        destination_check = check_space_bmp,
185        ascii_punctuation = true);
186}
187
188fn ksx1001_encode_misc(bmp: u16) -> Option<(usize, usize)> {
189    if in_inclusive_range16(bmp, 0x3000, 0x3015)
190        && let Some(pos) = position(&KSX1001_SYMBOLS[..(0xAB - 0x60)], bmp)
191    {
192        return Some((0xA1, pos + 0xA1));
193    }
194    if let Some(other_pointer) = ksx1001_other_encode(bmp) {
195        let other_lead = ((other_pointer as usize) / 94) + (0x81 + 0x22);
196        let other_trail = ((other_pointer as usize) % 94) + 0xA1;
197        return Some((other_lead, other_trail));
198    }
199    if in_range16(bmp, 0x00AA, 0x0168) {
200        // Latin
201        if let Some(pos) = position(&KSX1001_LOWERCASE[..], bmp) {
202            return Some((0x81 + 0x28, 0xA1 + pos));
203        }
204        if let Some(pos) = position(&KSX1001_UPPERCASE[..], bmp) {
205            return Some((0x81 + 0x27, 0xA1 + pos));
206        }
207    } else if in_range16(bmp, 0x2500, 0x254C)
208        && let Some(pos) = position(&KSX1001_BOX[..], bmp)
209    {
210        return Some((0x81 + 0x25, 0xA1 + pos));
211    }
212    if (in_inclusive_range16(bmp, 0x2015, 0x266D)
213        || in_inclusive_range16(bmp, 0x321C, 0x33D8)
214        || in_inclusive_range16(bmp, 0xFF3C, 0xFFE5)
215        || in_inclusive_range16(bmp, 0x00A1, 0x00F7)
216        || in_inclusive_range16(bmp, 0x02C7, 0x02DD))
217        && let Some(pos) = position(&KSX1001_SYMBOLS[3..], bmp)
218    {
219        if pos < (94 - 3) {
220            return Some((0xA1, pos + 0xA1 + 3));
221        }
222        return Some((0xA2, pos - (94 - 3) + 0xA1));
223    }
224    None
225}
226
227#[cfg(not(feature = "fast-hangul-encode"))]
228#[inline(always)]
229fn ksx1001_encode_hangul(bmp: u16, _: u16) -> (u8, u8) {
230    match KSX1001_HANGUL.binary_search(&bmp) {
231        Ok(ksx_hangul_pointer) => {
232            let ksx_hangul_lead = (ksx_hangul_pointer / 94) + (0x81 + 0x2F);
233            let ksx_hangul_trail = (ksx_hangul_pointer % 94) + 0xA1;
234            (ksx_hangul_lead as u8, ksx_hangul_trail as u8)
235        }
236        Err(_) => {
237            let (lead, cp949_trail) = if bmp < 0xC8A5 {
238                // Above KS X 1001
239                let top_pointer = cp949_top_hangul_encode(bmp) as usize;
240                let top_lead = (top_pointer / (190 - 12)) + 0x81;
241                let top_trail = top_pointer % (190 - 12);
242                (top_lead as u8, top_trail as u8)
243            } else {
244                // To the left of KS X 1001
245                let left_pointer = cp949_left_hangul_encode(bmp) as usize;
246                let left_lead = (left_pointer / (190 - 94 - 12)) + (0x81 + 0x20);
247                let left_trail = left_pointer % (190 - 94 - 12);
248                (left_lead as u8, left_trail as u8)
249            };
250            let offset = if cp949_trail >= (0x40 - 12) {
251                0x41 + 12
252            } else if cp949_trail >= (0x20 - 6) {
253                0x41 + 6
254            } else {
255                0x41
256            };
257            (lead, (cp949_trail + offset))
258        }
259    }
260}
261
262#[cfg(feature = "fast-hangul-encode")]
263#[inline(always)]
264fn ksx1001_encode_hangul(_: u16, bmp_minus_hangul_start: u16) -> (u8, u8) {
265    cp949_hangul_encode(bmp_minus_hangul_start)
266}
267
268#[cfg(not(feature = "fast-hanja-encode"))]
269#[inline(always)]
270fn ksx1001_encode_hanja(bmp: u16) -> Option<(u8, u8)> {
271    if let Some(hanja_pointer) = position(&KSX1001_HANJA[..], bmp) {
272        let hanja_lead = (hanja_pointer / 94) + (0x81 + 0x49);
273        let hanja_trail = (hanja_pointer % 94) + 0xA1;
274        Some((hanja_lead as u8, hanja_trail as u8))
275    } else {
276        None
277    }
278}
279
280#[cfg(feature = "fast-hanja-encode")]
281#[inline(always)]
282fn ksx1001_encode_hanja(bmp: u16) -> Option<(u8, u8)> {
283    if bmp < 0xF900 {
284        ksx1001_unified_hangul_encode(bmp)
285    } else {
286        Some(ksx1001_compatibility_hangul_encode(bmp))
287    }
288}
289
290pub struct EucKrEncoder;
291
292impl EucKrEncoder {
293    pub fn new(encoding: &'static Encoding) -> Encoder {
294        Encoder::new(encoding, VariantEncoder::EucKr(EucKrEncoder))
295    }
296
297    pub fn max_buffer_length_from_utf16_without_replacement(
298        &self,
299        u16_length: usize,
300    ) -> Option<usize> {
301        u16_length.checked_mul(2)
302    }
303
304    pub fn max_buffer_length_from_utf8_without_replacement(
305        &self,
306        byte_length: usize,
307    ) -> Option<usize> {
308        byte_length.checked_add(1)
309    }
310
311    ascii_compatible_bmp_encoder_functions!(
312        {
313            let bmp_minus_hangul_start = bmp.wrapping_sub(0xAC00);
314            let (lead, trail) = if bmp_minus_hangul_start < (0xD7A4 - 0xAC00) {
315                // Hangul
316                ksx1001_encode_hangul(bmp, bmp_minus_hangul_start)
317            } else if in_range16(bmp, 0x33DE, 0xFF01) {
318                // Vast range that includes no other
319                // mappables except Hangul (already
320                // processed) and Hanja.
321                // Narrow the range further to Unified and
322                // Compatibility ranges of Hanja.
323                if in_range16(bmp, 0x4E00, 0x9F9D) || in_range16(bmp, 0xF900, 0xFA0C) {
324                    if let Some((hanja_lead, hanja_trail)) = ksx1001_encode_hanja(bmp) {
325                        (hanja_lead, hanja_trail)
326                    } else {
327                        return (
328                            EncoderResult::unmappable_from_bmp(bmp),
329                            source.consumed(),
330                            handle.written(),
331                        );
332                    }
333                } else {
334                    return (
335                        EncoderResult::unmappable_from_bmp(bmp),
336                        source.consumed(),
337                        handle.written(),
338                    );
339                }
340            } else if let Some((lead, trail)) = ksx1001_encode_misc(bmp) {
341                (lead as u8, trail as u8)
342            } else {
343                return (
344                    EncoderResult::unmappable_from_bmp(bmp),
345                    source.consumed(),
346                    handle.written(),
347                );
348            };
349            handle.write_two(lead, trail)
350        },
351        bmp,
352        self,
353        source,
354        handle,
355        copy_ascii_to_check_space_two,
356        check_space_two,
357        true
358    );
359}
360
361// Any copyright to the test code below this comment is dedicated to the
362// Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
363
364#[cfg(all(test, feature = "alloc"))]
365mod tests {
366    use super::super::testing::*;
367    use super::super::*;
368
369    fn decode_euc_kr(bytes: &[u8], expect: &str) {
370        decode(EUC_KR, bytes, expect);
371    }
372
373    fn encode_euc_kr(string: &str, expect: &[u8]) {
374        encode(EUC_KR, string, expect);
375    }
376
377    #[test]
378    fn test_euc_kr_decode() {
379        // Empty
380        decode_euc_kr(b"", &"");
381
382        // ASCII
383        decode_euc_kr(b"\x61\x62", "\u{0061}\u{0062}");
384
385        decode_euc_kr(b"\x81\x41", "\u{AC02}");
386        decode_euc_kr(b"\x81\x5B", "\u{FFFD}\x5B");
387        decode_euc_kr(b"\xFD\xFE", "\u{8A70}");
388        decode_euc_kr(b"\xFE\x41", "\u{FFFD}\x41");
389        decode_euc_kr(b"\xFF\x41", "\u{FFFD}\x41");
390        decode_euc_kr(b"\x80\x41", "\u{FFFD}\x41");
391        decode_euc_kr(b"\xA1\xFF", "\u{FFFD}");
392        decode_euc_kr(b"\x81\xFF", "\u{FFFD}");
393    }
394
395    #[test]
396    fn test_euc_kr_encode() {
397        // Empty
398        encode_euc_kr("", b"");
399
400        // ASCII
401        encode_euc_kr("\u{0061}\u{0062}", b"\x61\x62");
402
403        encode_euc_kr("\u{AC02}", b"\x81\x41");
404        encode_euc_kr("\u{8A70}", b"\xFD\xFE");
405    }
406
407    #[test]
408    fn test_euc_kr_encode_from_two_low_surrogates() {
409        let expectation = b"&#65533;&#65533;";
410        let mut output = [0u8; 40];
411        let mut encoder = EUC_KR.new_encoder();
412        let (result, read, written, had_errors) =
413            encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
414        assert_eq!(result, CoderResult::InputEmpty);
415        assert_eq!(read, 2);
416        assert_eq!(written, expectation.len());
417        assert!(had_errors);
418        assert_eq!(&output[..written], expectation);
419    }
420}
encoding_rs/euc_kr.rs

encoding_rs/
euc_kr.rs
