tendril/
utf8_decode.rs

1// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
2// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
3// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
4// option. This file may not be copied, modified, or distributed
5// except according to those terms.
6
7use crate::fmt;
8use crate::{Atomicity, Tendril};
9
10use std::cmp;
11use std::str;
12
13/// The replacement character, U+FFFD. In lossy decoding, insert it for every decoding error.
14pub(crate) const REPLACEMENT_CHARACTER: &str = "\u{FFFD}";
15
16#[derive(Debug, Copy, Clone)]
17pub(crate) enum DecodeError<'a> {
18    /// In lossy decoding insert `valid_prefix`, then `"\u{FFFD}"`,
19    /// then call `decode()` again with `remaining_input`.
20    Invalid {
21        valid_prefix: &'a str,
22        invalid_sequence: &'a [u8],
23    },
24
25    /// Call the `incomplete_suffix.try_to_complete_codepoint` method with more input when available.
26    /// If no more input is available, this is an invalid byte sequence.
27    Incomplete {
28        valid_prefix: &'a str,
29        incomplete_suffix: IncompleteUtf8,
30    },
31}
32
33#[derive(Debug, Copy, Clone)]
34pub struct IncompleteUtf8 {
35    pub buffer: [u8; 4],
36    pub buffer_len: u8,
37}
38
39pub(crate) fn decode_utf8(input: &[u8]) -> Result<&str, DecodeError<'_>> {
40    let error = match str::from_utf8(input) {
41        Ok(valid) => return Ok(valid),
42        Err(error) => error,
43    };
44
45    // FIXME: separate function from here to guide inlining?
46    let (valid, after_valid) = input.split_at(error.valid_up_to());
47    let valid = unsafe { str::from_utf8_unchecked(valid) };
48
49    match error.error_len() {
50        Some(invalid_sequence_length) => {
51            let invalid = &after_valid[..invalid_sequence_length];
52            Err(DecodeError::Invalid {
53                valid_prefix: valid,
54                invalid_sequence: invalid,
55            })
56        },
57        None => Err(DecodeError::Incomplete {
58            valid_prefix: valid,
59            incomplete_suffix: IncompleteUtf8::new(after_valid),
60        }),
61    }
62}
63
64enum Utf8CompletionResult {
65    NotEnoughInput,
66    MalformedUtf8Buffer,
67    Valid,
68}
69
70impl IncompleteUtf8 {
71    fn new(bytes: &[u8]) -> Self {
72        let mut buffer = [0, 0, 0, 0];
73        let len = bytes.len();
74        buffer[..len].copy_from_slice(bytes);
75
76        Self {
77            buffer,
78            buffer_len: len as u8,
79        }
80    }
81
82    fn take_buffer(&mut self) -> &[u8] {
83        let len = self.buffer_len as usize;
84        self.buffer_len = 0;
85        &self.buffer[..len]
86    }
87
88    /// Consumes bytes from the input and attempts to form a valid utf8 codepoint.
89    ///
90    /// Returns how many bytes were consumed and whether a valid code point was found.
91    fn try_complete_offsets(&mut self, input: &[u8]) -> (usize, Utf8CompletionResult) {
92        let initial_buffer_len = self.buffer_len as usize;
93        let copied_from_input;
94        {
95            let unwritten = &mut self.buffer[initial_buffer_len..];
96            copied_from_input = cmp::min(unwritten.len(), input.len());
97            unwritten[..copied_from_input].copy_from_slice(&input[..copied_from_input]);
98        }
99        let spliced = &self.buffer[..initial_buffer_len + copied_from_input];
100        match str::from_utf8(spliced) {
101            Ok(_) => {
102                self.buffer_len = spliced.len() as u8;
103                (copied_from_input, Utf8CompletionResult::Valid)
104            },
105            Err(error) => {
106                let valid_up_to = error.valid_up_to();
107                if valid_up_to > 0 {
108                    let consumed = valid_up_to.checked_sub(initial_buffer_len).unwrap();
109                    self.buffer_len = valid_up_to as u8;
110                    (consumed, Utf8CompletionResult::Valid)
111                } else {
112                    match error.error_len() {
113                        Some(invalid_sequence_length) => {
114                            let consumed = invalid_sequence_length
115                                .checked_sub(initial_buffer_len)
116                                .unwrap();
117                            self.buffer_len = invalid_sequence_length as u8;
118                            (consumed, Utf8CompletionResult::MalformedUtf8Buffer)
119                        },
120                        None => {
121                            self.buffer_len = spliced.len() as u8;
122                            (copied_from_input, Utf8CompletionResult::NotEnoughInput)
123                        },
124                    }
125                }
126            },
127        }
128    }
129
130    /// Attempts to complete the codepoint given the bytes from `input`.
131    ///
132    /// Returns `None` if more input is required to complete the codepoint. In this case, no
133    /// input is consumed.
134    ///
135    /// Otherwise, returns either the decoded `&str` or malformed `&[u8]` and the remaining input.
136    #[allow(clippy::type_complexity)]
137    pub fn try_to_complete_codepoint<'input>(
138        &mut self,
139        input: &'input [u8],
140    ) -> Option<(Result<&str, &[u8]>, &'input [u8])> {
141        let (consumed, completion_result) = self.try_complete_offsets(input);
142        let result = match completion_result {
143            Utf8CompletionResult::NotEnoughInput => return None,
144            Utf8CompletionResult::MalformedUtf8Buffer => Err(self.take_buffer()),
145            Utf8CompletionResult::Valid => {
146                Ok(unsafe { str::from_utf8_unchecked(self.take_buffer()) })
147            },
148        };
149        let remaining_input = &input[consumed..];
150
151        Some((result, remaining_input))
152    }
153
154    pub fn try_complete<A, F>(
155        &mut self,
156        mut input: Tendril<fmt::Bytes, A>,
157        mut push_utf8: F,
158    ) -> Result<Tendril<fmt::Bytes, A>, ()>
159    where
160        A: Atomicity,
161        F: FnMut(Tendril<fmt::UTF8, A>),
162    {
163        let Some((result, remaining_input)) = self.try_to_complete_codepoint(&input) else {
164            // Not enough input to complete codepoint
165            return Err(());
166        };
167
168        push_utf8(Tendril::from_slice(result.unwrap_or(REPLACEMENT_CHARACTER)));
169        let resume_at = input.len() - remaining_input.len();
170        input.pop_front(resume_at as u32);
171        Ok(input)
172    }
173}
174
175impl<A> Tendril<fmt::Bytes, A>
176where
177    A: Atomicity,
178{
179    pub fn decode_utf8_lossy<F>(mut self, mut push_utf8: F) -> Option<IncompleteUtf8>
180    where
181        F: FnMut(Tendril<fmt::UTF8, A>),
182    {
183        loop {
184            if self.is_empty() {
185                return None;
186            }
187            let unborrowed_result = match decode_utf8(&self) {
188                Ok(string) => {
189                    debug_assert!(string.as_ptr() == self.as_ptr());
190                    debug_assert!(string.len() == self.len());
191                    Ok(())
192                },
193                Err(DecodeError::Invalid {
194                    valid_prefix,
195                    invalid_sequence,
196                    ..
197                }) => {
198                    debug_assert!(valid_prefix.as_ptr() == self.as_ptr());
199                    debug_assert!(valid_prefix.len() <= self.len());
200                    Err((
201                        valid_prefix.len(),
202                        Err(valid_prefix.len() + invalid_sequence.len()),
203                    ))
204                },
205                Err(DecodeError::Incomplete {
206                    valid_prefix,
207                    incomplete_suffix,
208                }) => {
209                    debug_assert!(valid_prefix.as_ptr() == self.as_ptr());
210                    debug_assert!(valid_prefix.len() <= self.len());
211                    Err((valid_prefix.len(), Ok(incomplete_suffix)))
212                },
213            };
214            match unborrowed_result {
215                Ok(()) => {
216                    unsafe { push_utf8(self.reinterpret_without_validating()) }
217                    return None;
218                },
219                Err((valid_len, and_then)) => {
220                    if valid_len > 0 {
221                        let subtendril = self.subtendril(0, valid_len as u32);
222                        unsafe { push_utf8(subtendril.reinterpret_without_validating()) }
223                    }
224                    match and_then {
225                        Ok(incomplete) => return Some(incomplete),
226                        Err(offset) => {
227                            push_utf8(Tendril::from_slice(REPLACEMENT_CHARACTER));
228                            self.pop_front(offset as u32)
229                        },
230                    }
231                },
232            }
233        }
234    }
235}
tendril/utf8_decode.rs

tendril/
utf8_decode.rs
