mozjs_normalizer_glue/
lib.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5use core::ffi::c_void;
6use smallvec::SmallVec;
7
8// The same as js::intl::INITIAL_CHAR_BUFFER_SIZE
9const INLINE_SIZE: usize = 32;
10
11type Buffer = SmallVec<[u16; INLINE_SIZE]>;
12
13#[derive(Debug, PartialEq, Clone, Copy)]
14#[repr(C)]
15pub enum NormalizationForm {
16    NFC = 0,
17    NFKC = 1,
18    NFD = 2,
19    NFKD = 3,
20}
21
22#[repr(transparent)]
23pub struct JSContext {
24    _inner: c_void,
25}
26
27#[repr(transparent)]
28pub struct JSLinearString {
29    _inner: c_void,
30}
31
32extern "C" {
33    fn js_call_js_normalize_utf16(
34        cx: *mut JSContext,
35        form: NormalizationForm,
36        in_string: *mut JSLinearString,
37        buffer: *mut c_void,
38    ) -> bool;
39
40    fn js_call_js_normalize_latin1(
41        cx: *mut JSContext,
42        form: NormalizationForm,
43        in_string: *mut JSLinearString,
44        buffer: *mut c_void,
45    ) -> bool;
46
47    fn js_new_ucstring_copy_n(
48        cx: *mut JSContext,
49        ptr: *const u16,
50        len: usize,
51    ) -> *mut JSLinearString;
52
53    fn js_new_ucstring_copy_n_dont_deflate(
54        cx: *mut JSContext,
55        ptr: *const u16,
56        len: usize,
57    ) -> *mut JSLinearString;
58}
59
60#[no_mangle]
61pub unsafe extern "C" fn js_normalize(
62    cx: *mut JSContext,
63    form: NormalizationForm,
64    in_string: *mut JSLinearString,
65    latin1: bool,
66) -> *mut JSLinearString {
67    // The purpose of this function is to establish `buffer` as a Rust type
68    // on the stack. We need to call through a layer of C++ to the actual
69    // normalization code so that we can use `AutoCheckCannotGC` as C++ RAII
70    // that goes out of scope before we (potentially) create a new string
71    // later in this function.
72
73    // An earlier attempt used a `JSStringBuilder` as the buffer, but the
74    // cases where a `JSString` ends up reusing the buffer within
75    // `JSStringBuilder` are so slim, that it's better not to go ever FFI
76    // for every write to the buffer and instead do a bulk copy out of
77    // the buffer at the end of this function.
78    let mut buffer: Buffer = SmallVec::new();
79    {
80        let buffer_borrow: &mut Buffer = &mut buffer;
81        let buffer_ptr: *mut Buffer = buffer_borrow as *mut Buffer;
82        let void_ptr: *mut c_void = buffer_ptr as *mut c_void;
83        if !if latin1 {
84            js_call_js_normalize_latin1(cx, form, in_string, void_ptr)
85        } else {
86            js_call_js_normalize_utf16(cx, form, in_string, void_ptr)
87        } {
88            return std::ptr::null_mut();
89        }
90    }
91
92    // If nothing wrote to the buffer (and we haven't already returned), it's a
93    // signal that the input is its own normalization.
94    if buffer.is_empty() {
95        return in_string;
96    }
97
98    // If we are normalizing to NFD and the input isn't its own normalization,
99    // we know the output cannot be Latin1-only.
100    if form == NormalizationForm::NFD {
101        return js_new_ucstring_copy_n_dont_deflate(cx, buffer.as_ptr(), buffer.len());
102    }
103
104    return js_new_ucstring_copy_n(cx, buffer.as_ptr(), buffer.len());
105}
106
107#[no_mangle]
108pub unsafe extern "C" fn js_normalize_utf16(
109    form: NormalizationForm,
110    ptr: *const u16,
111    len: usize,
112    buffer: *mut c_void,
113) -> bool {
114    let buffer_ptr: *mut Buffer = buffer as *mut Buffer;
115    normalize_utf16(
116        form,
117        core::slice::from_raw_parts(ptr, len),
118        &mut *buffer_ptr,
119    )
120}
121
122#[no_mangle]
123pub unsafe extern "C" fn js_normalize_latin1(
124    form: NormalizationForm,
125    ptr: *const u8,
126    len: usize,
127    buffer: *mut c_void,
128) -> bool {
129    let buffer_ptr: *mut Buffer = buffer as *mut Buffer;
130    normalize_latin1(
131        form,
132        core::slice::from_raw_parts(ptr, len),
133        &mut *buffer_ptr,
134    )
135}
136
137fn maybe_reserve_buffer_space(
138    form: NormalizationForm,
139    input_len: usize,
140    tail_len: usize,
141    buffer: &mut Buffer,
142) -> bool {
143    if input_len <= INLINE_SIZE {
144        // Let's not preallocate on the heap.
145        return true;
146    }
147    // We're going to end up allocating on the heap. Since the allocation
148    // is short-lived, let's overallocate in order to reduce the probability
149    // of allocating multiple times during normalization. These are just
150    // guesses.
151
152    // Note that the normalizer itself calls the infallible `reserve`
153    // with `input_len` as the argument, so it will always do nothing,
154    // since we do greater than or equal to that with `try_reserve`
155    // below.
156    match form {
157        NormalizationForm::NFC => {
158            // Output typically shorter than input.
159            buffer.try_reserve(input_len).is_ok()
160        }
161        NormalizationForm::NFKC => {
162            // Output may expand a bit.
163            let extra = core::cmp::max(8, tail_len / 16);
164            buffer.try_reserve(input_len + extra).is_ok()
165        }
166        NormalizationForm::NFD | NormalizationForm::NFKD => {
167            // Output may expand some more.
168            // `tail_len / 8` is good for Greek
169            // `tail_len / 4 + tail_len / 32` is good for Vietnamese
170            // `tail_len` is good for Korean
171            // Let's pick an arbitrary threshold for sizing for no
172            // reallocation for Korean vs. no reallocation for Greek.
173            let extra = core::cmp::max(
174                16,
175                if tail_len <= 1024 {
176                    tail_len
177                } else {
178                    tail_len / 8
179                },
180            );
181            buffer.try_reserve(input_len + extra).is_ok()
182        }
183    }
184}
185
186fn normalize_utf16(form: NormalizationForm, input: &[u16], buffer: &mut Buffer) -> bool {
187    match form {
188        NormalizationForm::NFC | NormalizationForm::NFKC => {
189            let normalizer = if form == NormalizationForm::NFC {
190                icu_normalizer::ComposingNormalizerBorrowed::new_nfc()
191            } else {
192                icu_normalizer::ComposingNormalizerBorrowed::new_nfkc()
193            };
194            let (head, tail) = normalizer.split_normalized_utf16(input);
195            if tail.is_empty() {
196                return true;
197            }
198            // We make an effort to do a fallible allocation...
199            if !maybe_reserve_buffer_space(form, input.len(), tail.len(), buffer) {
200                return false;
201            }
202            buffer.extend_from_slice(head);
203            // ...but if more space is needed than what we reserved above and
204            // allocation fails during normalization, we abort the program
205            // instead of propagating the allocation error.
206            let r = normalizer.normalize_utf16_to(tail, buffer);
207            debug_assert!(r.is_ok());
208        }
209        NormalizationForm::NFD | NormalizationForm::NFKD => {
210            let normalizer = if form == NormalizationForm::NFD {
211                icu_normalizer::DecomposingNormalizer::new_nfd()
212            } else {
213                icu_normalizer::DecomposingNormalizer::new_nfkd()
214            };
215            let (head, tail) = normalizer.split_normalized_utf16(input);
216            if tail.is_empty() {
217                return true;
218            }
219            // We make an effort to do a fallible allocation...
220            if !maybe_reserve_buffer_space(form, input.len(), tail.len(), buffer) {
221                return false;
222            }
223            buffer.extend_from_slice(head);
224            // ...but if more space is needed than what we reserved above and
225            // allocation fails during normalization, we abort the program
226            // instead of propagating the allocation error.
227            let r = normalizer.normalize_utf16_to(tail, buffer);
228            debug_assert!(r.is_ok());
229        }
230    }
231    true
232}
233
234fn normalize_latin1(form: NormalizationForm, input: &[u8], buffer: &mut Buffer) -> bool {
235    let (head, tail) = match form {
236        NormalizationForm::NFKC => icu_normalizer::latin1::split_normalized_nfkc(input),
237        NormalizationForm::NFD => icu_normalizer::latin1::split_normalized_nfd(input),
238        NormalizationForm::NFKD => icu_normalizer::latin1::split_normalized_nfkd(input),
239        NormalizationForm::NFC => {
240            unreachable!("NFC should have been handled already");
241        }
242    };
243    if tail.is_empty() {
244        return true;
245    }
246    // We make an effort to do a fallible allocation...
247    if !maybe_reserve_buffer_space(form, input.len(), tail.len(), buffer) {
248        return false;
249    }
250    assert!(head.len() <= buffer.capacity());
251    unsafe {
252        // SAFETY: We have enough capacity. Exposure of slice of uninitialized
253        // of integers for writing should be OK in practice:
254        // https://github.com/hsivonen/encoding_rs/issues/79#issuecomment-1211870361
255        //
256        // For long term, see https://doc.rust-lang.org/std/vec/struct.Vec.html#method.spare_capacity_mut.
257        buffer.set_len(head.len());
258    }
259    encoding_rs::mem::convert_latin1_to_utf16(head, buffer);
260    let mut expansion_buffer: Buffer = Buffer::new();
261    if expansion_buffer.try_reserve_exact(tail.len()).is_err() {
262        return false;
263    }
264    unsafe {
265        // SAFETY: We have enough capacity. Exposure of slice of uninitialized
266        // of integers for writing should be OK in practice:
267        // https://github.com/hsivonen/encoding_rs/issues/79#issuecomment-1211870361
268        //
269        // For long term, see https://doc.rust-lang.org/std/vec/struct.Vec.html#method.spare_capacity_mut.
270        expansion_buffer.set_len(tail.len());
271    }
272    encoding_rs::mem::convert_latin1_to_utf16(tail, &mut expansion_buffer);
273    let r = match form {
274        NormalizationForm::NFKC => {
275            icu_normalizer::latin1::normalize_nfkc_to(&expansion_buffer, buffer)
276        }
277        NormalizationForm::NFD => {
278            icu_normalizer::latin1::normalize_nfd_to(&expansion_buffer, buffer)
279        }
280        NormalizationForm::NFKD => {
281            icu_normalizer::latin1::normalize_nfkd_to(&expansion_buffer, buffer)
282        }
283        NormalizationForm::NFC => {
284            unreachable!("NFC should have been handled already");
285        }
286    };
287    debug_assert!(r.is_ok());
288    true
289}
290
291// The items below are not used by SpiderMonkey but are offered through headers that
292// are supposed to work in SpiderMonkey.
293
294#[no_mangle]
295pub unsafe extern "C" fn mozilla_canonical_composition(a: u32, b: u32) -> u32 {
296    icu_normalizer::properties::CanonicalCompositionBorrowed::new()
297        .compose(
298            char::from_u32(a).unwrap_or('\u{0}'),
299            char::from_u32(b).unwrap_or('\u{0}'),
300        )
301        .unwrap_or('\u{0}')
302        .into()
303}
304
305#[no_mangle]
306pub unsafe extern "C" fn mozilla_canonical_combining_class(c: u32) -> u8 {
307    icu_normalizer::properties::CanonicalCombiningClassMapBorrowed::new().get32_u8(c)
308}
mozjs_normalizer_glue/lib.rs

mozjs_normalizer_glue/
lib.rs
