idna/
uts46.rs

1// Copyright The rust-url developers.
2//
3// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
4// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
5// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
6// option. This file may not be copied, modified, or distributed
7// except according to those terms.
8
9//! This module provides the lower-level API for UTS 46.
10//!
11//! [`Uts46::process`] is the core that the other convenience
12//! methods build on.
13//!
14//! UTS 46 flags map to this API as follows:
15//!
16//! * _CheckHyphens_ - _true_: [`Hyphens::Check`], _false_: [`Hyphens::Allow`]; the WHATWG URL Standard sets this to _false_ for normal (non-conformance-checker) user agents.
17//! * _CheckBidi_ - Always _true_; cannot be configured, since this flag is _true_ even when WHATWG URL Standard _beStrict_ is _false_.
18//! * _CheckJoiners_ - Always _true_; cannot be configured, since this flag is _true_ even when WHATWG URL Standard _beStrict_ is _false_.
19//! * _UseSTD3ASCIIRules_ - _true_: [`AsciiDenyList::STD3`], _false_: [`AsciiDenyList::EMPTY`]; however, the check the WHATWG URL Standard performs right after the UTS 46 invocation corresponds to [`AsciiDenyList::URL`].
20//! * _Transitional_Processing_ - Always _false_ but could be implemented as a preprocessing step. This flag is deprecated and for Web purposes the transition is over in the sense that all of Firefox, Safari, or Chrome set this flag to _false_.
21//! * _VerifyDnsLength_ - _true_: [`DnsLength::Verify`], _false_: [`DnsLength::Ignore`]; the WHATWG URL Standard sets this to _false_ for normal (non-conformance-checker) user agents.
22//! * _IgnoreInvalidPunycode_ - Always _false_; cannot be configured. (Not yet covered by the WHATWG URL Standard, but 2 out of 3 major browser clearly behave as if this was _false_).
23
24use crate::punycode::Decoder;
25use crate::punycode::InternalCaller;
26use alloc::borrow::Cow;
27use alloc::string::String;
28use core::fmt::Write;
29use idna_adapter::*;
30use smallvec::SmallVec;
31use utf8_iter::Utf8CharsEx;
32
33/// ICU4C-compatible constraint.
34/// https://unicode-org.atlassian.net/browse/ICU-13727
35const PUNYCODE_DECODE_MAX_INPUT_LENGTH: usize = 2000;
36
37/// ICU4C-compatible constraint. (Note: ICU4C measures
38/// UTF-16 and we measure UTF-32. This means that we
39/// allow longer non-BMP inputs. For this implementation,
40/// the denial-of-service scaling does not depend on BMP vs.
41/// non-BMP: only the scalar values matter.)
42///
43/// https://unicode-org.atlassian.net/browse/ICU-13727
44const PUNYCODE_ENCODE_MAX_INPUT_LENGTH: usize = 1000;
45
46/// For keeping track of what kind of numerals have been
47/// seen in an RTL label.
48#[derive(Debug, PartialEq, Eq)]
49enum RtlNumeralState {
50    Undecided,
51    European,
52    Arabic,
53}
54
55/// Computes the mask for upper-case ASCII.
56const fn upper_case_mask() -> u128 {
57    let mut accu = 0u128;
58    let mut b = 0u8;
59    while b < 128 {
60        if (b >= b'A') && (b <= b'Z') {
61            accu |= 1u128 << b;
62        }
63        b += 1;
64    }
65    accu
66}
67
68/// Bit set for upper-case ASCII.
69const UPPER_CASE_MASK: u128 = upper_case_mask();
70
71/// Computes the mask for glyphless ASCII.
72const fn glyphless_mask() -> u128 {
73    let mut accu = 0u128;
74    let mut b = 0u8;
75    while b < 128 {
76        if (b <= b' ') || (b == 0x7F) {
77            accu |= 1u128 << b;
78        }
79        b += 1;
80    }
81    accu
82}
83
84/// Bit set for glyphless ASCII.
85const GLYPHLESS_MASK: u128 = glyphless_mask();
86
87/// The mask for the ASCII dot.
88const DOT_MASK: u128 = 1 << b'.';
89
90/// Computes the ASCII deny list for STD3 ASCII rules.
91const fn ldh_mask() -> u128 {
92    let mut accu = 0u128;
93    let mut b = 0u8;
94    while b < 128 {
95        if !((b >= b'a' && b <= b'z') || (b >= b'0' && b <= b'9') || b == b'-' || b == b'.') {
96            accu |= 1u128 << b;
97        }
98        b += 1;
99    }
100    accu
101}
102
103const PUNYCODE_PREFIX: u32 =
104    ((b'-' as u32) << 24) | ((b'-' as u32) << 16) | ((b'N' as u32) << 8) | b'X' as u32;
105
106const PUNYCODE_PREFIX_MASK: u32 = (0xFF << 24) | (0xFF << 16) | (0xDF << 8) | 0xDF;
107
108fn write_punycode_label<W: Write + ?Sized>(
109    label: &[char],
110    sink: &mut W,
111) -> Result<(), ProcessingError> {
112    sink.write_str("xn--")?;
113    crate::punycode::encode_into::<_, _, InternalCaller>(label.iter().copied(), sink)?;
114    Ok(())
115}
116
117#[inline(always)]
118fn has_punycode_prefix(slice: &[u8]) -> bool {
119    if slice.len() < 4 {
120        return false;
121    }
122    // Sadly, the optimizer doesn't figure out that more idiomatic code
123    // should compile to masking on 32-bit value.
124    let a = slice[0];
125    let b = slice[1];
126    let c = slice[2];
127    let d = slice[3];
128    let u = (u32::from(d) << 24) | (u32::from(c) << 16) | (u32::from(b) << 8) | u32::from(a);
129    (u & PUNYCODE_PREFIX_MASK) == PUNYCODE_PREFIX
130}
131
132#[inline(always)]
133fn in_inclusive_range8(u: u8, start: u8, end: u8) -> bool {
134    u.wrapping_sub(start) <= (end - start)
135}
136
137#[inline(always)]
138fn in_inclusive_range_char(c: char, start: char, end: char) -> bool {
139    u32::from(c).wrapping_sub(u32::from(start)) <= (u32::from(end) - u32::from(start))
140}
141
142#[inline(always)]
143fn is_passthrough_ascii_label(label: &[u8]) -> bool {
144    // XXX if we aren't performing _CheckHyphens_, this could
145    // check for "xn--" and pass through YouTube CDN node names.
146    if label.len() >= 4 && label[2] == b'-' && label[3] == b'-' {
147        return false;
148    }
149    if let Some((&first, tail)) = label.split_first() {
150        // We need to check the first and last character
151        // more strictly in case this turns out to be a
152        // label in a bidi domain name. This has the side
153        // effect that this function only accepts labels
154        // that also conform to the STD3 rules.
155        //
156        // XXX: If we are in the fail-fast mode (i.e. we don't need
157        // to be able to overwrite anything with U+FFFD), we could
158        // merely record that we've seen a digit here and error out
159        // if we later discover that the domain name is a bidi
160        // domain name.
161        if !in_inclusive_range8(first, b'a', b'z') {
162            return false;
163        }
164        for &b in tail {
165            // If we used LDH_MASK, we'd have to check
166            // the bytes for the ASCII range anyhow.
167            if in_inclusive_range8(b, b'a', b'z') {
168                continue;
169            }
170            if in_inclusive_range8(b, b'0', b'9') {
171                continue;
172            }
173            if b == b'-' {
174                continue;
175            }
176            return false;
177        }
178        label.last() != Some(&b'-')
179    } else {
180        // empty
181        true
182    }
183}
184
185#[inline(always)]
186fn split_ascii_fast_path_prefix(label: &[u8]) -> (&[u8], &[u8]) {
187    if let Some(pos) = label.iter().position(|b| !b.is_ascii()) {
188        if pos == 0 {
189            // First is non-ASCII
190            (&[], label)
191        } else {
192            // Leave one ASCII character in the suffix
193            // in case it's a letter that a combining
194            // character combines with.
195            let (head, tail) = label.split_at(pos - 1);
196            (head, tail)
197        }
198    } else {
199        // All ASCII
200        (label, &[])
201    }
202}
203
204// Input known to be lower-case, but may contain non-ASCII.
205#[inline(always)]
206fn apply_ascii_deny_list_to_lower_cased_unicode(c: char, deny_list: u128) -> char {
207    if let Some(shifted) = 1u128.checked_shl(u32::from(c)) {
208        if (deny_list & shifted) == 0 {
209            c
210        } else {
211            '\u{FFFD}'
212        }
213    } else {
214        c
215    }
216}
217
218// Input known to be ASCII, but may contain upper case ASCII.
219#[inline(always)]
220fn apply_ascii_deny_list_to_potentially_upper_case_ascii(b: u8, deny_list: u128) -> char {
221    if (deny_list & (1u128 << b)) == 0 {
222        return char::from(b);
223    }
224    if in_inclusive_range8(b, b'A', b'Z') {
225        return char::from(b + 0x20);
226    }
227    '\u{FFFD}'
228}
229
230#[inline(always)]
231fn is_ascii(label: &[char]) -> bool {
232    for c in label.iter() {
233        if !c.is_ascii() {
234            return false;
235        }
236    }
237    true
238}
239
240#[derive(PartialEq, Eq, Copy, Clone)]
241enum PunycodeClassification {
242    Ascii,
243    Unicode,
244    Error,
245}
246
247#[inline(always)]
248fn classify_for_punycode(label: &[char]) -> PunycodeClassification {
249    let mut iter = label.iter().copied();
250    loop {
251        if let Some(c) = iter.next() {
252            if c.is_ascii() {
253                continue;
254            }
255            if c == '\u{FFFD}' {
256                return PunycodeClassification::Error;
257            }
258            for c in iter {
259                if c == '\u{FFFD}' {
260                    return PunycodeClassification::Error;
261                }
262            }
263            return PunycodeClassification::Unicode;
264        }
265        return PunycodeClassification::Ascii;
266    }
267}
268
269/// The ASCII deny list to be applied.
270#[derive(PartialEq, Eq, Copy, Clone)]
271#[repr(transparent)]
272pub struct AsciiDenyList {
273    bits: u128,
274}
275
276impl AsciiDenyList {
277    /// Computes (preferably at compile time) an ASCII deny list.
278    ///
279    /// Setting `deny_glyphless` to `true` denies U+0020 SPACE and below
280    /// as well as U+007F DELETE for convenience without having to list
281    /// these characters in the `deny_list` string.
282    ///
283    /// `deny_list` is the list of ASCII characters to deny. This
284    /// list must not contain any of:
285    /// * Letters
286    /// * Digits
287    /// * Hyphen
288    /// * Dot (period / full-stop)
289    /// * Non-ASCII
290    ///
291    /// # Panics
292    ///
293    /// If the deny list contains characters listed as prohibited above.
294    pub const fn new(deny_glyphless: bool, deny_list: &str) -> Self {
295        let mut bits = UPPER_CASE_MASK;
296        if deny_glyphless {
297            bits |= GLYPHLESS_MASK;
298        }
299        let mut i = 0;
300        let bytes = deny_list.as_bytes();
301        while i < bytes.len() {
302            let b = bytes[i];
303            assert!(b < 0x80, "ASCII deny list must be ASCII.");
304            // assert_ne not yet available in const context.
305            assert!(b != b'.', "ASCII deny list must not contain the dot.");
306            assert!(b != b'-', "ASCII deny list must not contain the hyphen.");
307            assert!(
308                !((b >= b'0') && (b <= b'9')),
309                "ASCII deny list must not contain digits."
310            );
311            assert!(
312                !((b >= b'a') && (b <= b'z')),
313                "ASCII deny list must not contain letters."
314            );
315            assert!(
316                !((b >= b'A') && (b <= b'Z')),
317                "ASCII deny list must not contain letters."
318            );
319            bits |= 1u128 << b;
320            i += 1;
321        }
322        Self { bits }
323    }
324
325    /// No ASCII deny list. This corresponds to _UseSTD3ASCIIRules=false_.
326    ///
327    /// Equivalent to `AsciiDenyList::new(false, "")`.
328    ///
329    /// Note: Not denying the space and control characters can result in
330    /// strange behavior. Without a deny list provided to the UTS 46
331    /// operation, the caller is expected perform filtering afterwards,
332    /// but it's more efficient to use `AsciiDenyList` than post-processing,
333    /// because the internals of this crate can optimize away checks in
334    /// certain cases.
335    pub const EMPTY: Self = Self::new(false, "");
336
337    /// The STD3 deny list. This corresponds to _UseSTD3ASCIIRules=true_.
338    ///
339    /// Note that this deny list rejects the underscore, which occurs in
340    /// pseudo-hosts used by various TXT record-based protocols, and also
341    /// characters that may occurs in non-DNS naming, such as NetBIOS.
342    pub const STD3: Self = Self { bits: ldh_mask() };
343
344    /// [Forbidden domain code point](https://url.spec.whatwg.org/#forbidden-domain-code-point) from the WHATWG URL Standard.
345    ///
346    /// Equivalent to `AsciiDenyList::new(true, "%#/:<>?@[\\]^|")`.
347    ///
348    /// Note that this deny list rejects IPv6 addresses, so (as in URL
349    /// parsing) you need to check for IPv6 addresses first and not
350    /// put them through UTS 46 processing.
351    pub const URL: Self = Self::new(true, "%#/:<>?@[\\]^|");
352}
353
354/// The _CheckHyphens_ mode.
355#[derive(PartialEq, Eq, Copy, Clone)]
356#[non_exhaustive] // non_exhaustive in case a middle mode that prohibits only first and last position needs to be added
357pub enum Hyphens {
358    /// _CheckHyphens=false_: Do not place positional restrictions on hyphens.
359    ///
360    /// This mode is used by the WHATWG URL Standard for normal User Agent processing
361    /// (i.e. not conformance checking).
362    Allow,
363
364    /// Prohibit hyphens in the first and last position in the label but allow in
365    /// the third and fourth position.
366    ///
367    /// Note that this mode rejects real-world names, including some GitHub user pages.
368    CheckFirstLast,
369
370    /// _CheckHyphens=true_: Prohibit hyphens in the first, third, fourth,
371    /// and last position in the label.
372    ///
373    /// Note that this mode rejects real-world names, including YouTube CDN nodes
374    /// and some GitHub user pages.
375    Check,
376}
377
378/// The UTS 46 _VerifyDNSLength_ flag.
379#[derive(PartialEq, Eq, Copy, Clone)]
380#[non_exhaustive]
381pub enum DnsLength {
382    /// _VerifyDNSLength=false_. (Possibly relevant for allowing non-DNS naming systems.)
383    Ignore,
384    /// _VerifyDNSLength=true_ with the exception that the trailing root label dot is
385    /// allowed.
386    VerifyAllowRootDot,
387    /// _VerifyDNSLength=true_. (The trailing root label dot is not allowed.)
388    Verify,
389}
390
391/// Policy for customizing behavior in case of an error.
392#[derive(PartialEq, Eq, Copy, Clone)]
393#[non_exhaustive]
394pub enum ErrorPolicy {
395    /// Return as early as possible without producing output in case of error.
396    FailFast,
397    /// In case of error, mark errors with the REPLACEMENT CHARACTER. (The output
398    /// containing REPLACEMENT CHARACTERs may be show to the user to illustrate
399    /// what was wrong but must not be used for naming in a network protocol.)
400    MarkErrors,
401}
402
403/// The success outcome of [`Uts46::process`]
404#[derive(PartialEq, Eq, Copy, Clone, Debug)]
405pub enum ProcessingSuccess {
406    /// There were no errors. The caller must consider the input to be the output.
407    ///
408    /// This asserts that the input can be safely passed to [`core::str::from_utf8_unchecked`].
409    ///
410    /// (Distinct from `WroteToSink` in order to allow `Cow` behavior to be implemented on top of
411    /// [`Uts46::process`].)
412    Passthrough,
413
414    /// There were no errors. The caller must consider what was written to the sink to be the output.
415    ///
416    /// (Distinct from `Passthrough` in order to allow `Cow` behavior to be implemented on top of
417    /// [`Uts46::process`].)
418    WroteToSink,
419}
420
421/// The failure outcome of [`Uts46::process`]
422#[derive(PartialEq, Eq, Copy, Clone, Debug)]
423pub enum ProcessingError {
424    /// There was a validity error according to the chosen options.
425    ///
426    /// In case of `Operation::ToAscii`, there is no output. Otherwise, output was written to the
427    /// sink and the output contains at least one U+FFFD REPLACEMENT CHARACTER to denote an error.
428    ValidityError,
429
430    /// The sink emitted [`core::fmt::Error`]. The partial output written to the sink must not
431    /// be used.
432    SinkError,
433}
434
435impl From<core::fmt::Error> for ProcessingError {
436    fn from(_: core::fmt::Error) -> Self {
437        Self::SinkError
438    }
439}
440
441impl From<crate::punycode::PunycodeEncodeError> for ProcessingError {
442    fn from(_: crate::punycode::PunycodeEncodeError) -> Self {
443        unreachable!(
444            "Punycode overflows should not be possible due to PUNYCODE_ENCODE_MAX_INPUT_LENGTH"
445        );
446    }
447}
448
449#[derive(Debug, Clone, Copy)]
450enum AlreadyAsciiLabel<'a> {
451    MixedCaseAscii(&'a [u8]),
452    MixedCasePunycode(&'a [u8]),
453    Other,
454}
455
456/// Performs the _VerifyDNSLength_ check on the output of the _ToASCII_ operation.
457///
458/// If the second argument is `false`, the trailing root label dot is allowed.
459///
460/// # Panics
461///
462/// Panics in debug mode if the argument isn't ASCII.
463pub fn verify_dns_length(domain_name: &str, allow_trailing_dot: bool) -> bool {
464    let bytes = domain_name.as_bytes();
465    debug_assert!(bytes.is_ascii());
466    let domain_name_without_trailing_dot = if let Some(without) = bytes.strip_suffix(b".") {
467        if !allow_trailing_dot {
468            return false;
469        }
470        without
471    } else {
472        bytes
473    };
474    if domain_name_without_trailing_dot.len() > 253 {
475        return false;
476    }
477    for label in domain_name_without_trailing_dot.split(|b| *b == b'.') {
478        if label.is_empty() {
479            return false;
480        }
481        if label.len() > 63 {
482            return false;
483        }
484    }
485    true
486}
487
488/// An implementation of UTS #46.
489pub struct Uts46 {
490    data: idna_adapter::Adapter,
491}
492
493#[cfg(feature = "compiled_data")]
494impl Default for Uts46 {
495    fn default() -> Self {
496        Self::new()
497    }
498}
499
500impl Uts46 {
501    /// Constructor using data compiled into the binary.
502    #[cfg(feature = "compiled_data")]
503    pub const fn new() -> Self {
504        Self {
505            data: idna_adapter::Adapter::new(),
506        }
507    }
508
509    // XXX Should there be an `icu_provider` feature for enabling
510    // a constructor for run-time data loading?
511
512    /// Performs the [ToASCII](https://www.unicode.org/reports/tr46/#ToASCII) operation
513    /// from UTS #46 with the options indicated.
514    ///
515    /// # Arguments
516    ///
517    /// * `domain_name` - The input domain name as UTF-8 bytes. (The UTF-8ness is checked by
518    ///   this method and input that is not well-formed UTF-8 is treated as an error. If you
519    ///   already have a `&str`, call `.as_bytes()` on it.)
520    /// * `ascii_deny_list` - What ASCII deny list, if any, to apply. The UTS 46
521    ///   _UseSTD3ASCIIRules_ flag or the WHATWG URL Standard forbidden domain code point
522    ///   processing is handled via this argument. Most callers are probably the best off
523    ///   by using [`AsciiDenyList::URL`] here.
524    /// * `hyphens` - The UTS 46 _CheckHyphens_ flag. Most callers are probably the best
525    ///   off by using [`Hyphens::Allow`] here.
526    /// * `dns_length` - The UTS 46 _VerifyDNSLength_ flag.
527    pub fn to_ascii<'a>(
528        &self,
529        domain_name: &'a [u8],
530        ascii_deny_list: AsciiDenyList,
531        hyphens: Hyphens,
532        dns_length: DnsLength,
533    ) -> Result<Cow<'a, str>, crate::Errors> {
534        self.to_ascii_from_cow(
535            Cow::Borrowed(domain_name),
536            ascii_deny_list,
537            hyphens,
538            dns_length,
539        )
540    }
541
542    pub(crate) fn to_ascii_from_cow<'a>(
543        &self,
544        domain_name: Cow<'a, [u8]>,
545        ascii_deny_list: AsciiDenyList,
546        hyphens: Hyphens,
547        dns_length: DnsLength,
548    ) -> Result<Cow<'a, str>, crate::Errors> {
549        let mut s = String::new();
550        match self.process(
551            &domain_name,
552            ascii_deny_list,
553            hyphens,
554            ErrorPolicy::FailFast,
555            |_, _, _| false,
556            &mut s,
557            None,
558        ) {
559            Ok(ProcessingSuccess::Passthrough) => {
560                // SAFETY: `ProcessingSuccess::Passthrough` asserts that `domain_name` is ASCII.
561                let cow = match domain_name {
562                    Cow::Borrowed(v) => Cow::Borrowed(unsafe { core::str::from_utf8_unchecked(v) }),
563                    Cow::Owned(v) => Cow::Owned(unsafe { String::from_utf8_unchecked(v) }),
564                };
565                if dns_length != DnsLength::Ignore
566                    && !verify_dns_length(&cow, dns_length == DnsLength::VerifyAllowRootDot)
567                {
568                    Err(crate::Errors::default())
569                } else {
570                    Ok(cow)
571                }
572            }
573            Ok(ProcessingSuccess::WroteToSink) => {
574                let cow: Cow<'_, str> = Cow::Owned(s);
575                if dns_length != DnsLength::Ignore
576                    && !verify_dns_length(&cow, dns_length == DnsLength::VerifyAllowRootDot)
577                {
578                    Err(crate::Errors::default())
579                } else {
580                    Ok(cow)
581                }
582            }
583            Err(ProcessingError::ValidityError) => Err(crate::Errors::default()),
584            Err(ProcessingError::SinkError) => unreachable!(),
585        }
586    }
587
588    /// Performs the [ToUnicode](https://www.unicode.org/reports/tr46/#ToUnicode) operation
589    /// from UTS #46 according to the options given. When there
590    /// are errors, there is still output, which may be rendered user, even through
591    /// the output must not be used in networking protocols. Errors are denoted
592    /// by U+FFFD REPLACEMENT CHARACTERs in the output. (That is, if the second item of the
593    /// return tuple is `Err`, the first item of the return tuple is guaranteed to contain
594    /// at least one U+FFFD.)
595    ///
596    /// Most applications probably shouldn't use this method and should be using
597    /// [`Uts46::to_user_interface`] instead.
598    ///
599    /// # Arguments
600    ///
601    /// * `domain_name` - The input domain name as UTF-8 bytes. (The UTF-8ness is checked by
602    ///   this method and input that is not well-formed UTF-8 is treated as an error. If you
603    ///   already have a `&str`, call `.as_bytes()` on it.)
604    /// * `ascii_deny_list` - What ASCII deny list, if any, to apply. The UTS 46
605    ///   _UseSTD3ASCIIRules_ flag or the WHATWG URL Standard forbidden domain code point
606    ///   processing is handled via this argument. Most callers are probably the best off
607    ///   by using [`AsciiDenyList::URL`] here.
608    /// * `hyphens` - The UTS 46 _CheckHyphens_ flag. Most callers are probably the best
609    ///   off by using [`Hyphens::Allow`] here.
610    pub fn to_unicode<'a>(
611        &self,
612        domain_name: &'a [u8],
613        ascii_deny_list: AsciiDenyList,
614        hyphens: Hyphens,
615    ) -> (Cow<'a, str>, Result<(), crate::Errors>) {
616        self.to_user_interface(domain_name, ascii_deny_list, hyphens, |_, _, _| true)
617    }
618
619    /// Performs the [ToUnicode](https://www.unicode.org/reports/tr46/#ToUnicode) operation
620    /// from UTS #46 according to options given with some
621    /// error-free Unicode labels output according to
622    /// [ToASCII](https://www.unicode.org/reports/tr46/#ToASCII) instead as decided by
623    /// application policy implemented via the `output_as_unicode` closure. The purpose
624    /// is to convert user-visible domains to the Unicode form in general but to render
625    /// potentially misleading labels as Punycode.
626    ///
627    /// This is an imperfect security mechanism, because [the Punycode form itself may be
628    /// resemble a user-recognizable name](https://www.unicode.org/reports/tr36/#TablePunycodeSpoofing).
629    /// However, since this mechanism is common practice, this API provides support for The
630    /// the mechanism.
631    ///
632    /// ASCII labels always pass through as ASCII and labels with errors always pass through
633    /// as Unicode. For non-erroneous labels that contain at least one non-ASCII character
634    /// (implies non-empty), `output_as_unicode` is called with the Unicode form of the label,
635    /// the TLD (potentially empty), and a flag indicating whether the domain name as a whole
636    /// is a bidi domain name. If the return value is `true`, the label passes through as
637    /// Unicode. If the return value is `false`, the label is converted to Punycode.
638    ///
639    /// When there are errors, there is still output, which may be rendered user, even through
640    /// the output must not be used in networking protocols. Errors are denoted by
641    /// U+FFFD REPLACEMENT CHARACTERs in the output. (That is, if the second item
642    /// of the return tuple is `Err`, the first item of the return tuple is guaranteed to contain
643    /// at least one U+FFFD.) Labels that contain errors are not converted to Punycode.
644    ///
645    /// # Arguments
646    ///
647    /// * `domain_name` - The input domain name as UTF-8 bytes. (The UTF-8ness is checked by
648    ///   this method and input that is not well-formed UTF-8 is treated as an error. If you
649    ///   already have a `&str`, call `.as_bytes()` on it.)
650    /// * `ascii_deny_list` - What ASCII deny list, if any, to apply. The UTS 46
651    ///   _UseSTD3ASCIIRules_ flag or the WHATWG URL Standard forbidden domain code point
652    ///   processing is handled via this argument. Most callers are probably the best off
653    ///   by using [`AsciiDenyList::URL`] here.
654    /// * `hyphens` - The UTS 46 _CheckHyphens_ flag. Most callers are probably the best
655    ///   off by using [`Hyphens::Allow`] here.
656    /// * `output_as_unicode` - A closure for deciding if a label should be output as Unicode
657    ///   (as opposed to Punycode). The first argument is the label for which a decision is
658    ///   needed (always non-empty slice). The second argument is the TLD (potentially empty).
659    ///   The third argument is `true` iff the domain name as a whole is a bidi domain name.
660    ///   Only non-erroneous labels that contain at least one non-ASCII character are passed
661    ///   to the closure as the first argument. The second and third argument values are
662    ///   guaranteed to remain the same during a single call to `process`, and the closure
663    ///   may cache computations derived from the second and third argument (hence the
664    ///   `FnMut` type).
665    pub fn to_user_interface<'a, OutputUnicode: FnMut(&[char], &[char], bool) -> bool>(
666        &self,
667        domain_name: &'a [u8],
668        ascii_deny_list: AsciiDenyList,
669        hyphens: Hyphens,
670        output_as_unicode: OutputUnicode,
671    ) -> (Cow<'a, str>, Result<(), crate::Errors>) {
672        let mut s = String::new();
673        match self.process(
674            domain_name,
675            ascii_deny_list,
676            hyphens,
677            ErrorPolicy::MarkErrors,
678            output_as_unicode,
679            &mut s,
680            None,
681        ) {
682            // SAFETY: `ProcessingSuccess::Passthrough` asserts that `domain_name` is ASCII.
683            Ok(ProcessingSuccess::Passthrough) => (
684                Cow::Borrowed(unsafe { core::str::from_utf8_unchecked(domain_name) }),
685                Ok(()),
686            ),
687            Ok(ProcessingSuccess::WroteToSink) => (Cow::Owned(s), Ok(())),
688            Err(ProcessingError::ValidityError) => (Cow::Owned(s), Err(crate::Errors::default())),
689            Err(ProcessingError::SinkError) => unreachable!(),
690        }
691    }
692
693    /// The lower-level function that [`Uts46::to_ascii`], [`Uts46::to_unicode`], and
694    /// [`Uts46::to_user_interface`] are built on to allow support for output types other
695    /// than `Cow<'a, str>` (e.g. string types in a non-Rust programming language).
696    ///
697    /// # Arguments
698    ///
699    /// * `domain_name` - The input domain name as UTF-8 bytes. (The UTF-8ness is checked by
700    ///   this method and input that is not well-formed UTF-8 is treated as an error. If you
701    ///   already have a `&str`, call `.as_bytes()` on it.)
702    /// * `ascii_deny_list` - What ASCII deny list, if any, to apply. The UTS 46
703    ///   _UseSTD3ASCIIRules_ flag or the WHATWG URL Standard forbidden domain code point
704    ///   processing is handled via this argument. Most callers are probably the best off
705    ///   by using [`AsciiDenyList::URL`] here.
706    /// * `hyphens` - The UTS 46 _CheckHyphens_ flag. Most callers are probably the best
707    ///   off by using [`Hyphens::Allow`] here.
708    /// * `error_policy` - Whether to fail fast or to produce output that may be rendered
709    ///   for the user to examine in case of errors.
710    /// * `output_as_unicode` - A closure for deciding if a label should be output as Unicode
711    ///   (as opposed to Punycode). The first argument is the label for which a decision is
712    ///   needed (always non-empty slice). The second argument is the TLD (potentially empty).
713    ///   The third argument is `true` iff the domain name as a whole is a bidi domain name.
714    ///   Only non-erroneous labels that contain at least one non-ASCII character are passed
715    ///   to the closure as the first argument. The second and third argument values are
716    ///   guaranteed to remain the same during a single call to `process`, and the closure
717    ///   may cache computations derived from the second and third argument (hence the
718    ///   `FnMut` type). To perform the _ToASCII_ operation, `|_, _, _| false` must be
719    ///   passed as the closure. To perform the _ToUnicode_ operation, `|_, _, _| true` must
720    ///   be passed as the closure. A more complex closure may be used to prepare a domain
721    ///   name for display in a user interface so that labels are converted to the Unicode
722    ///   form in general but potentially misleading labels are converted to the Punycode
723    ///   form.
724    /// * `sink` - The object that receives the output (in the non-passthrough case).
725    /// * `ascii_sink` - A second sink that receives the _ToASCII_ form only if there
726    ///   were no errors and `sink` received at least one character of non-ASCII output.
727    ///   The purpose of this argument is to enable a user interface display form of the
728    ///   domain and the _ToASCII_ form of the domain to be computed efficiently together.
729    ///   This argument is useless when `output_as_unicode` always returns `false`, in
730    ///   which case the _ToASCII_ form ends up in `sink` already. If `ascii_sink` receives
731    ///   no output and the return value is `Ok(ProcessingSuccess::WroteToSink)`, use the
732    ///   output received by `sink` also as the _ToASCII_ result.
733    ///
734    /// # Return value
735    ///
736    /// * `Ok(ProcessingSuccess::Passthrough)` - The caller must treat
737    ///   `unsafe { core::str::from_utf8_unchecked(domain_name) }` as the output. (This
738    ///   return value asserts that calling `core::str::from_utf8_unchecked(domain_name)`
739    ///   is safe.)
740    /// * `Ok(ProcessingSuccess::WroteToSink)` - The caller must treat was was written
741    ///   to `sink` as the output. If another sink was passed as `ascii_sink` but it did
742    ///   not receive output, the caller must treat what was written to `sink` also as
743    ///   the _ToASCII_ output. Otherwise, if `ascii_sink` received output, the caller
744    ///   must treat what was written to `ascii_sink` as the _ToASCII_ output.
745    /// * `Err(ProcessingError::ValidityError)` - The input was in error and must
746    ///   not be used for DNS lookup or otherwise in a network protocol. If `error_policy`
747    ///   was `ErrorPolicy::MarkErrors`, the output written to `sink` may be displayed
748    ///   to the user as an illustration of where the error was or the errors were.
749    /// * `Err(ProcessingError::SinkError)` - Either `sink` or `ascii_sink` returned
750    ///   [`core::fmt::Error`]. The partial output written to `sink` `ascii_sink` must not
751    ///   be used. If `W` never returns [`core::fmt::Error`], this method never returns
752    ///   `Err(ProcessingError::SinkError)`.
753    ///
754    /// # Safety-usable invariant
755    ///
756    /// If the return value is `Ok(ProcessingSuccess::Passthrough)`, `domain_name` is
757    /// ASCII and `core::str::from_utf8_unchecked(domain_name)` is safe. (Note:
758    /// Other return values do _not_ imply that `domain_name` wasn't ASCII!)
759    ///
760    /// # Security considerations
761    ///
762    /// Showing labels whose Unicode form might mislead the user as Punycode instead is
763    /// an imperfect security mechanism, because [the Punycode form itself may be resemble
764    /// a user-recognizable name](https://www.unicode.org/reports/tr36/#TablePunycodeSpoofing).
765    /// However, since this mechanism is common practice, this API provides support for the
766    /// the mechanism.
767    ///
768    /// Punycode processing is quadratic, so to avoid denial of service, this method imposes
769    /// length limits on Punycode treating especially long inputs as being in error. These
770    /// limits are well higher than the DNS length limits and are not more restrictive than
771    /// the limits imposed by ICU4C.
772    #[allow(clippy::too_many_arguments)]
773    pub fn process<W: Write + ?Sized, OutputUnicode: FnMut(&[char], &[char], bool) -> bool>(
774        &self,
775        domain_name: &[u8],
776        ascii_deny_list: AsciiDenyList,
777        hyphens: Hyphens,
778        error_policy: ErrorPolicy,
779        mut output_as_unicode: OutputUnicode,
780        sink: &mut W,
781        ascii_sink: Option<&mut W>,
782    ) -> Result<ProcessingSuccess, ProcessingError> {
783        let fail_fast = error_policy == ErrorPolicy::FailFast;
784        let mut domain_buffer = SmallVec::<[char; 253]>::new();
785        let mut already_punycode = SmallVec::<[AlreadyAsciiLabel; 8]>::new();
786        // `process_inner` could be pasted inline here, but it's out of line in order
787        // to avoid duplicating that code when monomorphizing over `W` and `OutputUnicode`.
788        let (passthrough_up_to, is_bidi, had_errors) = self.process_inner(
789            domain_name,
790            ascii_deny_list,
791            hyphens,
792            fail_fast,
793            &mut domain_buffer,
794            &mut already_punycode,
795        );
796        if passthrough_up_to == domain_name.len() {
797            debug_assert!(!had_errors);
798            return Ok(ProcessingSuccess::Passthrough);
799        }
800        // Checked only after passthrough as a micro optimization.
801        if fail_fast && had_errors {
802            return Err(ProcessingError::ValidityError);
803        }
804        debug_assert_eq!(had_errors, domain_buffer.contains(&'\u{FFFD}'));
805        let without_dot = if let Some(without_dot) = domain_buffer.strip_suffix(&['.']) {
806            without_dot
807        } else {
808            &domain_buffer[..]
809        };
810        // unwrap is OK, because we always have at least one label
811        let tld = without_dot.rsplit(|c| *c == '.').next().unwrap();
812        let mut had_unicode_output = false;
813        let mut seen_label = false;
814        let mut already_punycode_iter = already_punycode.iter();
815        let mut passthrough_up_to_extended = passthrough_up_to;
816        let mut flushed_prefix = false;
817        for label in domain_buffer.split(|c| *c == '.') {
818            // Unwrap is OK, because there are supposed to be as many items in
819            // `already_punycode` as there are labels.
820            let input_punycode = *already_punycode_iter.next().unwrap();
821            if seen_label {
822                if flushed_prefix {
823                    sink.write_char('.')?;
824                } else {
825                    debug_assert_eq!(domain_name[passthrough_up_to_extended], b'.');
826                    passthrough_up_to_extended += 1;
827                    if passthrough_up_to_extended == domain_name.len() {
828                        debug_assert!(!had_errors);
829                        return Ok(ProcessingSuccess::Passthrough);
830                    }
831                }
832            }
833            seen_label = true;
834
835            if let AlreadyAsciiLabel::MixedCaseAscii(mixed_case) = input_punycode {
836                if let Some(first_upper_case) =
837                    mixed_case.iter().position(|c| c.is_ascii_uppercase())
838                {
839                    let (head, tail) = mixed_case.split_at(first_upper_case);
840                    let slice_to_write = if flushed_prefix {
841                        head
842                    } else {
843                        flushed_prefix = true;
844                        passthrough_up_to_extended += head.len();
845                        debug_assert_ne!(passthrough_up_to_extended, domain_name.len());
846                        &domain_name[..passthrough_up_to_extended]
847                    };
848                    // SAFETY: `mixed_case` and `domain_name` up to `passthrough_up_to_extended` are known to be ASCII.
849                    sink.write_str(unsafe { core::str::from_utf8_unchecked(slice_to_write) })?;
850                    for c in tail.iter() {
851                        sink.write_char(char::from(c.to_ascii_lowercase()))?;
852                    }
853                } else if flushed_prefix {
854                    // SAFETY: `mixed_case` is known to be ASCII.
855                    sink.write_str(unsafe { core::str::from_utf8_unchecked(mixed_case) })?;
856                } else {
857                    passthrough_up_to_extended += mixed_case.len();
858                    if passthrough_up_to_extended == domain_name.len() {
859                        debug_assert!(!had_errors);
860                        return Ok(ProcessingSuccess::Passthrough);
861                    }
862                }
863                continue;
864            }
865
866            let potentially_punycode = if fail_fast {
867                debug_assert!(classify_for_punycode(label) != PunycodeClassification::Error);
868                !is_ascii(label)
869            } else {
870                classify_for_punycode(label) == PunycodeClassification::Unicode
871            };
872            let passthrough = if potentially_punycode {
873                let unicode = output_as_unicode(label, tld, is_bidi);
874                had_unicode_output |= unicode;
875                unicode
876            } else {
877                true
878            };
879            if passthrough {
880                if !flushed_prefix {
881                    flushed_prefix = true;
882                    // SAFETY: `domain_name` up to `passthrough_up_to_extended` is known to be ASCII.
883                    sink.write_str(unsafe {
884                        core::str::from_utf8_unchecked(&domain_name[..passthrough_up_to_extended])
885                    })?;
886                }
887                for c in label.iter().copied() {
888                    sink.write_char(c)?;
889                }
890            } else if let AlreadyAsciiLabel::MixedCasePunycode(mixed_case) = input_punycode {
891                if let Some(first_upper_case) =
892                    mixed_case.iter().position(|c| c.is_ascii_uppercase())
893                {
894                    let (head, tail) = mixed_case.split_at(first_upper_case);
895                    let slice_to_write = if flushed_prefix {
896                        head
897                    } else {
898                        flushed_prefix = true;
899                        passthrough_up_to_extended += head.len();
900                        debug_assert_ne!(passthrough_up_to_extended, domain_name.len());
901                        &domain_name[..passthrough_up_to_extended]
902                    };
903                    // SAFETY: `mixed_case` and `domain_name` up to `passthrough_up_to_extended` are known to be ASCII.
904                    sink.write_str(unsafe { core::str::from_utf8_unchecked(slice_to_write) })?;
905                    for c in tail.iter() {
906                        sink.write_char(char::from(c.to_ascii_lowercase()))?;
907                    }
908                } else if flushed_prefix {
909                    // SAFETY: `mixed_case` is known to be ASCII.
910                    sink.write_str(unsafe { core::str::from_utf8_unchecked(mixed_case) })?;
911                } else {
912                    passthrough_up_to_extended += mixed_case.len();
913                    if passthrough_up_to_extended == domain_name.len() {
914                        debug_assert!(!had_errors);
915                        return Ok(ProcessingSuccess::Passthrough);
916                    }
917                }
918            } else {
919                if !flushed_prefix {
920                    flushed_prefix = true;
921                    // SAFETY: `domain_name` up to `passthrough_up_to_extended` is known to be ASCII.
922                    sink.write_str(unsafe {
923                        core::str::from_utf8_unchecked(&domain_name[..passthrough_up_to_extended])
924                    })?;
925                }
926                write_punycode_label(label, sink)?;
927            }
928        }
929
930        if had_errors {
931            return Err(ProcessingError::ValidityError);
932        }
933
934        if had_unicode_output {
935            if let Some(sink) = ascii_sink {
936                let mut seen_label = false;
937                let mut already_punycode_iter = already_punycode.iter();
938                let mut passthrough_up_to_extended = passthrough_up_to;
939                let mut flushed_prefix = false;
940                for label in domain_buffer.split(|c| *c == '.') {
941                    // Unwrap is OK, because there are supposed to be as many items in
942                    // `already_punycode` as there are labels.
943                    let input_punycode = *already_punycode_iter.next().unwrap();
944                    if seen_label {
945                        if flushed_prefix {
946                            sink.write_char('.')?;
947                        } else {
948                            debug_assert_eq!(domain_name[passthrough_up_to_extended], b'.');
949                            passthrough_up_to_extended += 1;
950                        }
951                    }
952                    seen_label = true;
953
954                    if let AlreadyAsciiLabel::MixedCaseAscii(mixed_case) = input_punycode {
955                        if let Some(first_upper_case) =
956                            mixed_case.iter().position(|c| c.is_ascii_uppercase())
957                        {
958                            let (head, tail) = mixed_case.split_at(first_upper_case);
959                            let slice_to_write = if flushed_prefix {
960                                head
961                            } else {
962                                flushed_prefix = true;
963                                passthrough_up_to_extended += head.len();
964                                debug_assert_ne!(passthrough_up_to_extended, domain_name.len());
965                                &domain_name[..passthrough_up_to_extended]
966                            };
967                            // SAFETY: `mixed_case` and `domain_name` up to `passthrough_up_to_extended` are known to be ASCII.
968                            sink.write_str(unsafe {
969                                core::str::from_utf8_unchecked(slice_to_write)
970                            })?;
971                            for c in tail.iter() {
972                                sink.write_char(char::from(c.to_ascii_lowercase()))?;
973                            }
974                        } else if flushed_prefix {
975                            // SAFETY: `mixed_case` is known to be ASCII.
976                            sink.write_str(unsafe { core::str::from_utf8_unchecked(mixed_case) })?;
977                        } else {
978                            passthrough_up_to_extended += mixed_case.len();
979                        }
980                        continue;
981                    }
982
983                    if is_ascii(label) {
984                        if !flushed_prefix {
985                            flushed_prefix = true;
986                            // SAFETY: `domain_name` up to `passthrough_up_to_extended` is known to be ASCII.
987                            sink.write_str(unsafe {
988                                core::str::from_utf8_unchecked(
989                                    &domain_name[..passthrough_up_to_extended],
990                                )
991                            })?;
992                        }
993                        for c in label.iter().copied() {
994                            sink.write_char(c)?;
995                        }
996                    } else if let AlreadyAsciiLabel::MixedCasePunycode(mixed_case) = input_punycode
997                    {
998                        if let Some(first_upper_case) =
999                            mixed_case.iter().position(|c| c.is_ascii_uppercase())
1000                        {
1001                            let (head, tail) = mixed_case.split_at(first_upper_case);
1002                            let slice_to_write = if flushed_prefix {
1003                                head
1004                            } else {
1005                                flushed_prefix = true;
1006                                passthrough_up_to_extended += head.len();
1007                                debug_assert_ne!(passthrough_up_to_extended, domain_name.len());
1008                                &domain_name[..passthrough_up_to_extended]
1009                            };
1010                            // SAFETY: `mixed_case` and `domain_name` up to `passthrough_up_to_extended` are known to be ASCII.
1011                            sink.write_str(unsafe {
1012                                core::str::from_utf8_unchecked(slice_to_write)
1013                            })?;
1014                            for c in tail.iter() {
1015                                sink.write_char(char::from(c.to_ascii_lowercase()))?;
1016                            }
1017                        } else if flushed_prefix {
1018                            // SAFETY: `mixed_case` is known to be ASCII.
1019                            sink.write_str(unsafe { core::str::from_utf8_unchecked(mixed_case) })?;
1020                        } else {
1021                            passthrough_up_to_extended += mixed_case.len();
1022                        }
1023                    } else {
1024                        if !flushed_prefix {
1025                            flushed_prefix = true;
1026                            // SAFETY: `domain_name` up to `passthrough_up_to_extended` is known to be ASCII.
1027                            sink.write_str(unsafe {
1028                                core::str::from_utf8_unchecked(
1029                                    &domain_name[..passthrough_up_to_extended],
1030                                )
1031                            })?;
1032                        }
1033                        write_punycode_label(label, sink)?;
1034                    }
1035                }
1036                if !flushed_prefix {
1037                    // SAFETY: `domain_name` up to `passthrough_up_to_extended` is known to be ASCII.
1038                    sink.write_str(unsafe {
1039                        core::str::from_utf8_unchecked(&domain_name[..passthrough_up_to_extended])
1040                    })?;
1041                }
1042            }
1043        }
1044        Ok(ProcessingSuccess::WroteToSink)
1045    }
1046
1047    /// The part of `process` that doesn't need to be generic over the sink.
1048    #[inline(always)]
1049    fn process_inner<'a>(
1050        &self,
1051        domain_name: &'a [u8],
1052        ascii_deny_list: AsciiDenyList,
1053        hyphens: Hyphens,
1054        fail_fast: bool,
1055        domain_buffer: &mut SmallVec<[char; 253]>,
1056        already_punycode: &mut SmallVec<[AlreadyAsciiLabel<'a>; 8]>,
1057    ) -> (usize, bool, bool) {
1058        // Sadly, this even faster-path ASCII tier is needed to avoid regressing
1059        // performance.
1060        let mut iter = domain_name.iter();
1061        let mut most_recent_label_start = iter.clone();
1062        loop {
1063            if let Some(&b) = iter.next() {
1064                if in_inclusive_range8(b, b'a', b'z') {
1065                    continue;
1066                }
1067                if b == b'.' {
1068                    most_recent_label_start = iter.clone();
1069                    continue;
1070                }
1071                return self.process_innermost(
1072                    domain_name,
1073                    ascii_deny_list,
1074                    hyphens,
1075                    fail_fast,
1076                    domain_buffer,
1077                    already_punycode,
1078                    most_recent_label_start.as_slice(),
1079                );
1080            } else {
1081                // Success! The whole input passes through on the fastest path!
1082                return (domain_name.len(), false, false);
1083            }
1084        }
1085    }
1086
1087    /// The part of `process` that doesn't need to be generic over the sink and
1088    /// can avoid monomorphizing in the interest of code size.
1089    /// Separating this into a different stack frame compared to `process_inner`
1090    /// improves performance in the ICU4X case.
1091    #[allow(clippy::too_many_arguments)]
1092    #[inline(never)]
1093    fn process_innermost<'a>(
1094        &self,
1095        domain_name: &'a [u8],
1096        ascii_deny_list: AsciiDenyList,
1097        hyphens: Hyphens,
1098        fail_fast: bool,
1099        domain_buffer: &mut SmallVec<[char; 253]>,
1100        already_punycode: &mut SmallVec<[AlreadyAsciiLabel<'a>; 8]>,
1101        tail: &'a [u8],
1102    ) -> (usize, bool, bool) {
1103        let deny_list = ascii_deny_list.bits;
1104        let deny_list_deny_dot = deny_list | DOT_MASK;
1105
1106        let mut had_errors = false;
1107
1108        let mut passthrough_up_to = domain_name.len() - tail.len(); // Index into `domain_name`
1109                                                                    // 253 ASCII characters is the max length for a valid domain name
1110                                                                    // (excluding the root dot).
1111        let mut current_label_start; // Index into `domain_buffer`
1112        let mut seen_label = false;
1113        let mut in_prefix = true;
1114        for label in tail.split(|b| *b == b'.') {
1115            // We check for passthrough only for the prefix. That is, if we
1116            // haven't moved on and started filling `domain_buffer`. Keeping
1117            // this stuff in one loop where the first items keep being skipped
1118            // once they have been skipped at least once instead of working
1119            // this into a fancier loop structure in order to make sure that
1120            // no item from the iterator is lost or processed twice.
1121            // Furthermore, after the passthrough fails, restarting the
1122            // normalization process after each pre-existing ASCII dot also
1123            // provides an opportunity for the processing to get back onto
1124            // an ASCII fast path that bypasses the normalizer for ASCII
1125            // after a pre-existing ASCII dot (pre-existing in the sense
1126            // of not coming from e.g. normalizing an ideographic dot).
1127            if in_prefix && is_passthrough_ascii_label(label) {
1128                if seen_label {
1129                    debug_assert_eq!(domain_name[passthrough_up_to], b'.');
1130                    passthrough_up_to += 1;
1131                }
1132                seen_label = true;
1133
1134                passthrough_up_to += label.len();
1135                continue;
1136            }
1137            if seen_label {
1138                if in_prefix {
1139                    debug_assert_eq!(domain_name[passthrough_up_to], b'.');
1140                    passthrough_up_to += 1;
1141                } else {
1142                    domain_buffer.push('.');
1143                }
1144            }
1145            seen_label = true;
1146            in_prefix = false;
1147            current_label_start = domain_buffer.len();
1148            if !label.is_empty() {
1149                let (ascii, non_ascii) = split_ascii_fast_path_prefix(label);
1150                let non_punycode_ascii_label = if non_ascii.is_empty() {
1151                    if has_punycode_prefix(ascii) {
1152                        if (ascii.last() != Some(&b'-'))
1153                            && (ascii.len() - 4 <= PUNYCODE_DECODE_MAX_INPUT_LENGTH)
1154                        {
1155                            if let Ok(decode) =
1156                                Decoder::default().decode::<u8, InternalCaller>(&ascii[4..])
1157                            {
1158                                // 63 ASCII characters is the max length for a valid DNS label and xn-- takes 4
1159                                // characters.
1160                                let mut label_buffer = SmallVec::<[char; 59]>::new();
1161                                label_buffer.extend(decode);
1162
1163                                if self.after_punycode_decode(
1164                                    domain_buffer,
1165                                    current_label_start,
1166                                    &label_buffer,
1167                                    deny_list_deny_dot,
1168                                    fail_fast,
1169                                    &mut had_errors,
1170                                ) {
1171                                    return (0, false, true);
1172                                }
1173
1174                                if self.check_label(
1175                                    hyphens,
1176                                    &mut domain_buffer[current_label_start..],
1177                                    fail_fast,
1178                                    &mut had_errors,
1179                                    true,
1180                                    true,
1181                                ) {
1182                                    return (0, false, true);
1183                                }
1184                            } else {
1185                                // Punycode failed
1186                                if fail_fast {
1187                                    return (0, false, true);
1188                                }
1189                                had_errors = true;
1190                                domain_buffer.push('\u{FFFD}');
1191                                let mut iter = ascii.iter();
1192                                // Discard the first character that we replaced.
1193                                let _ = iter.next();
1194                                domain_buffer.extend(iter.map(|c| {
1195                                    // Can't have dot here, so `deny_list` vs `deny_list_deny_dot` does
1196                                    // not matter.
1197                                    apply_ascii_deny_list_to_potentially_upper_case_ascii(
1198                                        *c, deny_list,
1199                                    )
1200                                }));
1201                            };
1202                            // If there were errors, we won't be trying to use this
1203                            // anyway later, so it's fine to put it here unconditionally.
1204                            already_punycode.push(AlreadyAsciiLabel::MixedCasePunycode(label));
1205                            continue;
1206                        } else if fail_fast {
1207                            return (0, false, true);
1208                        }
1209                        // Else fall through to the complex path and rediscover error
1210                        // there.
1211                        false
1212                    } else {
1213                        true
1214                    }
1215                } else {
1216                    false
1217                };
1218                for c in ascii.iter().map(|c| {
1219                    // Can't have dot here, so `deny_list` vs `deny_list_deny_dot` does
1220                    // not matter.
1221                    apply_ascii_deny_list_to_potentially_upper_case_ascii(*c, deny_list)
1222                }) {
1223                    if c == '\u{FFFD}' {
1224                        if fail_fast {
1225                            return (0, false, true);
1226                        }
1227                        had_errors = true;
1228                    }
1229                    domain_buffer.push(c);
1230                }
1231                if non_punycode_ascii_label {
1232                    if hyphens != Hyphens::Allow
1233                        && check_hyphens(
1234                            &mut domain_buffer[current_label_start..],
1235                            hyphens == Hyphens::CheckFirstLast,
1236                            fail_fast,
1237                            &mut had_errors,
1238                        )
1239                    {
1240                        return (0, false, true);
1241                    }
1242                    already_punycode.push(if had_errors {
1243                        AlreadyAsciiLabel::Other
1244                    } else {
1245                        AlreadyAsciiLabel::MixedCaseAscii(label)
1246                    });
1247                    continue;
1248                }
1249                already_punycode.push(AlreadyAsciiLabel::Other);
1250                let mut first_needs_combining_mark_check = ascii.is_empty();
1251                let mut needs_contextj_check = !non_ascii.is_empty();
1252                let mut mapping = self
1253                    .data
1254                    .map_normalize(non_ascii.chars())
1255                    .map(|c| apply_ascii_deny_list_to_lower_cased_unicode(c, deny_list));
1256                loop {
1257                    let n = mapping.next();
1258                    match n {
1259                        None | Some('.') => {
1260                            if domain_buffer[current_label_start..]
1261                                .starts_with(&['x', 'n', '-', '-'])
1262                            {
1263                                let mut punycode_precondition_failed = false;
1264                                for c in domain_buffer[current_label_start + 4..].iter_mut() {
1265                                    if !c.is_ascii() {
1266                                        if fail_fast {
1267                                            return (0, false, true);
1268                                        }
1269                                        had_errors = true;
1270                                        *c = '\u{FFFD}';
1271                                        punycode_precondition_failed = true;
1272                                    }
1273                                }
1274
1275                                if let Some(last) = domain_buffer.last_mut() {
1276                                    if *last == '-' {
1277                                        // Either there's nothing after the "xn--" prefix
1278                                        // and we got the last hyphen of "xn--", or there
1279                                        // are no Punycode digits after the last delimiter
1280                                        // which would result in Punycode decode outputting
1281                                        // ASCII only.
1282                                        if fail_fast {
1283                                            return (0, false, true);
1284                                        }
1285                                        had_errors = true;
1286                                        *last = '\u{FFFD}';
1287                                        punycode_precondition_failed = true;
1288                                    }
1289                                } else {
1290                                    unreachable!();
1291                                }
1292
1293                                // Reject excessively long input
1294                                // https://github.com/whatwg/url/issues/824
1295                                // https://unicode-org.atlassian.net/browse/ICU-13727
1296                                if domain_buffer.len() - current_label_start - 4
1297                                    > PUNYCODE_DECODE_MAX_INPUT_LENGTH
1298                                {
1299                                    if fail_fast {
1300                                        return (0, false, true);
1301                                    }
1302                                    had_errors = true;
1303                                    domain_buffer[current_label_start
1304                                        + 4
1305                                        + PUNYCODE_DECODE_MAX_INPUT_LENGTH] = '\u{FFFD}';
1306                                    punycode_precondition_failed = true;
1307                                }
1308
1309                                if !punycode_precondition_failed {
1310                                    if let Ok(decode) = Decoder::default()
1311                                        .decode::<char, InternalCaller>(
1312                                            &domain_buffer[current_label_start + 4..],
1313                                        )
1314                                    {
1315                                        first_needs_combining_mark_check = true;
1316                                        needs_contextj_check = true;
1317                                        // 63 ASCII characters is the max length for a valid DNS label and xn-- takes 4
1318                                        // characters.
1319                                        let mut label_buffer = SmallVec::<[char; 59]>::new();
1320                                        label_buffer.extend(decode);
1321
1322                                        domain_buffer.truncate(current_label_start);
1323                                        if self.after_punycode_decode(
1324                                            domain_buffer,
1325                                            current_label_start,
1326                                            &label_buffer,
1327                                            deny_list_deny_dot,
1328                                            fail_fast,
1329                                            &mut had_errors,
1330                                        ) {
1331                                            return (0, false, true);
1332                                        }
1333                                    } else {
1334                                        // Punycode failed
1335                                        if fail_fast {
1336                                            return (0, false, true);
1337                                        }
1338                                        had_errors = true;
1339                                        domain_buffer[current_label_start] = '\u{FFFD}';
1340                                        needs_contextj_check = false; // ASCII label
1341                                        first_needs_combining_mark_check = false;
1342                                    };
1343                                } else {
1344                                    first_needs_combining_mark_check = false;
1345                                    needs_contextj_check = false; // Non-ASCII already turned to U+FFFD.
1346                                }
1347                            }
1348                            if self.check_label(
1349                                hyphens,
1350                                &mut domain_buffer[current_label_start..],
1351                                fail_fast,
1352                                &mut had_errors,
1353                                first_needs_combining_mark_check,
1354                                needs_contextj_check,
1355                            ) {
1356                                return (0, false, true);
1357                            }
1358
1359                            if n.is_none() {
1360                                break;
1361                            }
1362                            domain_buffer.push('.');
1363                            current_label_start = domain_buffer.len();
1364                            first_needs_combining_mark_check = true;
1365                            needs_contextj_check = true;
1366                            already_punycode.push(AlreadyAsciiLabel::Other);
1367                        }
1368                        Some(c) => {
1369                            if c == '\u{FFFD}' {
1370                                if fail_fast {
1371                                    return (0, false, true);
1372                                }
1373                                had_errors = true;
1374                            }
1375                            domain_buffer.push(c);
1376                        }
1377                    }
1378                }
1379            } else {
1380                // Empty label
1381                already_punycode.push(AlreadyAsciiLabel::MixedCaseAscii(label));
1382            }
1383        }
1384
1385        let is_bidi = self.is_bidi(domain_buffer);
1386        if is_bidi {
1387            for label in domain_buffer.split_mut(|c| *c == '.') {
1388                if let Some((first, tail)) = label.split_first_mut() {
1389                    let first_bc = self.data.bidi_class(*first);
1390                    if !FIRST_BC_MASK.intersects(first_bc.to_mask()) {
1391                        // Neither RTL label nor LTR label
1392                        if fail_fast {
1393                            return (0, false, true);
1394                        }
1395                        had_errors = true;
1396                        *first = '\u{FFFD}';
1397                        continue;
1398                    }
1399                    let is_ltr = first_bc.is_ltr();
1400                    // Trim NSM
1401                    let mut middle = tail;
1402                    #[allow(clippy::while_let_loop)]
1403                    loop {
1404                        if let Some((last, prior)) = middle.split_last_mut() {
1405                            let last_bc = self.data.bidi_class(*last);
1406                            if last_bc.is_nonspacing_mark() {
1407                                middle = prior;
1408                                continue;
1409                            }
1410                            let last_mask = if is_ltr { LAST_LTR_MASK } else { LAST_RTL_MASK };
1411                            if !last_mask.intersects(last_bc.to_mask()) {
1412                                if fail_fast {
1413                                    return (0, false, true);
1414                                }
1415                                had_errors = true;
1416                                *last = '\u{FFFD}';
1417                            }
1418                            if is_ltr {
1419                                for c in prior.iter_mut() {
1420                                    let bc = self.data.bidi_class(*c);
1421                                    if !MIDDLE_LTR_MASK.intersects(bc.to_mask()) {
1422                                        if fail_fast {
1423                                            return (0, false, true);
1424                                        }
1425                                        had_errors = true;
1426                                        *c = '\u{FFFD}';
1427                                    }
1428                                }
1429                            } else {
1430                                let mut numeral_state = RtlNumeralState::Undecided;
1431                                for c in prior.iter_mut() {
1432                                    let bc = self.data.bidi_class(*c);
1433                                    if !MIDDLE_RTL_MASK.intersects(bc.to_mask()) {
1434                                        if fail_fast {
1435                                            return (0, false, true);
1436                                        }
1437                                        had_errors = true;
1438                                        *c = '\u{FFFD}';
1439                                    } else {
1440                                        match numeral_state {
1441                                            RtlNumeralState::Undecided => {
1442                                                if bc.is_european_number() {
1443                                                    numeral_state = RtlNumeralState::European;
1444                                                } else if bc.is_arabic_number() {
1445                                                    numeral_state = RtlNumeralState::Arabic;
1446                                                }
1447                                            }
1448                                            RtlNumeralState::European => {
1449                                                if bc.is_arabic_number() {
1450                                                    if fail_fast {
1451                                                        return (0, false, true);
1452                                                    }
1453                                                    had_errors = true;
1454                                                    *c = '\u{FFFD}';
1455                                                }
1456                                            }
1457                                            RtlNumeralState::Arabic => {
1458                                                if bc.is_european_number() {
1459                                                    if fail_fast {
1460                                                        return (0, false, true);
1461                                                    }
1462                                                    had_errors = true;
1463                                                    *c = '\u{FFFD}';
1464                                                }
1465                                            }
1466                                        }
1467                                    }
1468                                }
1469                                if (numeral_state == RtlNumeralState::European
1470                                    && last_bc.is_arabic_number())
1471                                    || (numeral_state == RtlNumeralState::Arabic
1472                                        && last_bc.is_european_number())
1473                                {
1474                                    if fail_fast {
1475                                        return (0, false, true);
1476                                    }
1477                                    had_errors = true;
1478                                    *last = '\u{FFFD}';
1479                                }
1480                            }
1481                            break;
1482                        } else {
1483                            // One-character label or label where
1484                            // everything after the first character
1485                            // is just non-spacing marks.
1486                            break;
1487                        }
1488                    }
1489                }
1490            }
1491        }
1492
1493        (passthrough_up_to, is_bidi, had_errors)
1494    }
1495
1496    #[inline(never)]
1497    fn after_punycode_decode(
1498        &self,
1499        domain_buffer: &mut SmallVec<[char; 253]>,
1500        current_label_start: usize,
1501        label_buffer: &[char],
1502        deny_list_deny_dot: u128,
1503        fail_fast: bool,
1504        had_errors: &mut bool,
1505    ) -> bool {
1506        for c in self
1507            .data
1508            .normalize_validate(label_buffer.iter().copied())
1509            .map(|c| apply_ascii_deny_list_to_lower_cased_unicode(c, deny_list_deny_dot))
1510        {
1511            if c == '\u{FFFD}' {
1512                if fail_fast {
1513                    return true;
1514                }
1515                *had_errors = true;
1516            }
1517            domain_buffer.push(c);
1518        }
1519        let normalized = &mut domain_buffer[current_label_start..];
1520        if let Err(()) =
1521            normalized
1522                .iter_mut()
1523                .zip(label_buffer.iter())
1524                .try_for_each(|(norm_c, decoded_c)| {
1525                    if *norm_c == *decoded_c {
1526                        Ok(())
1527                    } else {
1528                        // Mark the first difference
1529                        *norm_c = '\u{FFFD}';
1530                        Err(())
1531                    }
1532                })
1533        {
1534            if fail_fast {
1535                return true;
1536            }
1537            *had_errors = true;
1538        }
1539        false
1540    }
1541
1542    #[inline(never)]
1543    fn check_label(
1544        &self,
1545        hyphens: Hyphens,
1546        mut_label: &mut [char],
1547        fail_fast: bool,
1548        had_errors: &mut bool,
1549        first_needs_combining_mark_check: bool,
1550        needs_contextj_check: bool,
1551    ) -> bool {
1552        if hyphens != Hyphens::Allow
1553            && check_hyphens(
1554                mut_label,
1555                hyphens == Hyphens::CheckFirstLast,
1556                fail_fast,
1557                had_errors,
1558            )
1559        {
1560            return true;
1561        }
1562        if first_needs_combining_mark_check {
1563            if let Some(first) = mut_label.first_mut() {
1564                if self.data.is_mark(*first) {
1565                    if fail_fast {
1566                        return true;
1567                    }
1568                    *had_errors = true;
1569                    *first = '\u{FFFD}';
1570                }
1571            }
1572        }
1573        if needs_contextj_check {
1574            // ContextJ
1575            for i in 0..mut_label.len() {
1576                let c = mut_label[i];
1577                if !in_inclusive_range_char(c, '\u{200C}', '\u{200D}') {
1578                    continue;
1579                }
1580                let (head, joiner_and_tail) = mut_label.split_at_mut(i);
1581
1582                if let Some((joiner, tail)) = joiner_and_tail.split_first_mut() {
1583                    if let Some(previous) = head.last() {
1584                        if self.data.is_virama(*previous) {
1585                            continue;
1586                        }
1587                    } else {
1588                        // No preceding character
1589                        if fail_fast {
1590                            return true;
1591                        }
1592                        *had_errors = true;
1593                        *joiner = '\u{FFFD}';
1594                        continue;
1595                    }
1596                    if c == '\u{200D}' {
1597                        // ZWJ only has the virama rule
1598                        if fail_fast {
1599                            return true;
1600                        }
1601                        *had_errors = true;
1602                        *joiner = '\u{FFFD}';
1603                        continue;
1604                    }
1605                    debug_assert_eq!(c, '\u{200C}');
1606                    if !self.has_appropriately_joining_char(
1607                        head.iter().rev().copied(),
1608                        LEFT_OR_DUAL_JOINING_MASK,
1609                    ) || !self.has_appropriately_joining_char(
1610                        tail.iter().copied(),
1611                        RIGHT_OR_DUAL_JOINING_MASK,
1612                    ) {
1613                        if fail_fast {
1614                            return true;
1615                        }
1616                        *had_errors = true;
1617                        *joiner = '\u{FFFD}';
1618                    }
1619                } else {
1620                    debug_assert!(false);
1621                }
1622            }
1623        }
1624
1625        if !is_ascii(mut_label) && mut_label.len() > PUNYCODE_ENCODE_MAX_INPUT_LENGTH {
1626            // Limit quadratic behavior
1627            // https://github.com/whatwg/url/issues/824
1628            // https://unicode-org.atlassian.net/browse/ICU-13727
1629            if fail_fast {
1630                return true;
1631            }
1632            *had_errors = true;
1633            mut_label[PUNYCODE_ENCODE_MAX_INPUT_LENGTH] = '\u{FFFD}';
1634        }
1635        false
1636    }
1637
1638    #[inline(always)]
1639    fn has_appropriately_joining_char<I: Iterator<Item = char>>(
1640        &self,
1641        iter: I,
1642        required_mask: JoiningTypeMask,
1643    ) -> bool {
1644        for c in iter {
1645            let jt = self.data.joining_type(c);
1646            if jt.to_mask().intersects(required_mask) {
1647                return true;
1648            }
1649            if jt.is_transparent() {
1650                continue;
1651            }
1652            return false;
1653        }
1654        false
1655    }
1656
1657    #[inline(always)]
1658    fn is_bidi(&self, buffer: &[char]) -> bool {
1659        for &c in buffer {
1660            if c < '\u{0590}' {
1661                // Below Hebrew
1662                continue;
1663            }
1664            if in_inclusive_range_char(c, '\u{0900}', '\u{FB1C}') {
1665                debug_assert_ne!(c, '\u{200F}'); // disallowed
1666                continue;
1667            }
1668            if in_inclusive_range_char(c, '\u{1F000}', '\u{3FFFF}') {
1669                continue;
1670            }
1671            if in_inclusive_range_char(c, '\u{FF00}', '\u{107FF}') {
1672                continue;
1673            }
1674            if in_inclusive_range_char(c, '\u{11000}', '\u{1E7FF}') {
1675                continue;
1676            }
1677            if RTL_MASK.intersects(self.data.bidi_class(c).to_mask()) {
1678                return true;
1679            }
1680        }
1681        false
1682    }
1683}
1684
1685fn check_hyphens(
1686    mut_label: &mut [char],
1687    allow_third_fourth: bool,
1688    fail_fast: bool,
1689    had_errors: &mut bool,
1690) -> bool {
1691    if let Some(first) = mut_label.first_mut() {
1692        if *first == '-' {
1693            if fail_fast {
1694                return true;
1695            }
1696            *had_errors = true;
1697            *first = '\u{FFFD}';
1698        }
1699    }
1700    if let Some(last) = mut_label.last_mut() {
1701        if *last == '-' {
1702            if fail_fast {
1703                return true;
1704            }
1705            *had_errors = true;
1706            *last = '\u{FFFD}';
1707        }
1708    }
1709    if allow_third_fourth {
1710        return false;
1711    }
1712    if mut_label.len() >= 4 && mut_label[2] == '-' && mut_label[3] == '-' {
1713        if fail_fast {
1714            return true;
1715        }
1716        *had_errors = true;
1717        mut_label[2] = '\u{FFFD}';
1718        mut_label[3] = '\u{FFFD}';
1719    }
1720    false
1721}