icu_collator/

provider.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

// The reordering algorithms in this file are adapted from ICU4C and,
// therefore, are subject to the ICU license as described in LICENSE.

//! 🚧 \[Unstable\] Data provider struct definitions for this ICU4X component.
//!
//! <div class="stab unstable">
//! 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
//! including in SemVer minor releases. While the serde representation of data structs is guaranteed
//! to be stable, their Rust representation might not be. Use with caution.
//! </div>
//!
//! Read more about data providers: [`icu_provider`]

// Provider structs must be stable
#![allow(clippy::exhaustive_structs, clippy::exhaustive_enums)]

use icu_collections::char16trie::Char16TrieIterator;
use icu_collections::codepointtrie::SmallCodePointTrie;
use icu_collections::codepointtrie::TypedCodePointTrie;
use icu_provider::prelude::*;
use zerovec::ule::AsULE;
use zerovec::ZeroVec;
use zerovec::{zeroslice, ZeroSlice};

use crate::elements::CollationElement;
use crate::elements::CollationElement32;
use crate::elements::Tag;
use crate::elements::EMPTY_U16;
use crate::elements::FFFD_CE;
use crate::elements::FFFD_CE32;
use crate::elements::FFFD_CE32_VALUE;
use crate::elements::FFFD_CE_VALUE;
use crate::elements::FFFD_PRIMARY;
use crate::elements::NO_CE_PRIMARY;
use crate::preferences::CollationCaseFirst;

use crate::options::MaxVariable;

#[cfg(feature = "compiled_data")]
#[derive(Debug)]
/// Baked data
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. In particular, the `DataProvider` implementations are only
/// guaranteed to match with this version's `*_unstable` providers. Use with caution.
/// </div>
pub struct Baked;

#[cfg(feature = "compiled_data")]
#[allow(unused_imports)]
const _: () = {
    use icu_collator_data::*;
    pub mod icu {
        pub use crate as collator;
        pub use icu_collections as collections;
        pub use icu_locale as locale;
    }
    make_provider!(Baked);
    impl_collation_root_v1!(Baked);
    impl_collation_tailoring_v1!(Baked);
    impl_collation_diacritics_v1!(Baked);
    impl_collation_jamo_v1!(Baked);
    impl_collation_metadata_v1!(Baked);
    impl_collation_special_primaries_v1!(Baked);
    impl_collation_reordering_v1!(Baked);
};

const SCRIPT_FALLBACK: icu_provider::fallback::LocaleFallbackConfig = {
    let mut c = icu_provider::fallback::LocaleFallbackConfig::default();
    c.priority = icu_provider::fallback::LocaleFallbackPriority::Script;
    c
};

icu_provider::data_marker!(
    /// Data marker for singleton root collation data.
    CollationRootV1,
    "collation/root/v1",
    CollationData<'static>,
    is_singleton = true,
);
icu_provider::data_marker!(
    /// Data marker for collation tailorings.
    CollationTailoringV1,
    "collation/tailoring/v1",
    CollationData<'static>,
    fallback_config = SCRIPT_FALLBACK,
    #[cfg(feature = "datagen")]
    attributes_domain = "collator",
);
icu_provider::data_marker!(
    /// Data marker for collation diacritics data.
    CollationDiacriticsV1,
    "collation/diacritics/v1",
    CollationDiacritics<'static>,
    fallback_config = SCRIPT_FALLBACK,
    #[cfg(feature = "datagen")]
    attributes_domain = "collator",
);
icu_provider::data_marker!(
    /// Data marker for collation jamo data.
    CollationJamoV1,
    "collation/jamo/v1",
    CollationJamo<'static>,
    is_singleton = true,
);
icu_provider::data_marker!(
    /// Data marker for collation reordering data.
    CollationReorderingV1,
    "collation/reordering/v1",
    CollationReordering<'static>,
    fallback_config = SCRIPT_FALLBACK,
    #[cfg(feature = "datagen")]
    attributes_domain = "collator",
);
icu_provider::data_marker!(
    /// Data marker for collation metadata.
    CollationMetadataV1,
    "collation/metadata/v1",
    CollationMetadata,
    fallback_config = SCRIPT_FALLBACK,
    #[cfg(feature = "datagen")]
    attributes_domain = "collator",
);
icu_provider::data_marker!(
    /// Data marker for collcation special primaries data.
    CollationSpecialPrimariesV1,
    "collation/special/primaries/v1",
    CollationSpecialPrimaries<'static>,
    is_singleton = true,
);

#[cfg(feature = "datagen")]
/// The latest minimum set of markers required by this component.
pub const MARKERS: &[DataMarkerInfo] = &[
    CollationRootV1::INFO,
    CollationTailoringV1::INFO,
    CollationDiacriticsV1::INFO,
    CollationJamoV1::INFO,
    CollationMetadataV1::INFO,
    CollationReorderingV1::INFO,
    CollationSpecialPrimariesV1::INFO,
];

const SINGLE_U32: &ZeroSlice<u32> =
    zeroslice!(u32; <u32 as AsULE>::ULE::from_unsigned; [FFFD_CE32_VALUE]);
const SINGLE_U64: &ZeroSlice<u64> =
    zeroslice!(u64; <u64 as AsULE>::ULE::from_unsigned; [FFFD_CE_VALUE]);

fn data_ce_to_primary(data_ce: u64, c: char) -> u32 {
    // Collation::getThreeBytePrimaryForOffsetData
    let p = (data_ce >> 32) as u32; // three-byte primary pppppp00
    let lower32 = data_ce as u32 as i32; // base code point b & step s: bbbbbbss (bit 7: isCompressible)
    let mut offset = ((u32::from(c) as i32) - (lower32 >> 8)) * (lower32 & 0x7F); // delta * increment
    let is_compressible = (lower32 & 0x80) != 0;
    // Collation::incThreeBytePrimaryByOffset
    offset += (((p >> 8) & 0xFF) as i32) - 2;
    let mut primary = (((offset % 254) + 2) as u32) << 8;
    offset /= 254;
    // Same with the second byte,
    // but reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary.
    if is_compressible {
        offset += (((p >> 16) & 0xFF) as i32) - 4;
        primary |= (((offset % 251) + 4) as u32) << 16;
        offset /= 251;
    } else {
        offset += (((p >> 16) & 0xFF) as i32) - 2;
        primary |= (((offset % 254) + 2) as u32) << 16;
        offset /= 254;
    }
    primary | ((p & 0xFF000000) + ((offset as u32) << 24))
}

/// The main collation data either for the root or for a tailoring
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
/// to be stable, their Rust representation might not be. Use with caution.
/// </div>
#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
#[cfg_attr(feature = "datagen", databake(path = icu_collator::provider))]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub struct CollationData<'data> {
    /// Mapping from `char` to `CollationElement32` (represented
    /// as its `u32` bits).
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub trie: SmallCodePointTrie<'data, u32>,
    /// `CollationElement`s used in expansions and offset CE32s
    /// (represented as their `u64` bits)
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub ces: ZeroVec<'data, u64>,
    /// `CollationElement32`s used in expansions and as defaults
    /// for digits when the numeric mode is not in use
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub ce32s: ZeroVec<'data, u32>,
    /// Defaults and tries for prefix and contraction matching
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub contexts: ZeroVec<'data, u16>,
}

icu_provider::data_struct!(
    CollationData<'_>,
    #[cfg(feature = "datagen")]
);

impl<'data> CollationData<'data> {
    pub(crate) fn ce32_for_char(&self, c: char) -> CollationElement32 {
        CollationElement32::new(self.trie.get(c))
    }
    pub(crate) fn get_ce32(&'data self, index: usize) -> CollationElement32 {
        CollationElement32::new(if let Some(u) = self.ce32s.get(index) {
            u
        } else {
            // GIGO case
            debug_assert!(false);
            FFFD_CE32_VALUE
        })
    }
    pub(crate) fn get_ce32s(&'data self, index: usize, len: usize) -> &'data ZeroSlice<u32> {
        if len > 0 {
            if let Some(slice) = self.ce32s.get_subslice(index..index + len) {
                return slice;
            }
        }
        // GIGO case
        debug_assert!(false);
        SINGLE_U32
    }
    pub(crate) fn get_ces(&'data self, index: usize, len: usize) -> &'data ZeroSlice<u64> {
        if len > 0 {
            if let Some(slice) = self.ces.get_subslice(index..index + len) {
                return slice;
            }
        }
        // GIGO case
        debug_assert!(false);
        SINGLE_U64
    }
    pub(crate) fn get_primary_from_ces(&'data self, index: usize) -> u32 {
        if let Some(ce) = self.ces.get(index) {
            (ce >> 32) as u32
        } else {
            // GIGO case
            debug_assert!(false);
            FFFD_PRIMARY
        }
    }
    fn get_default_and_trie_impl(
        &'data self,
        index: usize,
    ) -> (CollationElement32, &'data ZeroSlice<u16>) {
        if let Some(slice) = self.contexts.get_subslice(index..self.contexts.len()) {
            #[expect(clippy::unwrap_used)]
            if slice.len() >= 2 {
                // `unwrap` must succeed due to the length check above.
                let first = slice.get(0).unwrap();
                let second = slice.get(1).unwrap();
                let trie = slice.get_subslice(2..slice.len()).unwrap();
                return (
                    CollationElement32::new((u32::from(first) << 16) | u32::from(second)),
                    trie,
                );
            }
        }
        // GIGO case
        debug_assert!(false);
        (FFFD_CE32, EMPTY_U16)
    }
    pub(crate) fn get_default_and_trie(
        &'data self,
        index: usize,
    ) -> (CollationElement32, Char16TrieIterator<'data>) {
        let (ce32, trie) = self.get_default_and_trie_impl(index);
        (ce32, Char16TrieIterator::new(trie))
    }
    pub(crate) fn get_default(&'data self, index: usize) -> CollationElement32 {
        let (ce32, _) = self.get_default_and_trie_impl(index);
        ce32
    }
    pub(crate) fn ce_from_offset_ce32(
        &self,
        c: char,
        ce32: CollationElement32,
    ) -> CollationElement {
        debug_assert!(ce32.tag() == Tag::Offset);
        if let Some(data_ce) = self.ces.get(ce32.index()) {
            CollationElement::new_from_primary(data_ce_to_primary(data_ce, c))
        } else {
            // GIGO case
            debug_assert!(false);
            FFFD_CE
        }
    }
}

/// Secondary weights for the start of the Combining Diacritics block.
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
/// to be stable, their Rust representation might not be. Use with caution.
/// </div>
#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
#[cfg_attr(feature = "datagen", databake(path = icu_collator::provider))]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub struct CollationDiacritics<'data> {
    /// Secondary weights for characters starting from U+0300 up
    /// to but not including U+034F. May be shorter than that;
    /// zero-length when a tailoring opts out of using this
    /// feature altogether.
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub secondaries: ZeroVec<'data, u16>,
}

icu_provider::data_struct!(
    CollationDiacritics<'_>,
    #[cfg(feature = "datagen")]
);

/// `CollationElement32`s for the Hangul Jamo Unicode Block
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
/// to be stable, their Rust representation might not be. Use with caution.
/// </div>
#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
#[cfg_attr(feature = "datagen", databake(path = icu_collator::provider))]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub struct CollationJamo<'data> {
    /// `CollationElement32`s (as `u32`s) for the Hangul Jamo Unicode Block.
    /// The length must be equal to the size of the block (256).
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub ce32s: ZeroVec<'data, u32>,
}

icu_provider::data_struct!(
    CollationJamo<'_>,
    #[cfg(feature = "datagen")]
);

/// Script reordering data
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
/// to be stable, their Rust representation might not be. Use with caution.
/// </div>
#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
#[cfg_attr(feature = "datagen", databake(path = icu_collator::provider))]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub struct CollationReordering<'data> {
    /// Limit of last reordered range. 0 if no reordering or no split bytes.
    ///
    /// Comment from ICU4C's `collationsettings.h`
    pub min_high_no_reorder: u32,
    /// 256-byte table for reordering permutation of primary lead
    /// bytes; NULL if no reordering. A 0 entry at a non-zero index means
    /// that the primary lead byte is "split" (there are different offsets
    /// for primaries that share that lead byte) and the reordering offset
    /// must be determined via the reorderRanges.
    ///
    /// Comment from ICU4C's `collationsettings.h`
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub reorder_table: ZeroVec<'data, u8>, // len always 256
    /// Primary-weight ranges for script reordering, to be used by
    /// reorder(p) for split-reordered primary lead bytes.
    ///
    /// Each entry is a (limit, offset) pair. The upper 16 bits of the
    /// entry are the upper 16 bits of the exclusive primary limit of
    /// a range. Primaries between the previous limit and this one have
    /// their lead bytes modified by the signed offset (-0xff..+0xff)
    /// stored in the lower 16 bits.
    ///
    /// `CollationData::makeReorderRanges()` writes a full list where the
    /// first range (at least for terminators and separators) has a 0
    /// offset. The last range has a non-zero offset. minHighNoReorder
    /// is set to the limit of that last range.
    ///
    /// In the settings object, the initial ranges before the first
    /// split lead byte are omitted for efficiency; they are handled
    /// by reorder(p) via the reorderTable. If there are no
    /// split-reordered lead bytes, then no ranges are needed.
    ///
    /// Comment from ICU4C's `collationsettings.h`; names refer to
    /// ICU4C.
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub reorder_ranges: ZeroVec<'data, u32>,
}

icu_provider::data_struct!(
    CollationReordering<'_>,
    #[cfg(feature = "datagen")]
);

impl CollationReordering<'_> {
    pub(crate) fn reorder(&self, primary: u32) -> u32 {
        if let Some(b) = self.reorder_table.get((primary >> 24) as usize) {
            if b != 0 || primary <= NO_CE_PRIMARY {
                (u32::from(b) << 24) | (primary & 0x00FFFFFF)
            } else {
                self.reorder_ex(primary)
            }
        } else {
            // GIGO case
            debug_assert!(false);
            primary
        }
    }

    fn reorder_ex(&self, primary: u32) -> u32 {
        if primary >= self.min_high_no_reorder {
            return primary;
        }
        let q = primary | 0xFFFF;
        for &range in self.reorder_ranges.as_ule_slice().iter() {
            let r = u32::from_unaligned(range);
            if q < r {
                return primary.wrapping_add(r << 24);
            }
        }
        // GIGO case
        debug_assert!(false);
        primary
    }
}

/// Each non-alias collation that the data provider knows
/// about explicitly has an data entry at least for this
/// struct.
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
/// to be stable, their Rust representation might not be. Use with caution.
/// </div>
#[derive(Debug, PartialEq, Clone, Copy, yoke::Yokeable, zerofrom::ZeroFrom)]
#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
#[cfg_attr(feature = "datagen", databake(path = icu_collator::provider))]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub struct CollationMetadata {
    /// See the mask constants in the `impl` block for the
    /// bit layout. The other bits are ignored: They could
    /// be from the future if their semantics such that
    /// old code may ignore them.
    ///
    /// Note: At present, it's bogus for the bit for "upper
    /// first" to be set if "case first" isn't also set.
    /// However, the methods handle this case gracefully,
    /// so there is no need for invariant validation.
    pub bits: u32,
}

icu_provider::data_struct!(
    CollationMetadata,
    #[cfg(feature = "datagen")]
);

impl CollationMetadata {
    const MAX_VARIABLE_MASK: u32 = 0b11;
    const TAILORED_MASK: u32 = 1 << 3;
    const TAILORED_DIACRITICS_MASK: u32 = 1 << 4;
    const REORDERING_MASK: u32 = 1 << 5;
    const LITHUANIAN_DOT_ABOVE_MASK: u32 = 1 << 6;
    const BACWARD_SECOND_LEVEL_MASK: u32 = 1 << 7;
    const ALTERNATE_SHIFTED_MASK: u32 = 1 << 8;
    const CASE_FIRST_MASK: u32 = 1 << 9;
    const UPPER_FIRST_MASK: u32 = 1 << 10;

    #[inline(always)]
    pub(crate) fn max_variable(self) -> MaxVariable {
        // Safety: the possible numeric values for `MaxVariable` are from 0 to 3, inclusive,
        // and it is repr(u8). MAX_VARIABLE_MASK here ensures our values have most 2 bits, which produces
        // the same range.
        unsafe { core::mem::transmute((self.bits & CollationMetadata::MAX_VARIABLE_MASK) as u8) }
    }

    #[inline(always)]
    pub(crate) fn tailored(self) -> bool {
        self.bits & CollationMetadata::TAILORED_MASK != 0
    }

    /// Vietnamese and Ewe
    #[inline(always)]
    pub(crate) fn tailored_diacritics(self) -> bool {
        self.bits & CollationMetadata::TAILORED_DIACRITICS_MASK != 0
    }

    /// Lithuanian
    #[inline(always)]
    pub(crate) fn lithuanian_dot_above(self) -> bool {
        self.bits & CollationMetadata::LITHUANIAN_DOT_ABOVE_MASK != 0
    }

    /// Canadian French
    #[inline(always)]
    pub(crate) fn backward_second_level(self) -> bool {
        self.bits & CollationMetadata::BACWARD_SECOND_LEVEL_MASK != 0
    }

    #[inline(always)]
    pub(crate) fn reordering(self) -> bool {
        self.bits & CollationMetadata::REORDERING_MASK != 0
    }

    /// Thai
    #[inline(always)]
    pub(crate) fn alternate_shifted(self) -> bool {
        self.bits & CollationMetadata::ALTERNATE_SHIFTED_MASK != 0
    }

    #[inline(always)]
    pub(crate) fn case_first(self) -> CollationCaseFirst {
        if self.bits & CollationMetadata::CASE_FIRST_MASK != 0 {
            if self.bits & CollationMetadata::UPPER_FIRST_MASK != 0 {
                CollationCaseFirst::Upper
            } else {
                CollationCaseFirst::Lower
            }
        } else {
            CollationCaseFirst::False
        }
    }
}

/// Root-associated additional data that doesn't change in tailorings
///
/// These are the fields that logically belong to the root data but
/// don't belong to the tailoring data and that are on this separate
/// struct, since we have the same struct for a tailoring and the
/// bulk of the root.
///
/// As a practical matter, this struct happens to only carry
/// information about what concrete numeric values for primary
/// weights are special in particular ways. In principle, when the
/// root data is built, the root builder is allowed to assign the
/// numeric values as it sees fit, which is why these aren't
/// hard-coded.
///
/// Note: In 2.0.0 and prior, this struct was loaded only if
/// it was known at collator construction time (based on options)
/// that the data here was going to be needed. With the introduction
/// of collation keys and the decision not to introduce a collator
/// key generator object separate from the collator, this struct
/// is now always loaded.
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
/// to be stable, their Rust representation might not be. Use with caution.
/// </div>
#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
#[cfg_attr(feature = "datagen", databake(path = icu_collator::provider))]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub struct CollationSpecialPrimaries<'data> {
    /// The primaries corresponding to `MaxVariable`
    /// character classes packed so that each fits in
    /// 16 bits. Length must match the number of enum
    /// variants in `MaxVariable`, currently 4.
    ///
    /// This is potentially followed by 256 bits
    /// (packed in 16 u16s) to classify every possible
    /// byte into compressible or non-compressible.
    #[cfg_attr(feature = "serde", serde(borrow))]
    pub last_primaries: ZeroVec<'data, u16>,
    /// The high 8 bits of the numeric primary
    pub numeric_primary: u8,
}

#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
pub(crate) struct CollationSpecialPrimariesValidated<'data> {
    /// The primaries corresponding to `MaxVariable`
    /// character classes packed so that each fits in
    /// 16 bits. Length must match the number of enum
    /// variants in `MaxVariable`, currently 4.
    pub last_primaries: ZeroVec<'data, u16>,
    /// The high 8 bits of the numeric primary
    pub numeric_primary: u8,
    /// 256 bits (packed in 16 u16s) to classify every possible
    /// byte into compressible or non-compressible.
    pub compressible_bytes: &'data [<u16 as AsULE>::ULE; 16],
}

impl CollationSpecialPrimariesValidated<'static> {
    pub(crate) const HARDCODED_COMPRESSIBLE_BYTES_FALLBACK: &'static [<u16 as AsULE>::ULE; 16] = &[
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b1111_1111_1111_1110),
        <u16 as AsULE>::ULE::from_unsigned(0b1111_1111_1111_1111),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0001),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
        <u16 as AsULE>::ULE::from_unsigned(0b0100_0000_0000_0000),
    ];
}

icu_provider::data_struct!(
    CollationSpecialPrimaries<'_>,
    #[cfg(feature = "datagen")]
);

impl CollationSpecialPrimariesValidated<'_> {
    #[expect(clippy::unwrap_used)]
    pub(crate) fn last_primary_for_group(&self, max_variable: MaxVariable) -> u32 {
        // `unwrap` is OK, because `Collator::try_new` validates the length.
        //
        // Minus one to generate the right lower 16 bits from the high 16 bits.
        // See parse.cpp in genrb and getLastPrimaryForGroup in ICU4C.
        (u32::from(self.last_primaries.get(max_variable as usize).unwrap()) << 16) - 1
    }

    #[allow(dead_code)]
    pub(crate) fn is_compressible(&self, b: u8) -> bool {
        // Indexing slicing OK by construction and pasting this
        // into Compiler Explorer shows that the panic
        // is optimized away.
        #[expect(clippy::indexing_slicing)]
        let field = u16::from_unaligned(self.compressible_bytes[usize::from(b >> 4)]);
        let mask = 1 << (b & 0b1111);
        (field & mask) != 0
    }
}

/// Lists the locale and collation keyword combinations that the collator knows about.
/// The `standard` collation is represented as the empty string.
/// The root collation is represented as `und`.
/// Chinese collations are listed as `und-Hani` with `und-Hant` and `und-Hans` resolving
/// to `stroke` and `pinyin` despite not listing the collation keyword.
///
/// The iterator may (in practice _will_) yield duplicate items.
#[cfg(all(feature = "compiled_data", feature = "unstable"))]
pub fn list_locales() -> impl Iterator<Item = (DataLocale, tinystr::TinyAsciiStr<8>)> {
    use icu_provider::baked::DataStore;
    Baked::DATA_COLLATION_METADATA_V1
        .iter()
        .chain(Baked::DATA_COLLATION_TAILORING_V1.iter())
        .chain(Baked::DATA_COLLATION_REORDERING_V1.iter())
        .chain(Baked::DATA_COLLATION_DIACRITICS_V1.iter())
        .map(|d| {
            (
                d.locale.clone(),
                tinystr::TinyAsciiStr::<8>::try_from_str(d.marker_attributes.as_str())
                    .expect("Marker attribute invariants upheld"),
            )
        })
}