icu_locale/fallback/
algorithms.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use super::LocaleFallbackPriority;
6use icu_locale_core::subtags::{Language, Region, Script};
7
8use super::*;
9
10impl LocaleFallbackerWithConfig<'_> {
11    pub(crate) fn normalize(&self, locale: &mut DataLocale, default_script: &mut Option<Script>) {
12        // 0. If there is an invalid "sd" subtag, drop it
13        if let Some(subdivision) = locale.subdivision.take() {
14            if let Some(region) = locale.region {
15                if subdivision
16                    .as_str()
17                    .starts_with(region.to_tinystr().to_ascii_lowercase().as_str())
18                {
19                    locale.subdivision = Some(subdivision);
20                }
21            }
22        }
23        let language = locale.language;
24        // 1. Populate the region (required for region fallback only)
25        if self.config.priority == LocaleFallbackPriority::Region && locale.region.is_none() {
26            // 1a. First look for region based on language+script
27            if let Some(script) = locale.script {
28                locale.region = self
29                    .likely_subtags
30                    .language_script
31                    .get(&(
32                        language.to_tinystr().to_unvalidated(),
33                        script.to_tinystr().to_unvalidated(),
34                    ))
35                    .copied();
36            }
37            // 1b. If that fails, try language only
38            if locale.region.is_none() {
39                locale.region = self
40                    .likely_subtags
41                    .language
42                    .get_copied(&language.to_tinystr().to_unvalidated())
43                    .map(|(_s, r)| r);
44            }
45        }
46        // 2. Remove the script if it is implied by the other subtags
47        if locale.script.is_some() || self.config.priority == LocaleFallbackPriority::Script {
48            *default_script = locale
49                .region
50                .and_then(|region| {
51                    self.likely_subtags.language_region.get_copied(&(
52                        language.to_tinystr().to_unvalidated(),
53                        region.to_tinystr().to_unvalidated(),
54                    ))
55                })
56                .or_else(|| {
57                    self.likely_subtags
58                        .language
59                        .get_copied(&language.to_tinystr().to_unvalidated())
60                        .map(|(s, _r)| s)
61                });
62            if locale.script == *default_script {
63                locale.script = None;
64            }
65        }
66    }
67}
68
69impl LocaleFallbackIteratorInner<'_> {
70    pub fn step(&mut self, locale: &mut DataLocale) {
71        match self.config.priority {
72            LocaleFallbackPriority::Language => self.step_language(locale),
73            LocaleFallbackPriority::Script => self.step_script(locale),
74            LocaleFallbackPriority::Region => self.step_region(locale),
75            // This case should not normally happen, but `LocaleFallbackPriority` is non_exhaustive.
76            // Make it go directly to `und`.
77            _ => {
78                debug_assert!(
79                    false,
80                    "Unknown LocaleFallbackPriority: {:?}",
81                    self.config.priority
82                );
83                *locale = Default::default()
84            }
85        }
86    }
87
88    fn step_language(&mut self, locale: &mut DataLocale) {
89        // 2. Remove the subdivision keyword
90        if let Some(value) = locale.subdivision.take() {
91            self.backup_subdivision = Some(value);
92            return;
93        }
94        // 4. Remove variants
95        if let Some(single_variant) = locale.variant.take() {
96            self.backup_variant = Some(single_variant);
97            return;
98        }
99        // 5. Check for parent override
100        if let Some((language, script, region)) = self.get_explicit_parent(locale) {
101            locale.language = language;
102            locale.script = script;
103            locale.region = region;
104            locale.variant = self.backup_variant.take();
105            return;
106        }
107        // 7. Remove region
108        if let Some(region) = locale.region {
109            // 6. Add the script subtag if necessary
110            if locale.script.is_none() {
111                let language = locale.language;
112                if let Some(script) = self.likely_subtags.language_region.get_copied(&(
113                    language.to_tinystr().to_unvalidated(),
114                    region.to_tinystr().to_unvalidated(),
115                )) {
116                    locale.script = Some(script);
117                }
118            }
119            locale.region = None;
120            locale.variant = self.backup_variant.take();
121            return;
122        }
123        // 8. Remove language+script
124        debug_assert!(!locale.language.is_unknown() || locale.script.is_some()); // don't call .step() on und
125        locale.script = None;
126        locale.language = Language::UNKNOWN;
127    }
128
129    fn step_region(&mut self, locale: &mut DataLocale) {
130        // TODO(#4413): -u-rg is not yet supported
131        // 2. Remove the subdivision keyword
132        if let Some(value) = locale.subdivision.take() {
133            self.backup_subdivision = Some(value);
134            return;
135        }
136        // 4. Remove variants
137        if let Some(variant) = locale.variant.take() {
138            self.backup_variant = Some(variant);
139            return;
140        }
141        // 5. Remove language+script
142        if !locale.language.is_unknown() || locale.script.is_some() {
143            locale.script = None;
144            locale.language = Language::UNKNOWN;
145            // Don't produce und-variant
146            if locale.region.is_some() {
147                locale.variant = self.backup_variant.take();
148                locale.subdivision = self.backup_subdivision.take();
149            }
150            return;
151        }
152        // 6. Remove region
153        debug_assert!(locale.region.is_some()); // don't call .step() on und
154        locale.region = None;
155    }
156
157    fn step_script(&mut self, locale: &mut DataLocale) {
158        // Remove the subdivision keyword
159        if let Some(value) = locale.subdivision.take() {
160            self.backup_subdivision = Some(value);
161            return;
162        }
163        // Remove variants
164        if let Some(variant) = locale.variant.take() {
165            self.backup_variant = Some(variant);
166            return;
167        }
168        // Check for parent override
169        if let Some((language, script, region)) = self.get_explicit_parent(locale) {
170            locale.language = language;
171            locale.script = script;
172            locale.region = region;
173            locale.variant = self.backup_variant.take();
174            return;
175        }
176        // Remove the region
177        if let Some(region) = locale.region {
178            self.backup_region = Some(region);
179            let language_implied_script = self
180                .likely_subtags
181                .language
182                .get_copied(&locale.language.to_tinystr().to_unvalidated())
183                .map(|(s, _r)| s);
184            if language_implied_script != self.max_script {
185                locale.script = self.max_script;
186            }
187            locale.region = None;
188            locale.variant = self.backup_variant.take();
189            return;
190        }
191
192        // Remove the script if we have a language
193        if !locale.language.is_unknown() {
194            let language_implied_script = self
195                .likely_subtags
196                .language
197                .get_copied(&locale.language.to_tinystr().to_unvalidated())
198                .map(|(s, _r)| s);
199            if locale.script.is_some() && language_implied_script == locale.script {
200                locale.script = None;
201                if let Some(region) = self.backup_region.take() {
202                    locale.region = Some(region);
203                    locale.subdivision = self.backup_subdivision.take();
204                    locale.variant = self.backup_variant.take();
205                }
206                // needed if more fallback is added at the end
207                return;
208            } else {
209                // 3. Remove the language and apply the maximized script
210                locale.language = Language::UNKNOWN;
211                locale.script = self.max_script;
212                // Don't produce und-variant
213                if locale.script.is_some() {
214                    locale.variant = self.backup_variant.take();
215                }
216                // needed if more fallback is added at the end
217                return;
218            }
219        }
220
221        // note: UTS #35 wants us to apply "other associated scripts" now. ICU4C/J does not do this,
222        // so we don't either. They would be found here if they are ever needed:
223        // https://github.com/unicode-cldr/cldr-core/blob/master/supplemental/languageData.json
224
225        // 6. Remove script
226        if locale.script.is_some() {
227            locale.script = None;
228        }
229    }
230
231    fn get_explicit_parent(
232        &self,
233        locale: &DataLocale,
234    ) -> Option<(Language, Option<Script>, Option<Region>)> {
235        self.parents
236            .parents
237            .get_copied_by(|uvstr| locale.strict_cmp(uvstr).reverse())
238    }
239}
240
241#[cfg(test)]
242mod tests {
243    use super::*;
244    use writeable::Writeable;
245
246    struct TestCase {
247        input: &'static str,
248        requires_data: bool,
249        // Note: The first entry in the chain is the normalized locale
250        expected_language_chain: &'static [&'static str],
251        expected_script_chain: &'static [&'static str],
252        expected_region_chain: &'static [&'static str],
253    }
254
255    // TODO: Consider loading these from a JSON file
256    const TEST_CASES: &[TestCase] = &[
257        TestCase {
258            input: "en-fonipa",
259            requires_data: false,
260            expected_language_chain: &["en-fonipa", "en"],
261            expected_script_chain: &["en-fonipa", "en"],
262            expected_region_chain: &["en-fonipa", "en"],
263        },
264        TestCase {
265            input: "en-US-u-sd-usca",
266            requires_data: false,
267            expected_language_chain: &["en-US-u-sd-usca", "en-US", "en"],
268            expected_script_chain: &["en-US-u-sd-usca", "en-US", "en"],
269            expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
270        },
271        TestCase {
272            input: "en-US-fonipa-u-sd-usca",
273            requires_data: false,
274            expected_language_chain: &[
275                "en-US-fonipa-u-sd-usca",
276                "en-US-fonipa",
277                "en-US",
278                "en-fonipa",
279                "en",
280            ],
281            expected_script_chain: &[
282                "en-US-fonipa-u-sd-usca",
283                "en-US-fonipa",
284                "en-US",
285                "en-fonipa",
286                "en",
287            ],
288            expected_region_chain: &[
289                "en-US-fonipa-u-sd-usca",
290                "en-US-fonipa",
291                "en-US",
292                "und-US-fonipa-u-sd-usca",
293                "und-US-fonipa",
294                "und-US",
295            ],
296        },
297        TestCase {
298            input: "en-fonipa",
299            requires_data: true,
300            expected_language_chain: &["en-fonipa", "en"],
301            expected_script_chain: &["en-fonipa", "en", "und-Latn-fonipa", "und-Latn"],
302            expected_region_chain: &["en-US-fonipa", "en-US", "und-US-fonipa", "und-US"],
303        },
304        TestCase {
305            input: "en-Latn-fonipa",
306            requires_data: true,
307            expected_language_chain: &["en-fonipa", "en"],
308            expected_script_chain: &["en-fonipa", "en", "und-Latn-fonipa", "und-Latn"],
309            expected_region_chain: &["en-US-fonipa", "en-US", "und-US-fonipa", "und-US"],
310        },
311        TestCase {
312            input: "en-Latn-US-u-sd-usca",
313            requires_data: true,
314            expected_language_chain: &["en-US-u-sd-usca", "en-US", "en"],
315            expected_script_chain: &["en-US-u-sd-usca", "en-US", "en", "und-Latn"],
316            expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
317        },
318        TestCase {
319            input: "sr-ME",
320            requires_data: true,
321            expected_language_chain: &["sr-ME", "sr-Latn"],
322            expected_script_chain: &["sr-ME", "sr-Latn", "und-Latn"],
323            expected_region_chain: &["sr-ME", "und-ME"],
324        },
325        TestCase {
326            input: "sr-Latn-ME",
327            requires_data: true,
328            expected_language_chain: &["sr-ME", "sr-Latn"],
329            expected_script_chain: &["sr-ME", "sr-Latn", "und-Latn"],
330            expected_region_chain: &["sr-ME", "und-ME"],
331        },
332        TestCase {
333            input: "sr-ME-fonipa",
334            requires_data: true,
335            expected_language_chain: &["sr-ME-fonipa", "sr-ME", "sr-Latn-fonipa", "sr-Latn"],
336            expected_script_chain: &[
337                "sr-ME-fonipa",
338                "sr-ME",
339                "sr-Latn-fonipa",
340                "sr-Latn",
341                "und-Latn-fonipa",
342                "und-Latn",
343            ],
344            expected_region_chain: &["sr-ME-fonipa", "sr-ME", "und-ME-fonipa", "und-ME"],
345        },
346        TestCase {
347            input: "sr-RS",
348            requires_data: true,
349            expected_language_chain: &["sr-RS", "sr"],
350            expected_script_chain: &["sr-RS", "sr", "und-Cyrl"],
351            expected_region_chain: &["sr-RS", "und-RS"],
352        },
353        TestCase {
354            input: "sr-Cyrl-RS",
355            requires_data: true,
356            expected_language_chain: &["sr-RS", "sr"],
357            expected_script_chain: &["sr-RS", "sr", "und-Cyrl"],
358            expected_region_chain: &["sr-RS", "und-RS"],
359        },
360        TestCase {
361            input: "sr-Latn-RS",
362            requires_data: true,
363            expected_language_chain: &["sr-Latn-RS", "sr-Latn"],
364            expected_script_chain: &["sr-Latn-RS", "sr-Latn", "und-Latn"],
365            expected_region_chain: &["sr-Latn-RS", "und-RS"],
366        },
367        TestCase {
368            input: "de-Latn-LI",
369            requires_data: true,
370            expected_language_chain: &["de-LI", "de"],
371            expected_script_chain: &["de-LI", "de", "und-Latn"],
372            expected_region_chain: &["de-LI", "und-LI"],
373        },
374        TestCase {
375            input: "ca-ES-valencia",
376            requires_data: true,
377            expected_language_chain: &["ca-ES-valencia", "ca-ES", "ca-valencia", "ca"],
378            expected_script_chain: &[
379                "ca-ES-valencia",
380                "ca-ES",
381                "ca-valencia",
382                "ca",
383                "und-Latn-valencia",
384                "und-Latn",
385            ],
386            expected_region_chain: &["ca-ES-valencia", "ca-ES", "und-ES-valencia", "und-ES"],
387        },
388        TestCase {
389            input: "es-AR",
390            requires_data: true,
391            expected_language_chain: &["es-AR", "es-419", "es"],
392            expected_script_chain: &["es-AR", "es-419", "es", "und-Latn"],
393            expected_region_chain: &["es-AR", "und-AR"],
394        },
395        TestCase {
396            input: "hi-IN",
397            requires_data: true,
398            expected_language_chain: &["hi-IN", "hi"],
399            expected_script_chain: &["hi-IN", "hi", "und-Deva"],
400            expected_region_chain: &["hi-IN", "und-IN"],
401        },
402        TestCase {
403            input: "hi-Latn-IN",
404            requires_data: true,
405            expected_language_chain: &["hi-Latn-IN", "hi-Latn", "en-IN", "en-001", "en"],
406            expected_script_chain: &["hi-Latn-IN", "hi-Latn", "en-IN", "en-001", "en", "und-Latn"],
407            expected_region_chain: &["hi-Latn-IN", "und-IN"],
408        },
409        TestCase {
410            input: "zh-CN",
411            requires_data: true,
412            // Note: "zh-Hans" is not reachable because it is the default script for "zh".
413            // The fallback algorithm does not visit the language-script bundle when the
414            // script is the default for the language
415            expected_language_chain: &["zh-CN", "zh"],
416            expected_script_chain: &["zh-CN", "zh", "und-Hans", "und-Hani"],
417            expected_region_chain: &["zh-CN", "und-CN"],
418        },
419        TestCase {
420            input: "zh-TW",
421            requires_data: true,
422            expected_language_chain: &["zh-TW", "zh-Hant"],
423            expected_script_chain: &["zh-TW", "zh-Hant", "und-Hant", "und-Hani"],
424            expected_region_chain: &["zh-TW", "und-TW"],
425        },
426        TestCase {
427            input: "yue-HK",
428            requires_data: true,
429            expected_language_chain: &["yue-HK", "yue"],
430            expected_script_chain: &["yue-HK", "yue", "und-Hant", "und-Hani"],
431            expected_region_chain: &["yue-HK", "und-HK"],
432        },
433        TestCase {
434            input: "yue-HK",
435            requires_data: true,
436            expected_language_chain: &["yue-HK", "yue"],
437            expected_script_chain: &["yue-HK", "yue", "und-Hant", "und-Hani"],
438            expected_region_chain: &["yue-HK", "und-HK"],
439        },
440        TestCase {
441            input: "yue-CN",
442            requires_data: true,
443            expected_language_chain: &["yue-CN", "yue-Hans"],
444            expected_script_chain: &["yue-CN", "yue-Hans", "und-Hans", "und-Hani"],
445            expected_region_chain: &["yue-CN", "und-CN"],
446        },
447        TestCase {
448            input: "az-Arab-IR",
449            requires_data: true,
450            expected_language_chain: &["az-IR", "az-Arab"],
451            expected_script_chain: &["az-IR", "az-Arab", "und-Arab"],
452            expected_region_chain: &["az-IR", "und-IR"],
453        },
454        TestCase {
455            input: "az-IR",
456            requires_data: true,
457            expected_language_chain: &["az-IR", "az-Arab"],
458            expected_script_chain: &["az-IR", "az-Arab", "und-Arab"],
459            expected_region_chain: &["az-IR", "und-IR"],
460        },
461        TestCase {
462            input: "az-Arab",
463            requires_data: true,
464            expected_language_chain: &["az-Arab"],
465            expected_script_chain: &["az-Arab", "und-Arab"],
466            expected_region_chain: &["az-IR", "und-IR"],
467        },
468    ];
469
470    #[test]
471    fn test_fallback() {
472        let fallbacker_no_data = LocaleFallbacker::new_without_data();
473        let fallbacker_no_data = fallbacker_no_data.as_borrowed();
474        let fallbacker_with_data = LocaleFallbacker::new();
475        for cas in TEST_CASES {
476            for (priority, expected_chain) in [
477                (
478                    LocaleFallbackPriority::Language,
479                    cas.expected_language_chain,
480                ),
481                (LocaleFallbackPriority::Script, cas.expected_script_chain),
482                (LocaleFallbackPriority::Region, cas.expected_region_chain),
483            ] {
484                let mut config = LocaleFallbackConfig::default();
485                config.priority = priority;
486                let fallbacker = if cas.requires_data {
487                    fallbacker_with_data
488                } else {
489                    fallbacker_no_data
490                };
491                let mut it = fallbacker
492                    .for_config(config)
493                    .fallback_for(cas.input.parse().unwrap());
494                let mut actual_chain = Vec::new();
495                for i in 0..20 {
496                    if i == 19 {
497                        eprintln!("20 iterations reached!");
498                    }
499                    if it.get().is_unknown() {
500                        break;
501                    }
502                    actual_chain.push(it.get().write_to_string().into_owned());
503                    it.step();
504                }
505                assert_eq!(
506                    expected_chain, &actual_chain,
507                    "{:?} ({:?})",
508                    cas.input, priority
509                );
510            }
511        }
512    }
513}
icu_locale/fallback/algorithms.rs

icu_locale/fallback/
algorithms.rs
