Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove -x- encoding for marker attributes #5038

Merged
merged 11 commits into from
Jun 14, 2024
Merged

Remove -x- encoding for marker attributes #5038

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8cd6063
wip
robertbastian Jun 11, 2024
4b92ef7
fix
robertbastian Jun 11, 2024
b435576
gen
robertbastian Jun 11, 2024
8253a5a
fix
robertbastian Jun 11, 2024
60ee8cd
ugh
robertbastian Jun 11, 2024
00734c8
Merge branch 'main' into x-
robertbastian Jun 12, 2024
2a8a8ac
const
robertbastian Jun 12, 2024
7e7ed16
don't deserialize in auxkey bench
robertbastian Jun 12, 2024
fff0e9d
deb
robertbastian Jun 13, 2024
c768a83
Merge branch 'main' into x-
robertbastian Jun 13, 2024
01de821
sizes
robertbastian Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
38 changes: 19 additions & 19 deletions ...onents/experimental/tests/transliterate/data/baked/macros/transliterator_rules_v1.rs.data
View file
Open in desktop

Large diffs are not rendered by default.

Binary file modified provider/adapters/tests/data/blob.postcard
View file
Open in desktop
Binary file not shown.
55 changes: 36 additions & 19 deletions provider/baked/src/binary_search.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,30 +16,47 @@ pub fn bake(
) -> TokenStream {
let mut data = data
.into_iter()
.map(|((l, a), i)| {
(
DataRequest {
locale: &l,
marker_attributes: &a,
..Default::default()
}
.legacy_encode(),
quote!(#i),
)
})
.map(|((l, a), i)| ((a.to_string(), l.to_string()), quote!(#i)))
.collect::<Vec<_>>();

data.sort_by(|(a, _), (b, _)| a.cmp(b));

let n = data.len();
let data = data.iter().map(|(r, i)| quote!((#r, &#i)));

quote! {
static DATA: [(&str, & #struct_type); #n] = [#(#data,)*];
fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
DATA.binary_search_by(|(k, _)| req.legacy_cmp(k.as_bytes()).reverse())
.map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
.ok()

if data.iter().all(|((a, _), _)| a.is_empty()) {
// Only DataLocales
let data = data.iter().map(|((_, l), i)| quote!((#l, &#i)));

quote! {
static DATA: [(&str, & #struct_type); #n] = [#(#data,)*];
fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
DATA.binary_search_by(|(l, _)| req.locale.strict_cmp(l.as_bytes()).reverse())
.map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
.ok()
}
}
} else if data.iter().all(|((_, l), _)| *l == "und") {
// Only marker attributes
let data = data.iter().map(|((a, _), i)| quote!((#a, &#i)));

quote! {
static DATA: [(&str, & #struct_type); #n] = [#(#data,)*];
fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
DATA.binary_search_by(|(a, _)| a.cmp(&&**req.marker_attributes))
.map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
.ok()
}
}
} else {
let data = data.iter().map(|((a, l), i)| quote!(((#a, #l), &#i)));

quote! {
static DATA: [((&str, &str), & #struct_type); #n] = [#(#data,)*];
fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
DATA.binary_search_by(|((a, l), _)| a.cmp(&&**req.marker_attributes).then_with(|| req.locale.strict_cmp(l.as_bytes()).reverse()))
.map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
.ok()
}
}
}
}
71 changes: 40 additions & 31 deletions provider/baked/src/export.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,21 +495,16 @@ impl DataExporter for BakedExporter {
let mut idents = reqs
.iter()
.map(|(locale, marker_attributes)| {
DataRequest {
locale,
marker_attributes,
..Default::default()
}
.legacy_encode()
.chars()
.map(|ch| {
if ch == '-' {
'_'
} else {
ch.to_ascii_uppercase()
}
})
.collect::<String>()
format!("_{}_{}", marker_attributes as &str, locale)
.chars()
.map(|ch| {
if ch == '-' {
'_'
} else {
ch.to_ascii_uppercase()
}
})
.collect::<String>()
})
.collect::<Vec<_>>();
idents.sort();
Expand Down Expand Up @@ -573,27 +568,41 @@ impl DataExporter for BakedExporter {
}
};

let mut legacy_encoded = deduplicated_values
let reqs = deduplicated_values
.values()
.flat_map(|s| {
s.iter().map(|(locale, marker_attributes)| {
DataRequest {
locale,
marker_attributes,
..Default::default()
}
.legacy_encode()
(marker_attributes.to_string(), locale.to_string())
})
})
.collect::<Vec<_>>();
legacy_encoded.sort();

let iterable_body = quote!(Ok(
[#(#legacy_encoded),*]
.into_iter()
.filter_map(icu_provider::DataRequest::legacy_decode)
.collect()
));
.collect::<BTreeSet<_>>();
let locales = reqs.iter().map(|(_, l)| l);
let attrs = reqs.iter().map(|(a, _)| a);

let iterable_body = if attrs.clone().all(|a| a.is_empty()) {
// Only DataLocales
quote!(Ok(
[#(icu_provider::_internal::locale_core::locale!(#locales)),*]
.into_iter()
.map(|l| (l.into(), Default::default()))
.collect()
))
} else if locales.clone().all(|l| l == "und") {
// Only attributes
quote!(Ok(
[#(#attrs),*]
.into_iter()
.map(|a| (Default::default(), a.parse().unwrap()))
.collect()
))
} else {
quote!(Ok(
[#((#attrs, icu_provider::_internal::locale_core::locale!(#locales))),*]
.into_iter()
.map(|(a, l)| (l.into(), a.parse().unwrap()))
.collect()
))
};

(load_body, iterable_body)
};
Expand Down
Loading
Loading