From d098455ae694fd99a5deab372a38c8e6e8e9c70d Mon Sep 17 00:00:00 2001 From: Conrad Nied Date: Mon, 18 Nov 2024 16:14:04 -0800 Subject: [PATCH] CLDR-18087 Remove --- common/dtd/ldmlSupplemental.dtd | 21 - common/dtd/ldmlSupplemental.xsd | 34 - .../supplemental/attributeValueValidity.xml | 4 - common/supplemental/likelySubtags.xml | 10 +- common/supplemental/supplementalData.xml | 1164 +---------------- docs/ldml/tr35-info.md | 31 +- .../java/org/unicode/cldr/test/CLDRTest.java | 1 - .../cldr/tool/ConvertLanguageData.java | 302 +---- .../cldr/util/CLDRFileBuiltConstants.java | 2 +- .../cldr/util/SupplementalDataInfo.java | 29 - .../cldr/json/JSON_config_supplemental.txt | 1 - .../org/unicode/cldr/util/data/PathHeader.txt | 3 - 12 files changed, 16 insertions(+), 1586 deletions(-) diff --git a/common/dtd/ldmlSupplemental.dtd b/common/dtd/ldmlSupplemental.dtd index 237fb957063..79a8f7d3ed4 100644 --- a/common/dtd/ldmlSupplemental.dtd +++ b/common/dtd/ldmlSupplemental.dtd @@ -135,27 +135,6 @@ CLDR data files are interpreted according to the LDML specification (http://unic - - - - - - - - - - - - - - - - - - - - - diff --git a/common/dtd/ldmlSupplemental.xsd b/common/dtd/ldmlSupplemental.xsd index 89c80824dfb..91cf6c42636 100644 --- a/common/dtd/ldmlSupplemental.xsd +++ b/common/dtd/ldmlSupplemental.xsd @@ -19,7 +19,6 @@ Note: DTD @-annotations are not currently converted to .xsd. For full CLDR file - @@ -333,39 +332,6 @@ Note: DTD @-annotations are not currently converted to .xsd. For full CLDR file - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/common/supplemental/attributeValueValidity.xml b/common/supplemental/attributeValueValidity.xml index 10f314f7fd3..517236ae3bc 100644 --- a/common/supplemental/attributeValueValidity.xml +++ b/common/supplemental/attributeValueValidity.xml @@ -192,10 +192,6 @@ $_bcp47_currency|$defaultCurrencyInfo [0-9]+ $_bcp47_keys - $_script - $_region - $localeOrDeprecated - $_variant [a-zA-Z0-9]{1,3}(_[A-Za-z0-9]{2,8})* diff --git a/common/supplemental/likelySubtags.xml b/common/supplemental/likelySubtags.xml index 6dd06b45d6f..cf598a3e499 100644 --- a/common/supplemental/likelySubtags.xml +++ b/common/supplemental/likelySubtags.xml @@ -244,11 +244,11 @@ not be patched by hand, as any changes made in that fashion may be lost. - + - + @@ -269,7 +269,7 @@ not be patched by hand, as any changes made in that fashion may be lost. - + @@ -325,7 +325,7 @@ not be patched by hand, as any changes made in that fashion may be lost. - + @@ -364,7 +364,6 @@ not be patched by hand, as any changes made in that fashion may be lost. - @@ -1097,6 +1096,7 @@ not be patched by hand, as any changes made in that fashion may be lost. + diff --git a/common/supplemental/supplementalData.xml b/common/supplemental/supplementalData.xml index 58893bc9318..5b181be51f2 100644 --- a/common/supplemental/supplementalData.xml +++ b/common/supplemental/supplementalData.xml @@ -1275,1169 +1275,7 @@ XXX Code for transations where no currency is involved - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + diff --git a/docs/ldml/tr35-info.md b/docs/ldml/tr35-info.md index 8dc53c0e2a3..57403dd994f 100644 --- a/docs/ldml/tr35-info.md +++ b/docs/ldml/tr35-info.md @@ -109,7 +109,7 @@ The LDML specification is divided into the following parts: The following represents the format for additional supplemental information. This is information that is important for internationalization and proper use of CLDR, but is not contained in the locale hierarchy. It is not localizable, nor is it overridden by locale data. The current CLDR data can be viewed in the [Supplemental Charts](https://www.unicode.org/cldr/charts/46/supplemental/index.html). ```xml - + ``` The data in CLDR is presently split into multiple files: supplementalData.xml, supplementalMetadata.xml, characters.xml, likelySubtags.xml, ordinals.xml, plurals.xml, telephoneCodeData.xml, genderList.xml, plus transforms (see _Part 2 [Transforms](tr35-general.md#Transforms)_ and _Part 2 [Transform Rule Syntax](tr35-general.md#Transform_Rules_Syntax)_). The split is just for convenience: logically, they are treated as though they were a single file. Future versions of CLDR may split the data in a different fashion. Do not depend on any specific XML filename or path for supplemental data. @@ -307,35 +307,6 @@ The exact format of the path is provisional in CLDR 29, but as currently shown: * An attribute value of `'*'` indicates that the path applies regardless of the value of the attribute. * Each path must have exactly one attribute whose value is marked here as `'#'`; in actual data items with this path, the corresponding value is a list of region codes. It is the region codes in this list that are compared with the region specified by the “rg” key to determine which data item to use for this path. -## Supplemental Language Data - -```xml - - - - - - - -``` - -The language data is used for consistency checking and testing. It provides a list of which languages are used with which scripts and in which countries. To a large extent, however, the territory list has been superseded by the data in _[Supplemental Territory Information](#Supplemental_Territory_Information)_ . - -```xml - - - - - ... -``` - -If the language is not a modern language, or the script is not a modern script, or the language not a major language of the territory, then the `alt` attribute is set to secondary. - -```xml - - ... -``` - ## Supplemental Language Grouping ```xml diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/test/CLDRTest.java b/tools/cldr-code/src/main/java/org/unicode/cldr/test/CLDRTest.java index 004432f3958..dfdd8258687 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/test/CLDRTest.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/test/CLDRTest.java @@ -751,7 +751,6 @@ private Map> getCompletionExceptions() { } // - // void getSupplementalData( Map> language_scripts, Map> language_territories, diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ConvertLanguageData.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ConvertLanguageData.java index 08e61d4f566..4fcd15305a0 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ConvertLanguageData.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ConvertLanguageData.java @@ -21,7 +21,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -58,7 +57,6 @@ import org.unicode.cldr.util.SupplementalDataInfo; import org.unicode.cldr.util.SupplementalDataInfo.BasicLanguageData; import org.unicode.cldr.util.SupplementalDataInfo.OfficialStatus; -import org.unicode.cldr.util.SupplementalDataInfo.PopulationData; import org.unicode.cldr.util.TransliteratorUtilities; import org.unicode.cldr.util.Validity; import org.unicode.cldr.util.Validity.Status; @@ -133,9 +131,14 @@ public static void main(String[] args) throws IOException, ParseException { try (final BufferedReader oldFile = FileUtilities.openUTF8Reader(oldSupp); final PrintWriter newFile = FileUtilities.openUTF8Writer(genSupp); final PrintWriter newLsraw = FileUtilities.openUTF8Writer(genLsraw); ) { - // load elements we care about + + // Copy all of the information from supplementalData up until the block of generated + // data CldrUtility.copyUpTo( - oldFile, PatternCache.get("\\s*\\s*"), newFile, false); + oldFile /* file copying from */, + PatternCache.get("\\s*"); + "\t"); out.println("\t"); for (RowData row : sortedInput) { @@ -2349,46 +2103,6 @@ static void addLanguageScriptData() throws IOException { } } - // private static void showAllBasicLanguageData(Relation - // language2basicData, String - // comment) { - // // now print - // Relation primaryCombos = new Relation(new TreeMap(), TreeSet.class); - // Relation secondaryCombos = new Relation(new TreeMap(), TreeSet.class); - // - // Log.println("\t" + (comment == null ? "" : " ")); - // - // for (String languageSubtag : language2basicData.keySet()) { - // String duplicate = ""; - // // script,territory - // primaryCombos.clear(); - // secondaryCombos.clear(); - // - // for (BasicLanguageData item : language2basicData.getAll(languageSubtag)) { - // Set scripts = item.getScripts(); - // if (scripts.size() == 0) scripts = new TreeSet(Arrays.asList(new String[] { "Zzzz" })); - // for (String script : scripts) { - // Set territories = item.getTerritories(); - // if (territories.size() == 0) territories = new TreeSet(Arrays.asList(new String[] { "ZZ" })); - // for (String territory : territories) { - // if (item.getType().equals(BasicLanguageData.Type.primary)) { - // primaryCombos.put(script, territory); - // } else { - // secondaryCombos.put(script, territory); - // } - // } - // } - // } - // secondaryCombos.removeAll(primaryCombos); - // showBasicLanguageData(languageSubtag, primaryCombos, null, BasicLanguageData.Type.primary); - // showBasicLanguageData(languageSubtag, secondaryCombos, primaryCombos.keySet(), - // BasicLanguageData.Type.secondary); - // // System.out.println(item.toString(languageSubtag) + duplicate); - // // duplicate = " "; - // } - // Log.println("\t"); - // } - private static void showBasicLanguageData( PrintWriter out, String languageSubtag, diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRFileBuiltConstants.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRFileBuiltConstants.java index 568dab64ebc..e9ef687fe3f 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRFileBuiltConstants.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRFileBuiltConstants.java @@ -4,7 +4,7 @@ public class CLDRFileBuiltConstants { // TODO, convert to use SupplementalInfo // Constants generated by FindDTDOrder. Don't edit by hand public static final String ELEMENT_ORDERING = - "ldml alternate attributeOrder attributes blockingItems calendarSystem character character-fallback codePattern codesByTerritory comment context cp deprecatedItems distinguishingItems elementOrder first_variable fractions identity info languageAlias languageCodes languageCoverage languagePopulation last_variable first_tertiary_ignorable last_tertiary_ignorable first_secondary_ignorable last_secondary_ignorable first_primary_ignorable last_primary_ignorable first_non_ignorable last_non_ignorable first_trailing last_trailing likelySubtag mapTimezones mapZone pluralRule pluralRules reference region scriptAlias scriptCoverage serialElements substitute suppress tRule telephoneCountryCode territoryAlias territoryCodes territoryCoverage currencyCoverage timezone timezoneCoverage transform usesMetazone validity alias appendItem base beforeCurrency afterCurrency currencyMatch dateFormatItem day deprecated distinguishing blocking coverageAdditions era eraNames eraAbbr eraNarrow exemplarCharacters fallback field generic greatestDifference height hourFormat hoursFormat gmtFormat intervalFormatFallback intervalFormatItem key localeDisplayNames layout localeDisplayPattern languages localePattern localeSeparator localizedPatternChars dateRangePattern calendars long mapping measurementSystem measurementSystemName messages minDays firstDay month months monthNames monthAbbr days dayNames dayAbbr orientation inList inText paperSize pattern displayName quarter quarters quotationStart quotationEnd alternateQuotationStart alternateQuotationEnd regionFormat fallbackFormat abbreviationFallback preferenceOrdering relative reset p pc rule s sc scripts segmentation settings short commonlyUsed exemplarCity singleCountries default calendar collation currency currencyFormat currencySpacing currencyFormatLength dateFormat dateFormatLength dateTimeFormat dateTimeFormatLength availableFormats appendItems dayContext dayWidth decimalFormat decimalFormatLength intervalFormats monthContext monthWidth percentFormat percentFormatLength quarterContext quarterWidth scientificFormat scientificFormatLength skipDefaultLocale defaultContent standard daylight suppress_contractions optimize rules surroundingMatch insertBetween symbol decimal group list percentSign nativeZeroDigit patternDigit plusSign minusSign exponential perMille infinity nan currencyDecimal currencyGroup symbols decimalFormats scientificFormats percentFormats currencyFormats currencies t tc i ic extend territories timeFormat timeFormatLength timeZoneNames type unit unitPattern variable attributeValues variables segmentRules variantAlias variants keys types measurementSystemNames codePatterns version generation currencyData language script territory territoryContainment languageData territoryInfo calendarData variant week am pm eras dateFormats timeFormats dateTimeFormats fields weekData measurementData timezoneData characters delimiters measurement dates numbers transforms metadata codeMappings likelySubtags metazoneInfo plurals telephoneCodeData units collations posix segmentations references weekendStart weekendEnd width x yesstr nostr yesexpr noexpr zone metazone special zoneAlias zoneFormatting zoneItem supplementalData"; + "ldml alternate attributeOrder attributes blockingItems calendarSystem character character-fallback codePattern codesByTerritory comment context cp deprecatedItems distinguishingItems elementOrder first_variable fractions identity info languageAlias languageCodes languageCoverage languagePopulation last_variable first_tertiary_ignorable last_tertiary_ignorable first_secondary_ignorable last_secondary_ignorable first_primary_ignorable last_primary_ignorable first_non_ignorable last_non_ignorable first_trailing last_trailing likelySubtag mapTimezones mapZone pluralRule pluralRules reference region scriptAlias scriptCoverage serialElements substitute suppress tRule telephoneCountryCode territoryAlias territoryCodes territoryCoverage currencyCoverage timezone timezoneCoverage transform usesMetazone validity alias appendItem base beforeCurrency afterCurrency currencyMatch dateFormatItem day deprecated distinguishing blocking coverageAdditions era eraNames eraAbbr eraNarrow exemplarCharacters fallback field generic greatestDifference height hourFormat hoursFormat gmtFormat intervalFormatFallback intervalFormatItem key localeDisplayNames layout localeDisplayPattern languages localePattern localeSeparator localizedPatternChars dateRangePattern calendars long mapping measurementSystem measurementSystemName messages minDays firstDay month months monthNames monthAbbr days dayNames dayAbbr orientation inList inText paperSize pattern displayName quarter quarters quotationStart quotationEnd alternateQuotationStart alternateQuotationEnd regionFormat fallbackFormat abbreviationFallback preferenceOrdering relative reset p pc rule s sc scripts segmentation settings short commonlyUsed exemplarCity singleCountries default calendar collation currency currencyFormat currencySpacing currencyFormatLength dateFormat dateFormatLength dateTimeFormat dateTimeFormatLength availableFormats appendItems dayContext dayWidth decimalFormat decimalFormatLength intervalFormats monthContext monthWidth percentFormat percentFormatLength quarterContext quarterWidth scientificFormat scientificFormatLength skipDefaultLocale defaultContent standard daylight suppress_contractions optimize rules surroundingMatch insertBetween symbol decimal group list percentSign nativeZeroDigit patternDigit plusSign minusSign exponential perMille infinity nan currencyDecimal currencyGroup symbols decimalFormats scientificFormats percentFormats currencyFormats currencies t tc i ic extend territories timeFormat timeFormatLength timeZoneNames type unit unitPattern variable attributeValues variables segmentRules variantAlias variants keys types measurementSystemNames codePatterns version generation currencyData language script territory territoryContainment territoryInfo calendarData variant week am pm eras dateFormats timeFormats dateTimeFormats fields weekData measurementData timezoneData characters delimiters measurement dates numbers transforms metadata codeMappings likelySubtags metazoneInfo plurals telephoneCodeData units collations posix segmentations references weekendStart weekendEnd width x yesstr nostr yesexpr noexpr zone metazone special zoneAlias zoneFormatting zoneItem supplementalData"; // END_ELEMENT_ORDERING public static final String ATTRIBUTE_ORDERING = "_q type id choice key registry source target path day date version count lines characters iso4217 before from to mzone number time casing list uri digits rounding iso3166 hex request direction alternate backwards caseFirst caseLevel hiraganaQuarternary hiraganaQuaternary maxVariable variableTop normalization numeric strength elements element attributes attribute aliases attributeValue contains multizone order other replacement scripts services territories territory tzidVersion value values variant variants visibility alpha3 code end exclude fips10 gdp internet literacyPercent locales officialStatus population populationPercent start used writingPercent validSubLocales standard references alt draft"; diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java index a298de5c959..2ee1034a988 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java @@ -1411,9 +1411,6 @@ public void handlePathValue(String path, String value) { } else if (level1.equals("calendarPreferenceData")) { handleCalendarPreferenceData(parts); return; - } else if (level1.equals("languageData")) { - handleLanguageData(parts); - return; } else if (level1.equals("territoryContainment")) { handleTerritoryContainment(parts); return; @@ -2411,32 +2408,6 @@ private void handleSubdivisionContainment(XPathValue parts) { } } - private void handleLanguageData(XPathValue parts) { - // - // - // - String language = parts.getAttributeValue(2, "type"); - BasicLanguageData languageData = new BasicLanguageData(); - languageData.setType( - parts.getAttributeValue(2, "alt") == null - ? BasicLanguageData.Type.primary - : BasicLanguageData.Type.secondary); - languageData - .setScripts(parts.getAttributeValue(2, "scripts")) - .setTerritories(parts.getAttributeValue(2, "territories")); - Map map = languageToBasicLanguageData.get(language); - if (map == null) { - languageToBasicLanguageData.put( - language, map = new EnumMap<>(BasicLanguageData.Type.class)); - } - if (map.containsKey(languageData.type)) { - throw new IllegalArgumentException("Duplicate value:\t" + parts); - } - map.put(languageData.type, languageData); - } - private boolean failsRangeCheck(String path, double input, double min, double max) { if (input >= min && input <= max) { return false; diff --git a/tools/cldr-code/src/main/resources/org/unicode/cldr/json/JSON_config_supplemental.txt b/tools/cldr-code/src/main/resources/org/unicode/cldr/json/JSON_config_supplemental.txt index 66042098b28..1caeedaee08 100644 --- a/tools/cldr-code/src/main/resources/org/unicode/cldr/json/JSON_config_supplemental.txt +++ b/tools/cldr-code/src/main/resources/org/unicode/cldr/json/JSON_config_supplemental.txt @@ -13,7 +13,6 @@ section=pluralRanges ; path=//cldr/supplemental/plurals/.* ; package=core section=postalCodeData ; path=//cldr/supplemental/postalCodeData/.* ; package=core section=currencyData ; path=//cldr/supplemental/currencyData/.* ; package=core section=territoryContainment ; path=//cldr/supplemental/territoryContainment/.* ; package=core -section=languageData ; path=//cldr/supplemental/languageData/.* ; package=core section=languageGroups ; path=//cldr/supplemental/languageGroups/.* ; package=core section=languageMatching ; path=//cldr/supplemental/languageMatching/.* ; package=core section=territoryInfo ; path=//cldr/supplemental/territoryInfo/.* ; package=core diff --git a/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/PathHeader.txt b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/PathHeader.txt index 30d3c5b2260..5e4e1b81da5 100644 --- a/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/PathHeader.txt +++ b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/PathHeader.txt @@ -442,9 +442,6 @@ //supplementalData/languageMatching/languageMatches[@type="%A"]/paradigmLocales/_%E ; Supplemental ; LanguageMatch ; $1 ; paradigmLocales ; HIDE //supplementalData/languageMatching/languageMatches[@type="%A"]/matchVariable[@id="%A"]/_%E ; Supplemental ; LanguageMatch ; $1 ; matchVariable-$2 ; HIDE -//supplementalData/languageData/language[@type="%A"]/_%E ; Supplemental ; Language ; $1 ; $2 ; HIDE -//supplementalData/languageData/language[@type="%A"][@alt="%A"]/_%E ; Supplemental ; Language ; $1 ; $3-$2 ; HIDE - //supplementalData/transforms/transform[@source="%A"][@target="%A"][@direction="%A"]/%E ; Supplemental ; Transform ; &transform($3,$1,$2) ; $4 ; HIDE //supplementalData/transforms/transform[@source="%A"][@target="%A"][@variant="%A"][@direction="%A"]/%E ; Supplemental ; Transform ; &transform($4,$1,$2,$3) ; $5 ; HIDE //supplementalData/transforms/transform[@source="%A"][@target="%A"] ; Supplemental ; Transform ; &transform("?",$1,$2) ; ? ; HIDE