Re-generate QLocale data from CLDR v29

* A bunch of fixes and additions to the locale data
* Add new scripts from Unicode 8.0 and 9.0
* Map some potentially useful languages and territories

[ChangeLog][QtCore] QLocale data updated to CLDR v29

Change-Id: I759ccb27fe19be2722be913c5c2e6aa5f36e5c14
Reviewed-by: Lars Knoll <lars.knoll@theqtcompany.com>

src/corelib/tools/qlocale.cpp

@@ -2390,9 +2390,11 @@ QLocale::MeasurementSystem QLocale::measurementSystem() const
 Qt::LayoutDirection QLocale::textDirection() const
 {
     switch (script()) {
+    case QLocale::AdlamScript:
     case QLocale::ArabicScript:
     case QLocale::AvestanScript:
     case QLocale::CypriotScript:
+    case QLocale::HatranScript:
     case QLocale::HebrewScript:
     case QLocale::ImperialAramaicScript:
     case QLocale::InscriptionalPahlaviScript:
@@ -2406,6 +2408,7 @@ Qt::LayoutDirection QLocale::textDirection() const
     case QLocale::MeroiticScript:
     case QLocale::NabataeanScript:
     case QLocale::NkoScript:
+    case QLocale::OldHungarianScript:
     case QLocale::OldNorthArabianScript:
     case QLocale::OldSouthArabianScript:
     case QLocale::OrkhonScript:

src/corelib/tools/qlocale.h

@@ -416,6 +416,27 @@ public:
         Mono = 337,
         TedimChin = 338,
         Maithili = 339,
+        Ahom = 340,
+        AmericanSignLanguage = 341,
+        ArdhamagadhiPrakrit = 342,
+        Bhojpuri = 343,
+        HieroglyphicLuwian = 344,
+        LiteraryChinese = 345,
+        Mazanderani = 346,
+        Mru = 347,
+        Newari = 348,
+        NorthernLuri = 349,
+        Palauan = 350,
+        Papiamento = 351,
+        Saraiki = 352,
+        TokelauLanguage = 353,
+        TokPisin = 354,
+        TuvaluLanguage = 355,
+        UncodedLanguages = 356,
+        Cantonese = 357,
+        Osage = 358,
+        Tangut = 359,
+
         Norwegian = NorwegianBokmal,
         Moldavian = Romanian,
         SerboCroatian = Serbian,
@@ -430,7 +451,8 @@ public:
         Chewa = Nyanja,
         Frisian = WesternFrisian,
         Uigur = Uighur,
-        LastLanguage = Maithili
+
+        LastLanguage = Tangut
     };
 
     enum Script {
@@ -562,9 +584,25 @@ public:
         KhudawadiScript = 125,
         TirhutaScript = 126,
         VarangKshitiScript = 127,
+        AhomScript = 128,
+        AnatolianHieroglyphsScript = 129,
+        HatranScript = 130,
+        MultaniScript = 131,
+        OldHungarianScript = 132,
+        SignWritingScript = 133,
+        AdlamScript = 134,
+        BhaiksukiScript = 135,
+        MarchenScript = 136,
+        NewaScript = 137,
+        OsageScript = 138,
+        TangutScript = 139,
+        HanWithBopomofoScript = 140,
+        JamoScript = 141,
+
         SimplifiedChineseScript = SimplifiedHanScript,
         TraditionalChineseScript = TraditionalHanScript,
-        LastScript = VarangKshitiScript
+
+        LastScript = JamoScript
     };
     enum Country {
         AnyCountry = 0,
@@ -780,14 +818,14 @@ public:
         Tanzania = 210,
         Thailand = 211,
         Togo = 212,
-        Tokelau = 213,
+        TokelauCountry = 213,
         Tonga = 214,
         TrinidadAndTobago = 215,
         Tunisia = 216,
         Turkey = 217,
         Turkmenistan = 218,
         TurksAndCaicosIslands = 219,
-        Tuvalu = 220,
+        TuvaluCountry = 220,
         Uganda = 221,
         Ukraine = 222,
         UnitedArabEmirates = 223,
@@ -825,13 +863,19 @@ public:
         Bonaire = 255,
         SintMaarten = 256,
         Kosovo = 257,
+        EuropeanUnion = 258,
+        OutlyingOceania = 259,
+
+        Tokelau = TokelauCountry,
+        Tuvalu = TuvaluCountry,
         DemocraticRepublicOfCongo = CongoKinshasa,
         PeoplesRepublicOfCongo = CongoBrazzaville,
         DemocraticRepublicOfKorea = NorthKorea,
         RepublicOfKorea = SouthKorea,
         RussianFederation = Russia,
         SyrianArabRepublic = Syria,
-        LastCountry = Kosovo
+
+        LastCountry = OutlyingOceania
     };
 // GENERATED PART ENDS HERE
 

src/corelib/tools/qlocale.qdoc

@@ -92,7 +92,7 @@
     \note For the current keyboard input locale take a look at
     QInputMethod::locale().
 
-    QLocale's data is based on Common Locale Data Repository v27.
+    QLocale's data is based on Common Locale Data Repository v29.
 
     The double-to-string and string-to-double conversion functions are
     covered by the following licenses:
@@ -483,6 +483,26 @@
     \value Maithili Since Qt 5.5
     \value LowerSorbian Since Qt 5.5
     \value UpperSorbian Since Qt 5.5
+    \value Ahom Since Qt 5.7
+    \value AmericanSignLanguage Since Qt 5.7
+    \value ArdhamagadhiPrakrit Since Qt 5.7
+    \value Bhojpuri Since Qt 5.7
+    \value Cantonese Since Qt 5.7
+    \value HieroglyphicLuwian Since Qt 5.7
+    \value LiteraryChinese Since Qt 5.7
+    \value Mazanderani Since Qt 5.7
+    \value Mru Since Qt 5.7
+    \value Newari Since Qt 5.7
+    \value NorthernLuri Since Qt 5.7
+    \value Osage Since Qt 5.7
+    \value Palauan Since Qt 5.7
+    \value Papiamento Since Qt 5.7
+    \value Saraiki Since Qt 5.7
+    \value Tangut Since Qt 5.7
+    \value TokelauLanguage Since Qt 5.7
+    \value TokPisin Since Qt 5.7
+    \value TuvaluLanguage Since Qt 5.7
+    \value UncodedLanguages Since Qt 5.7
     \omitvalue LastLanguage
 
     \sa language(), languageToString()
@@ -567,6 +587,7 @@
     \value Eritrea
     \value Estonia
     \value Ethiopia
+    \value EuropeanUnion Since Qt 5.7
     \value FalklandIslands
     \value FaroeIslands
     \value Fiji
@@ -662,6 +683,7 @@
     \value NorthernMarianaIslands
     \value Norway
     \value Oman
+    \value OutlyingOceania Since Qt 5.7
     \value Pakistan
     \value Palau
     \value PalestinianTerritories
@@ -714,14 +736,16 @@
     \value Tanzania
     \value Thailand
     \value Togo
-    \value Tokelau
+    \value TokelauCountry
+    \value Tokelau Obsolete, please use TokelauCountry
     \value Tonga
     \value TrinidadAndTobago
     \value Tunisia
     \value Turkey
     \value Turkmenistan
     \value TurksAndCaicosIslands
-    \value Tuvalu
+    \value TuvaluCountry
+    \value Tuvalu Obsolete, please use TuvaluCountry
     \value Uganda
     \value Ukraine
     \value UnitedArabEmirates
@@ -768,6 +792,9 @@
     This enumerated type is used to specify a script.
 
     \value AnyScript
+    \value AdlamScript Since Qt 5.7
+    \value AhomScript Since Qt 5.7
+    \value AnatolianHieroglyphsScript Since Qt 5.7
     \value ArabicScript
     \value ArmenianScript
     \value AvestanScript
@@ -776,6 +803,7 @@
     \value BassaVahScript Since Qt 5.5
     \value BatakScript
     \value BengaliScript
+    \value BhaiksukiScript Since Qt 5.7
     \value BopomofoScript
     \value BrahmiScript
     \value BrailleScript
@@ -807,11 +835,14 @@
     \value HanScript
     \value HangulScript
     \value HanunooScript
+    \value HanWithBopomofoScript Since Qt 5.7
+    \value HatranScript Since Qt 5.7
     \value HebrewScript
     \value HiraganaScript
     \value ImperialAramaicScript
     \value InscriptionalPahlaviScript
     \value InscriptionalParthianScript
+    \value JamoScript Since Qt 5.7
     \value JapaneseScript
     \value JavaneseScript
     \value KaithiScript
@@ -836,6 +867,7 @@
     \value MalayalamScript
     \value MandaeanScript
     \value ManichaeanScript Since Qt 5.5
+    \value MarchenScript Since Qt 5.7
     \value MeiteiMayekScript
     \value MendeKikakuiScript Since Qt 5.5
     \value MeroiticScript
@@ -843,19 +875,23 @@
     \value ModiScript Since Qt 5.5
     \value MongolianScript
     \value MroScript Since Qt 5.5
+    \value MultaniScript Since Qt 5.7
     \value MyanmarScript
     \value NabataeanScript Since Qt 5.5
     \value NkoScript
+    \value NewaScript Since Qt 5.7
     \value NewTaiLueScript
     \value OghamScript
     \value OlChikiScript
     \value OldItalicScript
+    \value OldHungarianScript Since Qt 5.7
     \value OldNorthArabianScript Since Qt 5.5
     \value OldPermicScript Since Qt 5.5
     \value OldPersianScript
     \value OldSouthArabianScript
     \value OriyaScript
     \value OrkhonScript
+    \value OsageScript Since Qt 5.7
     \value OsmanyaScript
     \value PahawhHmongScript Since Qt 5.5
     \value PalmyreneScript Since Qt 5.5
@@ -871,6 +907,7 @@
     \value SharadaScript
     \value ShavianScript
     \value SiddhamScript Since Qt 5.5
+    \value SignWritingScript Since Qt 5.7
     \value SimplifiedHanScript same as SimplifiedChineseScript
     \value SimplifiedChineseScript same as SimplifiedHanScript
     \value SinhalaScript
@@ -885,6 +922,7 @@
     \value TaiVietScript
     \value TakriScript
     \value TamilScript
+    \value TangutScript Since Qt 5.7
     \value TeluguScript
     \value ThaanaScript
     \value ThaiScript

tests/auto/corelib/tools/qlocale/tst_qlocale.cpp

@@ -1939,8 +1939,8 @@ void tst_QLocale::standaloneDayName_data()
     QTest::newRow("C short")  << QString("C") << QString("Sun") << 7 << QLocale::ShortFormat;
     QTest::newRow("C narrow")  << QString("C") << QString("S") << 7 << QLocale::NarrowFormat;
 
-    QTest::newRow("ru_RU long")  << QString("ru_RU") << QString::fromUtf8("\320\222\320\276\321\201\320\272\321\200\320\265\321\201\320\265\320\275\321\214\320\265") << 7 << QLocale::LongFormat;
+    QTest::newRow("ru_RU long")  << QString("ru_RU") << QString::fromUtf8("\320\262\320\276\321\201\320\272\321\200\320\265\321\201\320\265\320\275\321\214\320\265") << 7 << QLocale::LongFormat;
-    QTest::newRow("ru_RU short")  << QString("ru_RU") << QString::fromUtf8("\320\222\321\201") << 7 << QLocale::ShortFormat;
+    QTest::newRow("ru_RU short")  << QString("ru_RU") << QString::fromUtf8("\320\262\321\201") << 7 << QLocale::ShortFormat;
     QTest::newRow("ru_RU narrow")  << QString("ru_RU") << QString::fromUtf8("\320\222") << 7 << QLocale::NarrowFormat;
 }
 
@@ -2199,10 +2199,10 @@ void tst_QLocale::currency()
     QCOMPARE(en_US.toCurrencyString(double(-1234.56), NULL, 4), QString("$-1,234.5600"));
 
     const QLocale ru_RU("ru_RU");
-    QCOMPARE(ru_RU.toCurrencyString(qulonglong(1234)), QString::fromUtf8("1" "\xc2\xa0" "234\xc2\xa0\xd1\x80\xd1\x83\xd0\xb1."));
+    QCOMPARE(ru_RU.toCurrencyString(qulonglong(1234)), QString::fromUtf8("1" "\xc2\xa0" "234\xc2\xa0\xe2\x82\xbd"));
-    QCOMPARE(ru_RU.toCurrencyString(qlonglong(-1234)), QString::fromUtf8("-1" "\xc2\xa0" "234\xc2\xa0\xd1\x80\xd1\x83\xd0\xb1."));
+    QCOMPARE(ru_RU.toCurrencyString(qlonglong(-1234)), QString::fromUtf8("-1" "\xc2\xa0" "234\xc2\xa0\xe2\x82\xbd"));
-    QCOMPARE(ru_RU.toCurrencyString(double(1234.56)), QString::fromUtf8("1" "\xc2\xa0" "234,56\xc2\xa0\xd1\x80\xd1\x83\xd0\xb1."));
+    QCOMPARE(ru_RU.toCurrencyString(double(1234.56)), QString::fromUtf8("1" "\xc2\xa0" "234,56\xc2\xa0\xe2\x82\xbd"));
-    QCOMPARE(ru_RU.toCurrencyString(double(-1234.56)), QString::fromUtf8("-1" "\xc2\xa0" "234,56\xc2\xa0\xd1\x80\xd1\x83\xd0\xb1."));
+    QCOMPARE(ru_RU.toCurrencyString(double(-1234.56)), QString::fromUtf8("-1" "\xc2\xa0" "234,56\xc2\xa0\xe2\x82\xbd"));
 
     const QLocale de_DE("de_DE");
     QCOMPARE(de_DE.toCurrencyString(qulonglong(1234)), QString::fromUtf8("1.234\xc2\xa0\xe2\x82\xac"));
@@ -2362,14 +2362,20 @@ void tst_QLocale::textDirection_data()
         case QLocale::CentralKurdish:
         case QLocale::ClassicalMandaic:
         case QLocale::Divehi:
+//        case QLocale::Fulah:
+//        case QLocale::Hausa:
         case QLocale::Hebrew:
+//        case QLocale::Hungarian:
         case QLocale::Kashmiri:
+//        case QLocale::Kurdish:
         case QLocale::Lydian:
         case QLocale::Mandingo:
         case QLocale::ManichaeanMiddlePersian:
+        case QLocale::Mazanderani:
         case QLocale::Mende:
         case QLocale::Meroitic:
         case QLocale::Nko:
+        case QLocale::NorthernLuri:
         case QLocale::OldTurkish:
         case QLocale::Pahlavi:
         case QLocale::Parthian:
@@ -2383,6 +2389,7 @@ void tst_QLocale::textDirection_data()
         case QLocale::Syriac:
         case QLocale::Uighur:
         case QLocale::Urdu:
+        case QLocale::Yiddish:
             rightToLeft = QLocale(QLocale::Language(language)).language() == QLocale::Language(language); // false if there is no locale data for language
             break;
         default:

util/local_database/enumdata.py

@@ -371,7 +371,27 @@ language_list = {
     336 : [ "Bassa",              "bsq" ],
     337 : [ "Mono",               "mru" ],
     338 : [ "Tedim Chin",         "ctd" ],
-    339 : [ "Maithili",           "mai" ]
+    339 : [ "Maithili",           "mai" ],
+    340 : [ "Ahom",               "aho" ],
+    341 : [ "American Sign Language", "ase" ],
+    342 : [ "Ardhamagadhi Prakrit", "pka" ],
+    343 : [ "Bhojpuri",           "bho" ],
+    344 : [ "Hieroglyphic Luwian", "hlu" ],
+    345 : [ "Literary Chinese",   "lzh" ],
+    346 : [ "Mazanderani",        "mzn" ],
+    347 : [ "Mru",                "mro" ],
+    348 : [ "Newari",             "new" ],
+    349 : [ "Northern Luri",      "lrc" ],
+    350 : [ "Palauan",            "pau" ],
+    351 : [ "Papiamento",         "pap" ],
+    352 : [ "Saraiki",            "skr" ],
+    353 : [ "Tokelau",            "tkl" ],
+    354 : [ "Tok Pisin",          "tpi" ],
+    355 : [ "Tuvalu",             "tvl" ],
+    356 : [ "UncodedLanguages",   "mis" ],
+    357 : [ "Cantonese",          "yue" ],
+    358 : [ "Osage",              "osa" ],
+    359 : [ "Tangut",             "txg" ]
 }
 
 country_list = {
@@ -632,7 +652,9 @@ country_list = {
     254 : [ "SouthSudan",                               "SS"  ],
     255 : [ "Bonaire",                                  "BQ"  ],
     256 : [ "SintMaarten",                              "SX"  ],
-    257 : [ "Kosovo",                                   "XK"  ]
+    257 : [ "Kosovo",                                   "XK"  ],
+    258 : [ "European Union",                           "EU"  ],
+    259 : [ "Outlying Oceania",                         "QO"  ]
 }
 
 script_list = {
@@ -763,10 +785,21 @@ script_list = {
     124 : [ "Siddham",                "Sidd" ],
     125 : [ "Khudawadi",              "Sind" ],
     126 : [ "Tirhuta",                "Tirh" ],
-    127 : [ "Varang Kshiti",          "Wara" ]
+    127 : [ "Varang Kshiti",          "Wara" ],
-
+    128 : [ "Ahom",                   "Ahom" ],
-#    ### : [ "Blissymbols",            "Blis" ],
+    129 : [ "Anatolian Hieroglyphs",  "Hluw" ],
-#    ### : [ "Naxi Geba",              "Nkgb" ],
+    130 : [ "Hatran",                 "Hatr" ],
+    131 : [ "Multani",                "Mult" ],
+    132 : [ "Old Hungarian",          "Hung" ],
+    133 : [ "SignWriting",            "Sgnw" ],
+    134 : [ "Adlam",                  "Adlm" ],
+    135 : [ "Bhaiksuki",              "Bhks" ],
+    136 : [ "Marchen",                "Marc" ],
+    137 : [ "Newa",                   "Newa" ],
+    138 : [ "Osage",                  "Osge" ],
+    139 : [ "Tangut",                 "Tang" ],
+    140 : [ "Han With Bopomofo",      "Hanb" ],
+    141 : [ "Jamo",                   "Jamo" ]
 }
 
 def countryCodeToId(code):

util/local_database/qlocalexml2cpp.py

@@ -552,7 +552,7 @@ def main():
             cmnt_to = cmnt_to + country_map[to_country][1]
 
         data_temp_file.write("    ")
-        data_temp_file.write("{ %3d, %2d, %3d }, { %3d, %2d, %3d }" % (from_language, from_script, from_country, to_language, to_script, to_country))
+        data_temp_file.write("{ %3d, %3d, %3d }, { %3d, %3d, %3d }" % (from_language, from_script, from_country, to_language, to_script, to_country))
         index += 1
         if index != len(likely_subtags_map):
             data_temp_file.write(",")
@@ -872,6 +872,7 @@ def main():
         language = fixedLanguageName(language_map[key][0], dupes)
         qlocaleh_temp_file.write("        " + language + " = " + str(key) + ",\n")
     # legacy. should disappear at some point
+    qlocaleh_temp_file.write("\n")
     qlocaleh_temp_file.write("        Norwegian = NorwegianBokmal,\n")
     qlocaleh_temp_file.write("        Moldavian = Romanian,\n")
     qlocaleh_temp_file.write("        SerboCroatian = Serbian,\n")
@@ -887,6 +888,7 @@ def main():
     qlocaleh_temp_file.write("        Chewa = Nyanja,\n")
     qlocaleh_temp_file.write("        Frisian = WesternFrisian,\n")
     qlocaleh_temp_file.write("        Uigur = Uighur,\n")
+    qlocaleh_temp_file.write("\n")
     qlocaleh_temp_file.write("        LastLanguage = " + language + "\n")
     qlocaleh_temp_file.write("    };\n")
 
@@ -899,8 +901,10 @@ def main():
         script = fixedScriptName(script_map[key][0], dupes)
         qlocaleh_temp_file.write("        " + script + " = " + str(key) + ",\n")
     # renamings
+    qlocaleh_temp_file.write("\n")
     qlocaleh_temp_file.write("        SimplifiedChineseScript = SimplifiedHanScript,\n")
     qlocaleh_temp_file.write("        TraditionalChineseScript = TraditionalHanScript,\n")
+    qlocaleh_temp_file.write("\n")
     qlocaleh_temp_file.write("        LastScript = " + script + "\n")
     qlocaleh_temp_file.write("    };\n")
 
@@ -910,6 +914,10 @@ def main():
     for key in country_map.keys():
         country = fixedCountryName(country_map[key][0], dupes)
         qlocaleh_temp_file.write("        " + country + " = " + str(key) + ",\n")
+    # deprecated
+    qlocaleh_temp_file.write("\n")
+    qlocaleh_temp_file.write("        Tokelau = TokelauCountry,\n")
+    qlocaleh_temp_file.write("        Tuvalu = TuvaluCountry,\n")
     # renamings
     qlocaleh_temp_file.write("        DemocraticRepublicOfCongo = CongoKinshasa,\n")
     qlocaleh_temp_file.write("        PeoplesRepublicOfCongo = CongoBrazzaville,\n")
@@ -917,6 +925,7 @@ def main():
     qlocaleh_temp_file.write("        RepublicOfKorea = SouthKorea,\n")
     qlocaleh_temp_file.write("        RussianFederation = Russia,\n")
     qlocaleh_temp_file.write("        SyrianArabRepublic = Syria,\n")
+    qlocaleh_temp_file.write("\n")
     qlocaleh_temp_file.write("        LastCountry = " + country + "\n")
     qlocaleh_temp_file.write("    };\n")