Skip to content

Commit

Permalink
Merge pull request #51 from cquiroz/special-root
Browse files Browse the repository at this point in the history 
Fixes setting the root in some special cases
  • Loading branch information
@cquiroz
cquiroz authored Jul 23, 2016
2 parents 6050e0b + c79b9bd commit 92cb107
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 42 deletions.
71 changes: 47 additions & 24 deletions project/src/main/scala/locales/ScalaLocaleCodeGen.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ object NumberSymbols {
case class XMLLDMLLocale(language: String, territory: Option[String],
variant: Option[String], script: Option[String])

case class XMLLDML(locale: XMLLDMLLocale, defaultNS: Option[NumericSystem],
case class XMLLDML(locale: XMLLDMLLocale, fileName: String, defaultNS: Option[NumericSystem],
digitSymbols: Map[NumericSystem, NumberSymbols], calendar: Option[CalendarSymbols],
datePatterns: Option[CalendarPatterns]) {

Expand All @@ -89,16 +89,16 @@ object CodeGenerator {
val autoGeneratedCommend = "Auto-generated code from CLDR definitions, don't edit"

def buildClassTree(packageObject: String, ldmls: List[XMLLDML],
only: List[String]): Tree = {
only: List[String], parentLocales: Map[String, List[String]]): Tree = {
val langs = ldmls.map(_.scalaSafeName.split("_").toList)
// Root must always be available
val root = ldmls.find(_.scalaSafeName == "root").get

val objectBlock = if (only.nonEmpty) {
ldmls.filter(a => only.contains(a.scalaSafeName))
.map(buildClassTree(root, langs))
.map(buildClassTree(root, langs, parentLocales))
} else {
ldmls.map(buildClassTree(root, langs))
ldmls.map(buildClassTree(root, langs, parentLocales))
}

BLOCK (
Expand All @@ -112,27 +112,28 @@ object CodeGenerator {
}

def findParent(root: XMLLDML, langs: List[List[String]],
ldml: XMLLDML): Option[String] = {
ldml: XMLLDML, parentLocales: Map[String, List[String]]): Option[String] = {
// http://www.unicode.org/reports/tr35/#Locale_Inheritance

// This searches based on the simple hierarchy resolution based on bundle_name
// http://www.unicode.org/reports/tr35/#Bundle_vs_Item_Lookup
ldml.scalaSafeName.split("_").reverse.toList match {
case x :: Nil if x == root.scalaSafeName => None
case x :: Nil => Some(root.scalaSafeName)
case x :: xs if langs.contains(xs.reverse) => Some(xs.reverse.mkString("_"))
}
parentLocales.find(_._2.contains(ldml.fileName)).fold(
// This searches based on the simple hierarchy resolution based on bundle_name
// http://www.unicode.org/reports/tr35/#Bundle_vs_Item_Lookup
ldml.scalaSafeName.split("_").reverse.toList match {
case x :: Nil if x == root.scalaSafeName => None
case x :: Nil => Some(root.scalaSafeName)
case x :: xs if langs.contains(xs.reverse) => Some(xs.reverse.mkString("_"))
}
)(p => Some(p._1))
}

def buildClassTree(root: XMLLDML, langs: List[List[String]])
def buildClassTree(root: XMLLDML, langs: List[List[String]], parentLocales: Map[String, List[String]])
(ldml: XMLLDML): Tree = {
val ldmlSym = getModule("LDML")
val ldmlNumericSym = getModule("Symbols")
val ldmlCalendarSym = getModule("CalendarSymbols")
val ldmlCalendarPatternsSym = getModule("CalendarPatterns")
val ldmlLocaleSym = getModule("LDMLLocale")

val parent = findParent(root, langs, ldml).fold(NONE)(v => SOME(REF(v)))
val parent = findParent(root, langs, ldml, parentLocales).fold(NONE)(v => SOME(REF(v)))

val ldmlLocaleTree = Apply(ldmlLocaleSym, LIT(ldml.locale.language),
ldml.locale.territory.fold(NONE)(t => SOME(LIT(t))),
Expand Down Expand Up @@ -232,8 +233,6 @@ object ScalaLocaleCodeGen {
path.getParent.toFile.mkdirs()
println(s"Write to $path")

//println(treehugger.forest.treeToString(tree))

Files.write(path, treehugger.forest.treeToString(tree)
.getBytes(Charset.forName("UTF8")))
path.toFile
Expand Down Expand Up @@ -405,7 +404,10 @@ object ScalaLocaleCodeGen {
}
nsSymbols
}
XMLLDML(XMLLDMLLocale(language, territory, variant, script),

val fileName = f.getName.substring(0, f.getName.lastIndexOf("."))

XMLLDML(XMLLDMLLocale(language, territory, variant, script), fileName,
defaultNS.flatMap(ns.get), symbols.toMap, gregorian.flatten.headOption, gregorianDatePatterns.flatten.headOption)
}

Expand Down Expand Up @@ -458,9 +460,29 @@ object ScalaLocaleCodeGen {

def readCalendars(data: File): Seq[Calendar] = {
// Parse the numeric systems
val calendarsSupplementaldata = data.toPath.resolve("common")
val calendarsSupplementalData = data.toPath.resolve("common")
.resolve("supplemental").resolve("supplementalData.xml").toFile
parseCalendars(XML.withSAXParser(parser).loadFile(calendarsSupplementalData))
}

def parseParentLocales(xml: Elem): Map[String, List[String]] = {
val ns = xml \ "parentLocales" \\ "parentLocale"

val p = for {
n <- ns
} yield {
val parent = (n \ "@parent").text
val locales = (n \ "@locales").text
parent -> locales.split("\\s").toList
}
p.toMap
}

def readParentLocales(data: File): Map[String, List[String]] = {
// Parse the parent locales
val parentLocalesSupplementalData = data.toPath.resolve("common")
.resolve("supplemental").resolve("supplementalData.xml").toFile
parseCalendars(XML.withSAXParser(parser).loadFile(calendarsSupplementaldata))
parseParentLocales(XML.withSAXParser(parser).loadFile(parentLocalesSupplementalData))
}

def generateCalendarsFile(base: File, calendars: Seq[Calendar]): File = {
Expand All @@ -470,7 +492,7 @@ object ScalaLocaleCodeGen {
}

def buildLDMLDescriptors(data: File, numericSystemsMap: Map[String, NumericSystem],
latnNS: NumericSystem): List[XMLLDML] = {
latnNS: NumericSystem): List[XMLLDML] = {
// All files under common/main
val files = Files.newDirectoryStream(data.toPath.resolve("common")
.resolve("main")).iterator().asScala.toList
Expand All @@ -483,11 +505,11 @@ object ScalaLocaleCodeGen {
latnNS, numericSystemsMap)
}

def generateLocalesFile(base: File, clazzes: List[XMLLDML]): File = {
def generateLocalesFile(base: File, clazzes: List[XMLLDML], parentLocales: Map[String, List[String]]): File = {
val names = clazzes.map(_.scalaSafeName)

// Generate locales code
val stdTree = CodeGenerator.buildClassTree("data", clazzes, names)
val stdTree = CodeGenerator.buildClassTree("data", clazzes, names, parentLocales)
writeGeneratedTree(base, "data", stdTree)
}

Expand All @@ -508,6 +530,7 @@ object ScalaLocaleCodeGen {
val f1 = generateNumericSystemsFile(base, numericSystems)

val calendars = readCalendars(data)
val parentLocales = readParentLocales(data)
val f2 = generateCalendarsFile(base, calendars)

val numericSystemsMap: Map[String, NumericSystem] =
Expand All @@ -518,7 +541,7 @@ object ScalaLocaleCodeGen {
val ldmls = buildLDMLDescriptors(data, numericSystemsMap, latnNS)

val f3 = generateMetadataFile(base, ldmls)
val f4 = generateLocalesFile(base, ldmls)
val f4 = generateLocalesFile(base, ldmls, parentLocales)

println("Generation took " + (System.nanoTime() - nanos) / 1000000 + " [ms]")
Seq(f1, f2, f3, f4)
Expand Down
13 changes: 6 additions & 7 deletions testSuite/shared/src/test/scala/testsuite/javalib/text/DateFormatSymbolsTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,13 +315,12 @@ class DateFormatSymbolsTest extends LocaleTestSetup {
LocaleTestItem(az_Cyrl, "az-Cyrl", cldr21 = false,
List("јанвар", "феврал", "март", "апрел", "май", "ијун", "ијул", "август",
"сентјабр", "октјабр", "нојабр", "декабр", ""),
List("yan", "fev", "mar", "apr", "may", "iyn", "iyl", "avq", "sen", "okt",
"noy", "dek", ""),
List("", "", "", "", "", "", "", "", "", "", "", "", ""),
List("", "базар", "базар ертәси", "чәршәнбә ахшамы", "чәршәнбә", "ҹүмә ахшамы",
"ҹүмә", "шәнбә"),
List("", "B.", "B.E.", "Ç.A.", "Ç.", "C.A.", "C.", "Ş."),
List("AM", "PM"),
List("e.ə.", "b.e.")),
List("", "", "", "", "", "", "", ""),
List("", ""),
List("BCE", "CE")),
LocaleTestItem(bn, "bn", cldr21 = true,
List("জানুয়ারী", "ফেব্রুয়ারী", "মার্চ", "এপ্রিল", "মে", "জুন", "জুলাই", "আগস্ট", "সেপ্টেম্বর",
"অক্টোবর", "নভেম্বর", "ডিসেম্বর", ""),
Expand Down Expand Up @@ -351,11 +350,11 @@ class DateFormatSymbolsTest extends LocaleTestSetup {
LocaleTestItem(es_CL, "es-CL", cldr21 = false,
List("enero", "febrero", "marzo", "abril", "mayo", "junio", "julio", "agosto",
"septiembre", "octubre", "noviembre", "diciembre", ""),
List("ene.", "feb.", "mar.", "abr.", "may.", "jun.", "jul.", "ago.", "sept.",
List("ene.", "feb.", "mar.", "abr.", "may.", "jun.", "jul.", "ago.", "sep.",
"oct.", "nov.", "dic.", ""),
List("", "domingo", "lunes", "martes", "miércoles", "jueves", "viernes", "sábado"),
List("", "dom.", "lun.", "mar.", "mié.", "jue.", "vie.", "sáb."),
List("a. m.", "p. m."),
List("a.m.", "p.m."),
List("a. C.", "d. C.")),
LocaleTestItem(it_CH, "it-CH", cldr21 = true,
List("gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio",
Expand Down
18 changes: 9 additions & 9 deletions testSuite/shared/src/test/scala/testsuite/javalib/text/DateFormatTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import locales.LocaleRegistry
import locales.cldr.LDML
import locales.cldr.data._
import org.junit.Assert._
import org.junit.Test
import org.junit.{Ignore, Test}
import testsuite.utils.{LocaleTestSetup, Platform}

class DateFormatTest extends LocaleTestSetup {
Expand Down Expand Up @@ -408,10 +408,10 @@ class DateFormatTest extends LocaleTestSetup {
DateFormat.MEDIUM -> "HH:mm:ss",
DateFormat.SHORT -> "HH:mm")),
TestCase(root, "", Locale.UK, cldr21 = false, Map(
DateFormat.FULL -> "EEEE, MMMM d, y",
DateFormat.LONG -> "MMMM d, y",
DateFormat.MEDIUM -> "MMM d, y",
DateFormat.SHORT -> "M/d/yy"),
DateFormat.FULL -> "EEEE, d MMMM y",
DateFormat.LONG -> "d MMMM y",
DateFormat.MEDIUM -> "d MMM y",
DateFormat.SHORT -> "dd/MM/y"),
Map(
DateFormat.FULL -> "HH:mm:ss zzzz",
DateFormat.LONG -> "HH:mm:ss z",
Expand Down Expand Up @@ -561,10 +561,10 @@ class DateFormatTest extends LocaleTestSetup {
DateFormat.MEDIUM -> "dd-MM-y",
DateFormat.SHORT -> "dd-MM-yy"),
Map(
DateFormat.FULL -> "H:mm:ss (zzzz)",
DateFormat.LONG -> "H:mm:ss z",
DateFormat.MEDIUM -> "H:mm:ss",
DateFormat.SHORT -> "H:mm")),
DateFormat.FULL -> "HH:mm:ss zzzz",
DateFormat.LONG -> "HH:mm:ss z",
DateFormat.MEDIUM -> "HH:mm:ss",
DateFormat.SHORT -> "HH:mm")),
TestCase(it_CH, "it-CH", Locale.ROOT, cldr21 = true, Map( // JVM
DateFormat.FULL -> "EEEE, d MMMM y",
DateFormat.LONG -> "d MMMM y",
Expand Down
10 changes: 8 additions & 2 deletions testSuite/shared/src/test/scala/testsuite/locales/LocaleRegistryTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,14 @@ class LocaleRegistryTest extends LocaleTestSetup {
LocaleTestCase(am, "am", "am", "am", hasExtensions = false, Some(root)),
LocaleTestCase(ar_001, "ar-001", "ar_001", "ar-001", hasExtensions = false, Some(ar)),
LocaleTestCase(zh_Hant_TW, "zh-TW", "zh_TW", "zh-TW", hasExtensions = false, Some(zh_Hant)),
LocaleTestCase(az_Cyrl, "az-Cyrl", "az__#Cyrl", "az-Cyrl", hasExtensions = false, Some(az)),
LocaleTestCase(az_Cyrl_AZ, "az-AZ-Cyrl", "az_AZ", "az-AZ", hasExtensions = false, Some(az_Cyrl))
// Locales with special roots
LocaleTestCase(az_Cyrl, "az-Cyrl", "az__#Cyrl", "az-Cyrl", hasExtensions = false, Some(root)),
LocaleTestCase(az_Cyrl_AZ, "az-AZ-Cyrl", "az_AZ", "az-AZ", hasExtensions = false, Some(az_Cyrl)),
LocaleTestCase(es_CL, "es-CL", "es_CL", "es-CL", hasExtensions = false, Some(es_419)), // es_CL parent is es_419 instead of es
LocaleTestCase(en_BS, "en-BS", "en_BS", "en-BS", hasExtensions = false, Some(en_001)), // en_BS parent is en_001 instead of en
LocaleTestCase(en_SI, "en-SI", "en_SI", "en-SI", hasExtensions = false, Some(en_150)), // en_SI parent is en_150 instead of en
LocaleTestCase(pt_MZ, "pt-MZ", "pt_MZ", "pt-MZ", hasExtensions = false, Some(pt_PT)),
LocaleTestCase(zh_Hant_MO, "zh-Hant-MO", "zh_MO_#Hant", "zh-Hant-MO", hasExtensions = false, Some(zh_Hant_HK))
)

@Test def test_install(): Unit = {
Expand Down

0 comments on commit 92cb107

Please sign in to comment.