Dictionary lookup window: use from/to language to determine glyph form (#10184)

For example, to determine whether the Japanese or Chinese form should be shown.

* Dictionaries.lua - use iso3 codes everywhere
readerdictionary.lua - convert iso to bcp tag, to construct ifo_lang
isolanguage.lua - map from iso3 to full language name, map from iso3 to bcp language tag

* Make the full language names translatable

* Store ifo information from dictionaries.lua in downloaded dictionaries

frontend/apps/reader/modules/readerdictionary.lua

@@ -136,6 +136,7 @@ function ReaderDictionary:init()
                 f:close()
                 local dictname = content:match("\nbookname=(.-)\r?\n")
                 local is_html = content:find("sametypesequence=h", 1, true) ~= nil
+                local lang_in, lang_out = content:match("lang=(%a+)-?(%a*)\r?\n?")
                 -- sdcv won't use dict that don't have a bookname=
                 if dictname then
                     table.insert(available_ifos, {
@@ -144,6 +145,7 @@ function ReaderDictionary:init()
                         is_html = is_html,
                         css = readDictionaryCss(ifo_file:gsub("%.ifo$", ".css")),
                         fix_html_func = getDictionaryFixHtmlFunc(ifo_file:gsub("%.ifo$", ".lua")),
+                        lang = lang_in and { lang_in = lang_in, lang_out = lang_out },
                     })
                 end
             end
@@ -265,7 +267,7 @@ function ReaderDictionary:addToMainMenu(menu_items)
             },
             {
                 text = _("Download dictionaries"),
-                sub_item_table = self:_genDownloadDictionariesMenu()
+                sub_item_table_func = function() return self:_genDownloadDictionariesMenu() end,
             },
             {
                 text = _("Enable fuzzy search"),
@@ -477,10 +479,19 @@ end
 
 function ReaderDictionary:_genDownloadDictionariesMenu()
     local downloadable_dicts = require("ui/data/dictionaries")
+    local IsoLanguage = require("ui/data/isolanguage")
     local languages = {}
 
     for i = 1, #downloadable_dicts do
         local dict = downloadable_dicts[i]
+        if not dict.ifo_lang then
+            -- this only needs to happen the first time this function is called
+            local ifo_in = IsoLanguage:getBCPLanguageTag(dict.lang_in)
+            local ifo_out = IsoLanguage:getBCPLanguageTag(dict.lang_out)
+            dict.ifo_lang = ("%s-%s"):format(ifo_in, ifo_out)
+            dict.lang_in = IsoLanguage:getLocalizedLanguage(dict.lang_in)
+            dict.lang_out = IsoLanguage:getLocalizedLanguage(dict.lang_out)
+        end
         local dict_lang_in = dict.lang_in
         local dict_lang_out = dict.lang_out
         if not languages[dict_lang_in] then
@@ -593,6 +604,9 @@ local function tidyMarkup(results)
     local format_escape = "&[29Ib%+]{(.-)}"
     for _, result in ipairs(results) do
         local ifo = getAvailableIfoByName(result.dict)
+        if ifo and ifo.lang then
+            result.ifo_lang = ifo.lang
+        end
         if ifo and ifo.is_html then
             result.is_html = ifo.is_html
             result.css = ifo.css
@@ -1118,9 +1132,15 @@ function ReaderDictionary:downloadDictionary(dict, download_location, continue)
         return false
     end
 
-    local ok, error = Device:unpackArchive(download_location, self.data_dir)
+    -- stable target directory is needed so we can look through the folder later
+    local dict_path = self.data_dir .. "/" .. dict.name
+    util.makePath(dict_path)
+    local ok, error = Device:unpackArchive(download_location, dict_path, true)
 
     if ok then
+        if dict.ifo_lang then
+            self:extendIfoWithLanguage(dict_path, dict.ifo_lang)
+        end
         available_ifos = false
         self:init()
         UIManager:show(InfoMessage:new{
@@ -1135,6 +1155,24 @@ function ReaderDictionary:downloadDictionary(dict, download_location, continue)
     end
 end
 
+function ReaderDictionary:extendIfoWithLanguage(dictionary_location, ifo_lang)
+    local function cb(path, filename)
+        if util.getFileNameSuffix(filename) == "ifo" then
+            local fmt_string = "lang=%s"
+            local f = io.open(path, "a+")
+            if f then
+                local ifo = f:read("a*")
+                if ifo[#ifo] ~= "\n" then
+                    fmt_string = "\n" .. fmt_string
+                end
+                f:write(fmt_string:format(ifo_lang))
+                f:close()
+            end
+        end
+    end
+    util.findFiles(dictionary_location, cb)
+end
+
 function ReaderDictionary:onReadSettings(config)
     self.preferred_dictionaries = config:readSetting("preferred_dictionaries") or {}
     if #self.preferred_dictionaries == 0 then

frontend/device/generic/device.lua

@@ -904,13 +904,14 @@ end
 -- filename extension. Inspired by luarocks archive_unpack()
 -- @param archive string: Filename of archive.
 -- @param extract_to string: Destination directory.
+-- @param with_stripped_root boolean: true if root directory in archive should be stripped
 -- @return boolean or (boolean, string): true on success, false and an error message on failure.
-function Device:unpackArchive(archive, extract_to)
+function Device:unpackArchive(archive, extract_to, with_stripped_root)
     require("dbg").dassert(type(archive) == "string")
     local BD = require("ui/bidi")
     local ok
     if archive:match("%.tar%.bz2$") or archive:match("%.tar%.gz$") or archive:match("%.tar%.lz$") or archive:match("%.tgz$") then
-        ok = self:untar(archive, extract_to)
+        ok = self:untar(archive, extract_to, with_stripped_root)
     else
         return false, T(_("Couldn't extract archive:\n\n%1\n\nUnrecognized filename extension."), BD.filepath(archive))
     end
@@ -920,8 +921,12 @@ function Device:unpackArchive(archive, extract_to)
     return true
 end
 
-function Device:untar(archive, extract_to)
-    return os.execute(("./tar xf %q -C %q"):format(archive, extract_to))
+function Device:untar(archive, extract_to, with_stripped_root)
+    local cmd = "./tar xf %q -C %q"
+    if with_stripped_root then
+        cmd = cmd .. " --strip-components=1"
+    end
+    return os.execute(cmd:format(archive, extract_to))
 end
 
 -- Set device event handlers common to all devices

frontend/ui/data/isolanguage.lua

@@ -0,0 +1,342 @@
+local _ = require("gettext")
+
+local locale_lang_map = {
+    -- from https://iso639-3.sil.org/sites/iso639-3/files/downloads/iso-639-3.tab
+    -- first chunk of names are languages present in iso-639-1
+    -- @translators Most of these language name have already been translated at <https://hosted.weblate.org/projects/iso-codes/iso-639-2/>. Click "Automatic suggestions" to see them below the textfield.
+    aar = _("Afar"),
+    afr = _("Afrikaans"),
+    ara = _("Arabic"), -- macrolanguage
+    asm = _("Assamese"),
+    aze = _("Azerbaijani"), -- macrolanguage
+    bak = _("Bashkir"),
+    bel = _("Belarusian"),
+    ben = _("Bengali"),
+    bod = _("Tibetan"),
+    bre = _("Breton"),
+    bul = _("Bulgarian"),
+    ces = _("Czech"),
+    chu = _("Church Slavic"),
+    cor = _("Cornish"),
+    cym = _("Welsh"),
+    dan = _("Danish"),
+    deu = _("German"),
+    div = _("Dhivehi"),
+    ell = _("Modern Greek"), -- (1453-)
+    eng = _("English"),
+    epo = _("Esperanto"), -- constructed language
+    est = _("Estonian"), -- macrolanguage
+    eus = _("Basque"),
+    fao = _("Faroese"),
+    fas = _("Persian"), -- macrolanguage
+    fin = _("Finnish"),
+    fra = _("French"),
+    fry = _("Western Frisian"),
+    gla = _("Scottish Gaelic"),
+    gle = _("Irish"),
+    glg = _("Galician"),
+    glv = _("Manx"),
+    guj = _("Gujarati"),
+    hat = _("Haitian"),
+    hau = _("Hausa"),
+    hbs = _("Serbo-Croatian"), -- macrolanguage
+    heb = _("Hebrew"),
+    hin = _("Hindi"),
+    hun = _("Hungarian"),
+    hye = _("Armenian"),
+    ido = _("Ido"), -- constructed language
+    ina = _("Interlingua"), -- (International Auxiliary Language Association) constructed language
+    ind = _("Indonesian"),
+    isl = _("Icelandic"),
+    ita = _("Italian"),
+    jav = _("Javanese"),
+    jpn = _("Japanese"),
+    kal = _("Kalaallisut"),
+    kan = _("Kannada"),
+    kas = _("Kashmiri"),
+    kat = _("Georgian"),
+    kaz = _("Kazakh"),
+    khm = _("Khmer"),
+    kik = _("Kikuyu"),
+    kir = _("Kirghiz"),
+    kor = _("Korean"),
+    kur = _("Kurdish"), -- macrolanguage
+    lao = _("Lao"),
+    lat = _("Latin"),
+    lav = _("Latvian"), -- macrolanguage
+    lit = _("Lithuanian"),
+    ltz = _("Luxembourgish"),
+    mah = _("Marshallese"),
+    mal = _("Malayalam"),
+    mar = _("Marathi"),
+    mkd = _("Macedonian"),
+    mlg = _("Malagasy"), -- macrolanguage
+    mlt = _("Maltese"),
+    mon = _("Mongolian"), -- macrolanguage
+    mri = _("Maori"),
+    msa = _("Malay"), -- macrolanguage
+    mya = _("Burmese"),
+    nav = _("Navajo"),
+    nld = _("Dutch"),
+    nno = _("Norwegian Nynorsk"),
+    nob = _("Norwegian Bokmål"),
+    nor = _("Norwegian"), -- macrolanguage
+    oci = _("Occitan"), -- (post 1500)
+    orm = _("Oromo"), -- macrolanguage
+    pan = _("Panjabi"),
+    pli = _("Pali"),
+    pol = _("Polish"),
+    por = _("Portuguese"),
+    pus = _("Pushto"), -- macrolanguage
+    que = _("Quechua"), -- macrolanguage
+    roh = _("Romansh"),
+    ron = _("Romanian"),
+    rus = _("Russian"),
+    san = _("Sanskrit"),
+    slk = _("Slovak"),
+    slv = _("Slovenian"),
+    spa = _("Spanish"),
+    sqi = _("Albanian"), -- macrolanguage
+    srp = _("Serbian"),
+    ssw = _("Swati"),
+    sun = _("Sundanese"),
+    swa = _("Swahili"), -- macrolanguage
+    swe = _("Swedish"),
+    tam = _("Tamil"),
+    tat = _("Tatar"),
+    tel = _("Telugu"),
+    tgk = _("Tajik"),
+    tgl = _("Tagalog"),
+    tha = _("Thai"),
+    tur = _("Turkish"),
+    uig = _("Uighur"),
+    ukr = _("Ukrainian"),
+    urd = _("Urdu"),
+    uzb = _("Uzbek"), -- macrolanguage
+    vie = _("Vietnamese"),
+    vol = _("Volapük"), -- constructed language
+    wln = _("Walloon"),
+    xho = _("Xhosa"),
+    yid = _("Yiddish"), -- macrolanguage
+    yor = _("Yoruba"),
+    zha = _("Zhuang"), -- macrolanguage
+    zho = _("Chinese"), -- macrolanguage
+    zul = _("Zulu"),
+
+    -- second chunk of names are languages present in iso-639-2
+    -- this is only as complete as it needs to be to cover the languages in dictionaries.lua
+    -- @translators Most of these language name have already been translated at <https://hosted.weblate.org/projects/iso-codes/iso-639-2/>. Click "Automatic suggestions" to see them below the textfield.
+    ady = _("Adyghe"),
+    alt = _("Southern Altai"),
+    ang = _("Old English"), -- (ca. 450-1100)
+    arc = _("Official Aramaic"), -- (700-300 BCE)
+    ast = _("Asturian"),
+    bik = _("Bikol"), -- macrolanguage
+    ceb = _("Cebuano"),
+    chr = _("Cherokee"),
+    cop = _("Coptic"),
+    crh = _("Crimean Tatar"),
+    dsb = _("Lower Sorbian"),
+    dum = _("Middle Dutch"), -- (ca. 1050-1350)
+    enm = _("Middle English"), --  (1100-1500)
+    frm = _("Middle French"), -- (ca. 1400-1600)
+    fro = _("Old French"), -- (842-ca. 1400)
+    fur = _("Friulian"),
+    goh = _("Old High German"), -- (ca. 750-1050)
+    got = _("Gothic"),
+    grc = _("Ancient Greek"), -- (to 1453)
+    gsw = _("Swiss German"),
+    haw = _("Hawaiian"),
+    hil = _("Hiligaynon"),
+    kum = _("Kumyk"),
+    lad = _("Ladino"),
+    mnc = _("Manchu"),
+    nds = _("Low German"),
+    non = _("Old Norse"),
+    ota = _("Ottoman Turkish"), -- (1500-1928)
+    rup = _("Macedo-Romanian"),
+    sah = _("Yakut"),
+    scn = _("Sicilian"),
+    sco = _("Scots"),
+    sga = _("Old Irish"), -- (to 900)
+    shn = _("Shan"),
+    sux = _("Sumerian"),
+    syc = _("Classical Syriac"),
+    tpi = _("Tok Pisin"),
+
+    -- third chunk of names are only present in iso639-3 standard
+    -- this is only as complete as it needs to be to cover the languages in dictionaries.lua
+    -- @translators Most of these language name have already been translated at <https://hosted.weblate.org/projects/iso-codes/iso-639-2/>. Click "Automatic suggestions" to see them below the textfield.
+    aii = _("Assyrian Neo-Aramaic"),
+    apc = _("Levantine Arabic"),
+    ary = _("Moroccan Arabic"),
+    bnt = _("Bantu"),
+    cim = _("Cimbrian"),
+    cmn = _("Mandarin Chinese"),
+    dlm = _("Dalmatian"),
+    hrx = _("Hunsrik"),
+    izh = _("Ingrian"),
+    jje = _("Jejueo"),
+    kea = _("Kabuverdianu"),
+    lld = _("Ladin"),
+    lzz = _("Laz"),
+    mch = _("Maquiritari"),
+    nch = _("Central Huasteca Nahuatl"),
+    nci = _("Classical Nahuatl"),
+    nrf = _("Jèrriais"), -- Norman
+    pdc = _("Pennsylvania German"),
+    pdt = _("Plautdietsch"),
+    phl = _("Phalura"),
+    stq = _("Saterfriesisch"), -- Saterland Frisian
+    tft = _("Ternate"),
+    txb = _("Tokharian B"),
+    txg = _("Tangut"),
+    vec = _("Venetian"),
+    vep = _("Veps"),
+    wym = _("Wymysorys"), -- Vilamovian
+    xcl = _("Classical Armenian"),
+    yol = _("Yola"),
+    yue = _("Yue Chinese"),
+
+    -- these are made up language codes for very specific languages which do not map to a specific ISO3 code
+    xxpf = _("Proto-Finnic"),
+    xxpg = _("Proto-Germanic"),
+    xxpw = _("Proto-West Germanic"),
+    xxps = _("Proto-Sami"),
+    xxsl = _("Proto-Slavic"),
+}
+
+local iso_bcp47_map = {
+    aar = "aa",
+    afr = "af",
+    ara = "ar",
+    asm = "as",
+    aze = "az",
+    bak = "ba",
+    bel = "be",
+    bul = "bg",
+    ben = "bn",
+    bod = "bo",
+    bre = "br",
+    ces = "cs",
+    chu = "cu",
+    cym = "cy",
+    dan = "da",
+    deu = "de",
+    div = "dv",
+    ell = "el",
+    eng = "en",
+    epo = "eo",
+    spa = "es",
+    est = "et",
+    eus = "eu",
+    fas = "fa",
+    fin = "fi",
+    fao = "fo",
+    fra = "fr",
+    fry = "fy",
+    gle = "ga",
+    gla = "gd",
+    glg = "gl",
+    glv = "gv",
+    hau = "ha",
+    heb = "he",
+    hin = "hi",
+    hat = "ht",
+    hun = "hu",
+    hye = "hy",
+    ina = "ia",
+    ind = "id",
+    ido = "io",
+    isl = "is",
+    ita = "it",
+    jpn = "ja",
+    jav = "jv",
+    kat = "ka",
+    kik = "ki",
+    kaz = "kk",
+    kal = "kl",
+    khm = "km",
+    kan = "kn",
+    kor = "ko",
+    kas = "ks",
+    kur = "ku",
+    cor = "kw",
+    kir = "ky",
+    lat = "la",
+    ltz = "lb",
+    lao = "lo",
+    lit = "lt",
+    lav = "lv",
+    mlg = "mg",
+    mah = "mh",
+    mri = "mi",
+    mkd = "mk",
+    mal = "ml",
+    mon = "mn",
+    mar = "mr",
+    msa = "ms",
+    mlt = "mt",
+    mya = "my",
+    nob = "nb",
+    nld = "nl",
+    nno = "nn",
+    nor = "no",
+    nav = "nv",
+    oci = "oc",
+    orm = "om",
+    pan = "pa",
+    pli = "pi",
+    pol = "pl",
+    pus = "ps",
+    por = "pt",
+    que = "qu",
+    roh = "rm",
+    ron = "ro",
+    rus = "ru",
+    san = "sa",
+    hbs = "sh",
+    slk = "sk",
+    slv = "sl",
+    sqi = "sq",
+    srp = "sr",
+    ssw = "ss",
+    sun = "su",
+    swe = "sv",
+    swa = "sw",
+    tam = "ta",
+    tel = "te",
+    tgk = "tg",
+    tha = "th",
+    tgl = "tl",
+    tur = "tr",
+    tat = "tt",
+    uig = "ug",
+    ukr = "uk",
+    urd = "ur",
+    uzb = "uz",
+    vie = "vi",
+    vol = "vo",
+    wln = "wa",
+    xho = "xh",
+    yid = "yi",
+    yor = "yo",
+    zha = "za",
+    zho = "zh",
+    zul = "zu",
+}
+
+
+local IsoLanguage = {}
+
+function IsoLanguage:getLocalizedLanguage(iso3_language)
+    local language = locale_lang_map[iso3_language]
+    return language or iso3_language
+end
+
+function IsoLanguage:getBCPLanguageTag(iso3_language)
+    -- BCP47 language tag is either the ISO2 code, or ISO3 if no ISO2 exists for it
+    return iso_bcp47_map[iso3_language] or iso3_language
+end
+
+return IsoLanguage

frontend/ui/widget/dictquicklookup.lua

@@ -215,6 +215,7 @@ function DictQuickLookup:init()
         -- visual hint: title left aligned for dict, centered for Wikipedia
         align = self.is_wiki and "center" or "left",
         show_parent = self,
+        lang = self.lang_out,
     }
 
     -- This padding and the resulting width apply to the content
@@ -294,6 +295,7 @@ function DictQuickLookup:init()
         bold = true,
         max_width = self.content_width - math.max(lookup_edit_button_w, lookup_word_nb_w),
         padding = 0, -- to be aligned with lookup_word_nb
+        lang = self.lang_in
     }
     -- Group these 3 widgets
     local lookup_word = OverlapGroup:new{
@@ -795,7 +797,7 @@ function DictQuickLookup:_instantiateScrollWidget()
             height = self.definition_height,
             dialog = self,
             justified = G_reader_settings:nilOrTrue("dict_justify"), -- allow for disabling justification
-            lang = self.lang and self.lang:lower(), -- only available on wikipedia results
+            lang = self.lang and self.lang:lower() or self.lang_out,
             para_direction_rtl = self.rtl_lang,     -- only available on wikipedia results
             auto_para_direction = not self.is_wiki, -- only for dict results (we don't know their lang)
             image_alt_face = self.image_alt_face,
@@ -817,6 +819,7 @@ function DictQuickLookup:update()
         self.displaynb_text:setText(self.displaynb)
     end
     self.lookup_word_text:setText(self.displayword)
+    self.lookup_word_text.lang = self.lang_in
 
     -- Update Buttons
     if not self.is_wiki_fullpage then
@@ -842,7 +845,7 @@ function DictQuickLookup:update()
         -- Update properties that may change across results (as done in DictQuickLookup:_instantiateScrollWidget())
         self.text_widget.text_widget.text = self.definition
         self.text_widget.text_widget.charlist = nil -- (required when use_xtext=false for proper re-init)
-        self.text_widget.text_widget.lang = self.lang and self.lang:lower()
+        self.text_widget.text_widget.lang = self.lang and self.lang:lower() or self.lang_out
         self.text_widget.text_widget.para_direction_rtl = self.rtl_lang
         self.text_widget.text_widget.images = self.images
         -- Scroll back to the top, àla TextBoxWidget:scrollToTop
@@ -995,6 +998,9 @@ function DictQuickLookup:changeDictionary(index, skip_update)
     self.is_html = self.results[index].is_html
     self.css = self.results[index].css
     self.lang = self.results[index].lang
+    local ifo_lang = self.results[index].ifo_lang
+    self.lang_in = ifo_lang and ifo_lang.lang_in or nil
+    self.lang_out = ifo_lang and ifo_lang.lang_out or nil
     self.rtl_lang = self.results[index].rtl_lang
     self.images = self.results[index].images
     if self.images and #self.images > 0 then

frontend/ui/widget/titlebar.lua

@@ -44,6 +44,8 @@ local TitleBar = OverlapGroup:extend{
     info_text_face = Font:getFace("x_smallinfofont"),
     info_text_h_padding = nil, -- default to title_h_padding
 
+    lang = nil, -- use this language (string) instead of the UI language
+
     title_top_padding = nil, -- computed if none provided
     title_h_padding = Size.padding.large, -- horizontal padding (this replaces button_padding on the inner/title side)
     title_subtitle_v_padding = Screen:scaleBySize(3),
@@ -134,6 +136,7 @@ function TitleBar:init()
             alignment = self.align,
             width = title_max_width,
             face = title_face,
+            lang = self.lang,
         }
     else
         while true do
@@ -141,6 +144,7 @@ function TitleBar:init()
                 text = self.title,
                 face = title_face,
                 padding = 0,
+                lang = self.lang,
                 max_width = not self.title_shrink_font_to_fit and title_max_width,
                     -- truncate if not self.title_shrink_font_to_fit
             }
@@ -207,6 +211,7 @@ function TitleBar:init()
                 alignment = self.align,
                 width = subtitle_max_width,
                 face = self.subtitle_face,
+                lang = self.lang,
             }
         else
             self.subtitle_widget = TextWidget:new{
@@ -215,6 +220,7 @@ function TitleBar:init()
                 max_width = subtitle_max_width,
                 truncate_left = self.subtitle_truncate_left,
                 padding = 0,
+                lang = self.lang,
             }
         end
     end
@@ -332,6 +338,7 @@ function TitleBar:init()
                     text = self.info_text,
                     face = self.info_text_face,
                     width = self.width - 2 * h_padding,
+                    lang = self.lang,
                 }
             }
         }