renun/koreader
1
0
Fork 0
mirror of https://github.com/koreader/koreader.git synced 2025-08-10 00:52:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[feat] Add dictionary download option (#3176)

Browse Source 
You can now download pretty much all of the easily available freely licensed dictionaries I could find.
This commit is contained in:
Frans de Jonge
2018-12-13 06:27:49 +00:00
committed by GitHub
parent 8fec0f330f
commit b261a647c2
4 changed files with 330 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.luacheckrc
View File

@@ -122,4 +122,5 @@ files["spec/unit/*"].globals = {
ignore = {
"211/__*",
"631",
"dummy",
}

172
frontend/apps/reader/modules/readerdictionary.lua
View File

@@ -8,8 +8,10 @@ local InputContainer = require("ui/widget/container/inputcontainer")
local JSON = require("json")
local KeyValuePage = require("ui/widget/keyvaluepage")
local LuaData = require("luadata")
local NetworkMgr = require("ui/network/manager")
local Trapper = require("ui/trapper")
local UIManager = require("ui/uimanager")
local ffiUtil = require("ffi/util")
local logger = require("logger")
local util = require("util")
local _ = require("gettext")
@@ -238,6 +240,10 @@ function ReaderDictionary:addToMainMenu(menu_items)
})
end,
},
{
text = _("Download dictionaries"),
sub_item_table = self:_genDownloadDictionariesMenu()
},
{
text = _("Enable fuzzy search"),
checked_func = function()
@@ -370,6 +376,50 @@ function ReaderDictionary:getNumberOfDictionaries()
return nb_available, nb_enabled, nb_disabled
end
function ReaderDictionary:_genDownloadDictionariesMenu()
local downloadable_dicts = require("ui/data/dictionaries")
local languages = {}
for i = 1, #downloadable_dicts do
local dict = downloadable_dicts[i]
local dict_lang_in = dict.lang_in
local dict_lang_out = dict.lang_out
if not languages[dict_lang_in] then
languages[dict_lang_in] = {}
end
table.insert(languages[dict_lang_in], dict)
if not languages[dict_lang_out] then
languages[dict_lang_out] = {}
end
table.insert(languages[dict_lang_out], dict)
end
-- remove duplicates
for lang_key,lang in pairs(languages) do
local hash = {}
local res = {}
for k,v in ipairs(lang) do
if not hash[v.name] then
res[#res+1] = v
hash[v.name] = true
end
end
languages[lang_key] = res
end
local menu_items = {}
for lang_key, available_langs in ffiUtil.orderedPairs(languages) do
table.insert(menu_items, {
text = lang_key,
callback = function()
self:showDownload(available_langs)
end
})
end
return menu_items
end
function ReaderDictionary:genDictionariesMenu()
local items = {}
for _, ifo in pairs(available_ifos) do
@@ -677,6 +727,128 @@ function ReaderDictionary:showDict(word, results, box, link)
end
end
function ReaderDictionary:showDownload(downloadable_dicts)
local kv_pairs = {}
table.insert(kv_pairs, {_("Tap dictionary name to download"), ""})
table.insert(kv_pairs, "----------------------------")
for dummy, dict in ipairs(downloadable_dicts) do
table.insert(kv_pairs, {dict.name, "",
callback = function()
if not NetworkMgr:isOnline() then
NetworkMgr:promptWifiOn()
return
end
self:downloadDictionaryPrep(dict)
end})
local lang
if dict.lang_in == dict.lang_out then
lang = string.format(" %s", dict.lang_in)
else
lang = string.format(" %s–%s", dict.lang_in, dict.lang_out)
end
table.insert(kv_pairs, {lang, ""})
table.insert(kv_pairs, {" ".._("License"), dict.license})
table.insert(kv_pairs, {" ".._("Entries"), dict.entries})
table.insert(kv_pairs, "----------------------------")
end
self.download_window = KeyValuePage:new{
title = _("Download dictionaries"),
kv_pairs = kv_pairs,
}
UIManager:show(self.download_window)
end
function ReaderDictionary:downloadDictionaryPrep(dict, size)
local dummy, filename = util.splitFilePathName(dict.url)
local download_location = string.format("%s/%s", self.data_dir, filename)
local lfs = require("libs/libkoreader-lfs")
if lfs.attributes(download_location) then
UIManager:show(ConfirmBox:new{
text = _("File already exists. Overwrite?"),
ok_text = _("Overwrite"),
ok_callback = function()
self:downloadDictionary(dict, download_location)
end,
})
else
self:downloadDictionary(dict, download_location)
end
end
function ReaderDictionary:downloadDictionary(dict, download_location, continue)
continue = continue or false
local socket = require("socket")
local http = socket.http
local https = require("ssl.https")
local ltn12 = require("ltn12")
local url = socket.url
local parsed = url.parse(dict.url)
local httpRequest = parsed.scheme == "http" and http.request or https.request
if not continue then
local file_size
--local r, c, h = httpRequest {
local dummy, headers, dummy = socket.skip(1, httpRequest{
method = "HEAD",
url = dict.url,
--redirect = true,
})
--logger.dbg(status)
--logger.dbg(headers)
--logger.dbg(code)
file_size = headers and headers["content-length"]
UIManager:show(ConfirmBox:new{
text = T(_("Dictionary filesize is %1 (%2 bytes). Continue with download?"), util.getFriendlySize(file_size), util.getFormattedSize(file_size)),
ok_text = _("Download"),
ok_callback = function()
-- call ourselves with continue = true
self:downloadDictionary(dict, download_location, true)
end,
})
return
else
UIManager:nextTick(function()
UIManager:show(InfoMessage:new{
text = _("Downloading…"),
timeout = 3,
})
end)
end
local dummy, c, dummy = httpRequest{
url = dict.url,
sink = ltn12.sink.file(io.open(download_location, "w")),
}
if c == 200 then
logger.dbg("file downloaded to", download_location)
else
UIManager:show(InfoMessage:new{
text = _("Could not save file to:\n") .. download_location,
--timeout = 3,
})
return false
end
local ok, error = util.unpackArchive(download_location, self.data_dir)
if ok then
available_ifos = false
self:init()
UIManager:show(InfoMessage:new{
text = _("Dictionary downloaded:\n") .. dict.name,
})
return true
else
UIManager:show(InfoMessage:new{
text = _("Dictionary failed to download:\n") .. string.format("%s\n%s", dict.name, error),
})
return false
end
end
function ReaderDictionary:onUpdateDefaultDict(dict)
logger.dbg("make default dictionary:", dict)
self.default_dictionary = dict

127
frontend/ui/data/dictionaries.lua Normal file
View File

@@ -0,0 +1,127 @@
local _ = require("gettext")
-- largely thanks to https://tuxor1337.github.io/firedict/dictionaries.html
local dictionaries = {
{
name = "CIA World Factbook 2014",
lang_in = "English",
lang_out = "English",
entries = 2577,
license = _("Public Domain"),
url = "http://build.koreader.rocks/download/dict/factbook.tar.lz",
},
{
name = "GNU Collaborative International Dictionary of English",
lang_in = "English",
lang_out = "English",
entries = 108121,
license = "GPLv3+",
url = "http://build.koreader.rocks/download/dict/gcide.tar.lz",
},
{
name = "Douglas Harper's Online Etymology Dictionary",
lang_in = "English",
lang_out = "English",
entries = 46133,
license = "Unknown/©Douglas Harper",
url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/1f3a66fe3f776e718590a66958df1b9b/etymonline.tar.lz",
},
{
name = "Folkets lexikon",
lang_in = "English",
lang_out = "Swedish",
entries = 53618,
license = "CC-BY-SA 2.5",
url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/fad990cc29c6dd448f22469ce8648825/folkets_en-sv.tar.lz",
},
{
name = "Folkets lexikon",
lang_in = "Swedish",
lang_out = "English",
entries = 36513,
license = "CC-BY-SA 2.5",
url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/fdc4265bbfc9af27fa4b1742ce3bdadd/folkets_sv-en.tar.lz",
},
{
name = "Dictionnaire Littré (xmlittre)",
lang_in = "French",
lang_out = "French",
entries = 78428,
license = "CC-BY-SA 3.0",
url = "http://http.debian.net/debian/pool/main/s/stardict-xmlittre/stardict-xmlittre_1.0.orig.tar.gz",
},
{
name = "Dictionnaire de l'Académie Française: 8ème edition",
lang_in = "French",
lang_out = "French",
entries = 31934,
license = _("Public Domain (copyright expired, published 1935)"),
url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/a8b3027e84f344f64723fb7fe5c63c04/acadfran.tar.lz",
},
{
name = "Pape: Handwörterbuch der griechischen Sprache",
lang_in = "Ancient Greek",
lang_out = "German",
entries = 98893,
license = _("Public Domain (copyright expired, published 1880)"),
url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/3ee9cb3acaca679bb9a95d845b813673/pape_gr-de.tar.lz",
},
{
name = "Georges: Ausführliches lateinisch-deutsches Handwörterbuch",
lang_in = "Latin",
lang_out = "German",
entries = 54831,
license = _("Public Domain (copyright expired, published 1913)"),
url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/8d1e52d6c28d3b6865415979bc221fa6/georges_lat-de.tar.lz",
},
{
name = "Georges: Kleines deutsch-lateinisches Handwörterbuch",
lang_in = "German",
lang_out = "Latin",
entries = 26608,
license = _("Public Domain (copyright expired, published 1910)"),
url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/a66b2a7655a4b102150fb5c6f1789cc5/georges_de-lat.tar.lz",
},
{
name = "Dicionário Aberto",
lang_in = "Portuguese",
lang_out = "Portuguese",
entries = 128521,
license = _("CC-BY-SA 2.5"),
url = "http://www.dicionario-aberto.net/stardict-DicAberto.tar.bz2",
},
{
name = "GNU/FDL Anglicko/Český slovník",
lang_in = "English",
lang_out = "Czech",
entries = 178904, -- ~90000 each way
license = _("GNU/FDL"),
url = "http://http.debian.net/debian/pool/non-free/s/stardict-english-czech/stardict-english-czech_20161201.orig.tar.gz",
},
{
name = "GNU/FDL Anglicko/Český slovník",
lang_in = "Czech",
lang_out = "English",
entries = 178904, -- ~90000 each way
license = _("GNU/FDL"),
url = "http://http.debian.net/debian/pool/non-free/s/stardict-english-czech/stardict-english-czech_20161201.orig.tar.gz",
},
{
name = "GNU/FDL Německo/Český slovník",
lang_in = "German",
lang_out = "Czech",
entries = 2341, -- ~1200 each way
license = _("GNU/FDL"),
url = "http://http.debian.net/debian/pool/non-free/s/stardict-german-czech/stardict-german-czech_20161201.orig.tar.gz",
},
{
name = "GNU/FDL Německo/Český slovník",
lang_in = "Czech",
lang_out = "German",
entries = 2341, -- ~1200 each way
license = _("GNU/FDL"),
url = "http://http.debian.net/debian/pool/non-free/s/stardict-german-czech/stardict-german-czech_20161201.orig.tar.gz",
},
}
return dictionaries

34
frontend/util.lua
View File

@@ -3,6 +3,10 @@ This module contains miscellaneous helper functions for the KOReader frontend.
]]
local BaseUtil = require("ffi/util")
local dbg = require("dbg")
local _ = require("gettext")
local T = BaseUtil.template
local util = {}
--- Strips all punctuation and spaces from a string.
@@ -136,11 +140,11 @@ function util.tableEquals(o1, o2, ignore_mt)
end
--- Returns number of keys in a table.
---- @param T Lua table
---- @treturn int number of keys in table T
function util.tableSize(T)
---- @param t Lua table
---- @treturn int number of keys in table t
function util.tableSize(t)
local count = 0
for _ in pairs(T) do count = count + 1 end
for _ in pairs(t) do count = count + 1 end
return count
end
@@ -431,6 +435,7 @@ end
---- @int size (bytes)
---- @treturn string
function util.getFriendlySize(size)
size = tonumber(size)
if not size or type(size) ~= "number" then return end
local s
if size > 1024*1024*1024 then
@@ -691,4 +696,25 @@ function util.checkLuaSyntax(lua_text)
return err
end
--- Unpack an archive.
-- Extract the contents of an archive, detecting its format by
-- filename extension. Inspired by luarocks archive_unpack()
-- @param archive string: Filename of archive.
-- @param extract_to string: Destination directory.
-- @return boolean or (boolean, string): true on success, false and an error message on failure.
function util.unpackArchive(archive, extract_to)
dbg.dassert(type(archive) == "string")
local ok
if archive:match("%.tar%.bz2$") or archive:match("%.tar%.gz$") or archive:match("%.tar%.lz$") or archive:match("%.tgz$") then
ok = os.execute(("./tar xf %q -C %q"):format(archive, extract_to))
else
return false, T(_("Couldn't extract archive:\n\n%1\n\nUnrecognized filename extension."), archive)
end
if not ok then
return false, T(_("Extracting archive failed:\n\n%1", archive))
end
return true
end
return util