OPDS: group downloading (#13338)

This commit is contained in:
hius07
2025-03-02 11:22:18 +02:00
committed by GitHub
parent e05d79a68a
commit daeacef838
4 changed files with 385 additions and 212 deletions

View File

@@ -12,7 +12,7 @@ local util = require("util")
local _ = require("gettext")
-- Date at which the last migration snippet was added
local CURRENT_MIGRATION_DATE = 20250207
local CURRENT_MIGRATION_DATE = 20250302
-- Retrieve the date of the previous migration, if any
local last_migration_date = G_reader_settings:readSetting("last_migration_date", 0)
@@ -842,5 +842,19 @@ if last_migration_date < 20250207 then
end
end
-- 20250302, Move OPDS settings from settings.reader.ui to settings/opds.lua.
-- https://github.com/koreader/koreader/pull/13338
if last_migration_date < 20250302 then
logger.info("Performing one-time migration for 20250302")
local servers = G_reader_settings:readSetting("opds_servers")
if servers then
G_reader_settings:delSetting("opds_servers")
local settings = LuaSettings:open(DataStorage:getSettingsDir() .. "/opds.lua")
settings:saveSetting("servers", servers)
settings:flush()
end
end
-- We're done, store the current migration date
G_reader_settings:saveSetting("last_migration_date", CURRENT_MIGRATION_DATE)

View File

@@ -1,51 +1,129 @@
local BD = require("ui/bidi")
local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local Dispatcher = require("dispatcher")
local LuaSettings = require("luasettings")
local OPDSBrowser = require("opdsbrowser")
local UIManager = require("ui/uimanager")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local util = require("util")
local _ = require("gettext")
local T = require("ffi/util").template
local OPDS = WidgetContainer:extend{
name = "opds",
is_doc_only = false,
opds_settings_file = DataStorage:getSettingsDir() .. "/opds.lua",
settings = nil,
servers = nil,
downloads = nil,
default_servers = {
{
title = "Project Gutenberg",
url = "https://m.gutenberg.org/ebooks.opds/?format=opds",
},
{
title = "Standard Ebooks",
url = "https://standardebooks.org/feeds/opds",
},
{
title = "ManyBooks",
url = "http://manybooks.net/opds/index.php",
},
{
title = "Internet Archive",
url = "https://bookserver.archive.org/",
},
{
title = "textos.info (Spanish)",
url = "https://www.textos.info/catalogo.atom",
},
{
title = "Gallica (French)",
url = "https://gallica.bnf.fr/opds",
},
},
}
function OPDS:init()
self.settings = LuaSettings:open(self.opds_settings_file)
if next(self.settings.data) == nil then
self.updated = true -- first run, force flush
end
self.servers = self.settings:readSetting("servers", self.default_servers)
self.downloads = self.settings:readSetting("downloads", {})
self:onDispatcherRegisterActions()
self.ui.menu:registerToMainMenu(self)
end
function OPDS:onDispatcherRegisterActions()
Dispatcher:registerAction("opds_show_catalog",
{category="none", event="ShowOPDSCatalog", title=_("OPDS Catalog"), filemanager=true,}
)
end
function OPDS:init()
self:onDispatcherRegisterActions()
self.ui.menu:registerToMainMenu(self)
end
function OPDS:showCatalog()
local OPDSCatalog = require("opdscatalog")
local filemanagerRefresh = function() self.ui:onRefresh() end
function OPDSCatalog:onClose()
UIManager:close(self)
local FileManager = require("apps/filemanager/filemanager")
if FileManager.instance then
filemanagerRefresh()
else
FileManager:showFiles(G_reader_settings:readSetting("download_dir"))
end
end
OPDSCatalog:showCatalog()
end
function OPDS:onShowOPDSCatalog()
self:showCatalog()
return true
end
function OPDS:addToMainMenu(menu_items)
if not self.ui.view then
if self.ui.file_chooser then
menu_items.opds = {
text = _("OPDS catalog"),
callback = function() self:showCatalog() end
callback = function()
self:onShowOPDSCatalog()
end,
}
end
end
function OPDS:onShowOPDSCatalog()
self.opds_browser = OPDSBrowser:new{
servers = self.servers,
downloads = self.downloads,
title = _("OPDS catalog"),
is_popout = false,
is_borderless = true,
title_bar_fm_style = true,
_manager = self,
file_downloaded_callback = function(file)
self:showFileDownloadedDialog(file)
end,
close_callback = function()
if self.opds_browser.download_list then
self.opds_browser.download_list.close_callback()
end
UIManager:close(self.opds_browser)
self.opds_browser = nil
if self.last_downloaded_file then
if self.ui.file_chooser then
local pathname = util.splitFilePathName(self.last_downloaded_file)
self.ui.file_chooser:changeToPath(pathname, self.last_downloaded_file)
end
self.last_downloaded_file = nil
end
end,
}
UIManager:show(self.opds_browser)
end
function OPDS:showFileDownloadedDialog(file)
self.last_downloaded_file = file
UIManager:show(ConfirmBox:new{
text = T(_("File saved to:\n%1\nWould you like to read the downloaded book now?"), BD.filepath(file)),
ok_text = _("Read now"),
ok_callback = function()
self.last_downloaded_file = nil
self.opds_browser.close_callback()
if self.ui.document then
self.ui:switchDocument(file)
else
self.ui:openFile(file)
end
end,
})
end
function OPDS:onFlushSettings()
if self.updated then
self.settings:flush()
self.updated = nil
end
end
return OPDS

View File

@@ -9,6 +9,7 @@ local InputDialog = require("ui/widget/inputdialog")
local Menu = require("ui/widget/menu")
local MultiInputDialog = require("ui/widget/multiinputdialog")
local NetworkMgr = require("ui/network/manager")
local Notification = require("ui/widget/notification")
local OPDSParser = require("opdsparser")
local OPDSPSE = require("opdspse")
local UIManager = require("ui/uimanager")
@@ -21,6 +22,7 @@ local socketutil = require("socketutil")
local url = require("socket.url")
local util = require("util")
local _ = require("gettext")
local N_ = _.ngettext
local T = require("ffi/util").template
-- cache catalog parsed from feed xml
@@ -30,37 +32,6 @@ local CatalogCache = Cache:new{
}
local OPDSBrowser = Menu:extend{
opds_servers = G_reader_settings:readSetting("opds_servers", {
{
title = "Project Gutenberg",
url = "https://m.gutenberg.org/ebooks.opds/?format=opds",
},
{
title = "Standard Ebooks",
url = "https://standardebooks.org/feeds/opds",
},
{
title = "Feedbooks",
url = "https://catalog.feedbooks.com/catalog/public_domain.atom",
},
{
title = "ManyBooks",
url = "http://manybooks.net/opds/index.php",
},
{
title = "Internet Archive",
url = "https://bookserver.archive.org/",
},
{
title = "textos.info (Spanish)",
url = "https://www.textos.info/catalogo.atom",
},
{
title = "Gallica (French)",
url = "https://gallica.bnf.fr/opds",
},
}),
catalog_type = "application/atom%+xml",
search_type = "application/opensearchdescription%+xml",
search_template_type = "application/atom%+xml",
@@ -89,19 +60,28 @@ function OPDSBrowser:init()
Menu.init(self) -- call parent's init()
end
local function buildRootEntry(server)
return {
text = server.title,
mandatory = server.username and "\u{f2c0}",
url = server.url,
username = server.username,
password = server.password,
raw_names = server.raw_names, -- use server raw filenames for download
searchable = server.url and server.url:match("%%s") and true or false,
}
end
-- Builds the root list of catalogs
function OPDSBrowser:genItemTableFromRoot()
local item_table = {}
for _, server in ipairs(self.opds_servers) do
table.insert(item_table, {
text = server.title,
mandatory = server.username and "\u{f2c0}",
url = server.url,
username = server.username,
password = server.password,
raw_names = server.raw_names, -- use server raw filenames for download
searchable = server.url:match("%%s") and true or false,
})
local item_table = {
{
text = _("Downloads"),
mandatory = #self.downloads,
},
}
for _, server in ipairs(self.servers) do
table.insert(item_table, buildRootEntry(server))
end
return item_table
end
@@ -193,7 +173,7 @@ function OPDSBrowser:addSubCatalog(item_url)
UIManager:close(dialog)
local fields = {name, item_url,
self.root_catalog_username, self.root_catalog_password, self.root_catalog_raw_names}
self:editCatalogFromInput(fields, false, true) -- no init, stay in the subcatalog
self:editCatalogFromInput(fields, nil, true) -- no init, stay in the subcatalog
end
end,
},
@@ -205,40 +185,37 @@ function OPDSBrowser:addSubCatalog(item_url)
end
-- Saves catalog properties from input dialog
function OPDSBrowser:editCatalogFromInput(fields, item, no_init)
local new_server
if item then -- edit old
for _, server in ipairs(self.opds_servers) do
if server.title == item.text and server.url == item.url then
new_server = server
break
end
end
else -- add new
new_server = {}
function OPDSBrowser:editCatalogFromInput(fields, item, no_refresh)
local new_server = {
title = fields[1],
url = fields[2]:match("^%a+://") and fields[2] or "http://" .. fields[2],
username = fields[3] ~= "" and fields[3] or nil,
password = fields[4] ~= "" and fields[4] or nil,
raw_names = fields[5],
}
local new_item = buildRootEntry(new_server)
local new_idx, itemnumber
if item then
new_idx = item.idx
itemnumber = -1
else
new_idx = #self.servers + 2
itemnumber = new_idx
end
new_server.title = fields[1]
new_server.url = fields[2]:match("^%a+://") and fields[2] or "http://" .. fields[2]
new_server.username = fields[3] ~= "" and fields[3] or nil
new_server.password = fields[4]
new_server.raw_names = fields[5]
if not item then
table.insert(self.opds_servers, new_server)
end
if not no_init then
self:init()
self.servers[new_idx - 1] = new_server
self.item_table[new_idx] = new_item
if not no_refresh then
self:switchItemTable(nil, self.item_table, itemnumber)
end
self._manager.updated = true
end
-- Deletes catalog from the root list
function OPDSBrowser:deleteCatalog(item)
for i, server in ipairs(self.opds_servers) do
if server.title == item.text and server.url == item.url then
table.remove(self.opds_servers, i)
break
end
end
self:init()
table.remove(self.servers, item.idx - 1)
table.remove(self.item_table, item.idx)
self:switchItemTable(nil, self.item_table, -1)
self._manager.updated = true
end
-- Fetches feed from server
@@ -642,10 +619,20 @@ function OPDSBrowser:showDownloads(item)
table.insert(download_buttons, {
text = text .. "\u{2B07}", -- append DOWNWARDS BLACK ARROW
callback = function()
local file = filename and filename .. "." .. string.lower(filetype)
or self:getServerFileName(acquisition.href)
self:downloadFile(file, acquisition.href)
UIManager:close(self.download_dialog)
local local_path = self:getLocalDownloadPath(filename, filetype, acquisition.href)
self:downloadFile(local_path, acquisition.href, self._manager.file_downloaded_callback)
end,
hold_callback = function()
UIManager:close(self.download_dialog)
table.insert(self.downloads, {
file = self:getLocalDownloadPath(filename, filetype, acquisition.href),
url = acquisition.href,
info = type(item.content) == "string" and util.htmlToPlainTextIfHtml(item.content),
catalog = self.root_catalog_title,
})
self._manager.updated = true
Notification:notify(_("Book added to download list"))
end,
})
end
@@ -756,20 +743,24 @@ function OPDSBrowser.getCurrentDownloadDir()
return G_reader_settings:readSetting("download_dir") or G_reader_settings:readSetting("lastdir")
end
-- Downloads a book (with "File already exists" dialog)
function OPDSBrowser:downloadFile(filename, remote_url)
local download_dir = self.getCurrentDownloadDir()
function OPDSBrowser:getLocalDownloadPath(filename, filetype, remote_url)
local download_dir = OPDSBrowser.getCurrentDownloadDir()
filename = filename and filename .. "." .. filetype:lower() or self:getServerFileName(remote_url)
filename = util.getSafeFilename(filename, download_dir)
local local_path = (download_dir ~= "/" and download_dir or "") .. '/' .. filename
local_path = util.fixUtf8(local_path, "_")
filename = (download_dir ~= "/" and download_dir or "") .. '/' .. filename
return util.fixUtf8(filename, "_")
end
-- Downloads a book (with "File already exists" dialog)
function OPDSBrowser:downloadFile(local_path, remote_url, caller_callback)
local ask_to_overwrite = caller_callback ~= nil -- single file downloading
local code
local function download()
UIManager:scheduleIn(1, function()
-- UIManager:scheduleIn(1, function()
logger.dbg("Downloading file", local_path, "from", remote_url)
local parsed = url.parse(remote_url)
local code, headers, status
local headers, status
if parsed.scheme == "http" or parsed.scheme == "https" then
socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT)
code, headers, status = socket.skip(1, http.request {
@@ -790,7 +781,9 @@ function OPDSBrowser:downloadFile(filename, remote_url)
if code == 200 then
logger.dbg("File downloaded to", local_path)
self.file_downloaded_callback(local_path)
if caller_callback then
caller_callback(local_path)
end
elseif code == 302 and remote_url:match("^https") and headers.location:match("^http[^s]") then
util.removeFile(local_path)
UIManager:show(InfoMessage:new{
@@ -807,7 +800,7 @@ function OPDSBrowser:downloadFile(filename, remote_url)
status or code or "network unreachable"),
})
end
end)
-- end)
UIManager:show(InfoMessage:new{
text = _("Downloading may take several minutes…"),
@@ -815,7 +808,7 @@ function OPDSBrowser:downloadFile(filename, remote_url)
})
end
if lfs.attributes(local_path) then
if ask_to_overwrite and lfs.attributes(local_path) then
UIManager:show(ConfirmBox:new{
text = T(_("The file %1 already exists. Do you want to overwrite it?"), BD.filepath(local_path)),
ok_text = _("Overwrite"),
@@ -826,6 +819,7 @@ function OPDSBrowser:downloadFile(filename, remote_url)
else
download()
end
return code == 200
end
-- Menu action on item tap (Download a book / Show subcatalog / Search in catalog)
@@ -835,6 +829,12 @@ function OPDSBrowser:onMenuSelect(item)
self:showDownloads(item)
else -- catalog or Search item
if #self.paths == 0 then -- root list
if item.idx == 1 then
if #self.downloads > 0 then
self:showDownloadList()
end
return true
end
self.root_catalog_title = item.text
self.root_catalog_username = item.username
self.root_catalog_password = item.password
@@ -858,7 +858,7 @@ end
-- Menu action on item long-press (dialog Edit / Delete catalog)
function OPDSBrowser:onMenuHold(item)
if #self.paths > 0 then return end -- not root list
if #self.paths > 0 or item.idx == 1 then return true end -- not root list or Downloads item
local dialog
dialog = ButtonDialog:new{
title = item.text,
@@ -894,31 +894,22 @@ end
-- Menu action on return-arrow tap (go to one-level upper catalog)
function OPDSBrowser:onReturn()
if #self.paths > 0 then -- not root list
table.remove(self.paths)
local path = self.paths[#self.paths]
if path then
-- return to last path
self.catalog_title = path.title
self:updateCatalog(path.url, true)
else
-- return to root path, we simply reinit opdsbrowser
self:init()
end
table.remove(self.paths)
local path = self.paths[#self.paths]
if path then
-- return to last path
self.catalog_title = path.title
self:updateCatalog(path.url, true)
else
-- return to root path, we simply reinit opdsbrowser
self:init()
end
return true
end
-- Menu action on return-arrow long-press (go to the catalog home page)
-- Menu action on return-arrow long-press (return to root path)
function OPDSBrowser:onHoldReturn()
if #self.paths > 1 then -- not catalog home page
local path = self.paths[1]
for i = #self.paths, 2, -1 do
table.remove(self.paths)
end
self.catalog_title = path.title
self:updateCatalog(path.url, true)
end
self:init()
return true
end
@@ -944,4 +935,171 @@ function OPDSBrowser:onNextPage(fill_only)
return true
end
function OPDSBrowser:showDownloadList()
self.download_list = Menu:new{
covers_fullscreen = true,
is_borderless = true,
is_popout = false,
title_bar_fm_style = true,
onMenuSelect = self.showDownloadListItemDialog,
_manager = self,
}
self.download_list.close_callback = function()
UIManager:close(self.download_list)
self.download_list = nil
if self.download_list_updated then
self.download_list_updated = nil
self.item_table[1].mandatory = #self.downloads
self:updateItems(1, true)
end
end
self:updateDownloadListItemTable()
UIManager:show(self.download_list)
end
function OPDSBrowser:updateDownloadListItemTable(item_table)
if item_table == nil then
item_table = {}
for i, item in ipairs(self.downloads) do
item_table[i] = {
text = item.file:gsub(".*/", ""),
mandatory = item.catalog,
}
end
end
local title = T(_("Downloads (%1)"), #item_table)
self.download_list:switchItemTable(title, item_table)
end
function OPDSBrowser:showDownloadListItemDialog(item)
local dl_item = self._manager.downloads[item.idx]
local textviewer
local function remove_item()
textviewer:onClose()
table.remove(self._manager.downloads, item.idx)
table.remove(self.item_table, item.idx)
self._manager:updateDownloadListItemTable(self.item_table)
self._manager.download_list_updated = true
self._manager._manager.updated = true
end
local buttons_table = {
{
{
text = _("Remove"),
callback = function()
remove_item()
end,
},
{
text = _("Download"),
callback = function()
local function file_downloaded_callback(local_path)
remove_item()
self._manager.file_downloaded_callback(local_path)
end
self._manager:downloadFile(dl_item.file, dl_item.url, file_downloaded_callback)
end,
},
},
{}, -- separator
{
{
text = _("Remove all"),
callback = function()
textviewer:onClose()
UIManager:show(ConfirmBox:new{
text = _("Remove all downloads?"),
ok_text = _("Remove"),
ok_callback = function()
for i in ipairs(self._manager.downloads) do
self._manager.downloads[i] = nil
end
self._manager.download_list_updated = true
self._manager._manager.updated = true
self:close_callback()
end,
})
end,
},
{
text = _("Download all"),
callback = function()
textviewer:onClose()
UIManager:show(ConfirmBox:new{
text = _("Download all books?\nExisting files will be overwritten."),
ok_text = _("Download"),
ok_callback = function()
local Trapper = require("ui/trapper")
Trapper:wrap(function()
self._manager:downloadDownloadList()
end)
end,
})
end,
},
},
}
local TextBoxWidget = require("ui/widget/textboxwidget")
local text = table.concat({
TextBoxWidget.PTF_HEADER,
TextBoxWidget.PTF_BOLD_START, _("Folder"), TextBoxWidget.PTF_BOLD_END, "\n",
util.splitFilePathName(dl_item.file), "\n", "\n",
TextBoxWidget.PTF_BOLD_START, _("File"), TextBoxWidget.PTF_BOLD_END, "\n",
item.text, "\n", "\n",
TextBoxWidget.PTF_BOLD_START, _("Description"), TextBoxWidget.PTF_BOLD_END, "\n",
dl_item.info,
})
local TextViewer = require("ui/widget/textviewer")
textviewer = TextViewer:new{
title = dl_item.catalog,
text = text,
text_type = "book_info",
buttons_table = buttons_table,
}
UIManager:show(textviewer)
return true
end
function OPDSBrowser:downloadDownloadList()
local info = InfoMessage:new{ text = _("Downloading… (tap to cancel)") }
UIManager:show(info)
UIManager:forceRePaint()
local Trapper = require("ui/trapper")
local completed, downloaded = Trapper:dismissableRunInSubprocess(function()
local dl = {}
for _, item in ipairs(self.downloads) do
if self:downloadFile(item.file, item.url) then
dl[item.file] = true
end
end
return dl
end, info)
if completed then
UIManager:close(info)
end
local dl_count = #self.downloads
for i = dl_count, 1, -1 do
local item = self.downloads[i]
if downloaded and downloaded[item.file] then
table.remove(self.downloads, i)
else -- if subprocess has been interrupted, check for the downloaded file
local attr = lfs.attributes(item.file)
if attr then
if attr.size > 0 then
table.remove(self.downloads, i)
else -- incomplete download
os.remove(item.file)
end
end
end
end
dl_count = dl_count - #self.downloads
if dl_count > 0 then
self:updateDownloadListItemTable()
self.download_list_updated = true
self._manager.updated = true
UIManager:show(InfoMessage:new{ text = T(N_("1 book downloaded", "%1 books downloaded", dl_count), dl_count) })
end
end
return OPDSBrowser

View File

@@ -1,77 +0,0 @@
local BD = require("ui/bidi")
local Blitbuffer = require("ffi/blitbuffer")
local ConfirmBox = require("ui/widget/confirmbox")
local FrameContainer = require("ui/widget/container/framecontainer")
local OPDSBrowser = require("opdsbrowser")
local UIManager = require("ui/uimanager")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local logger = require("logger")
local _ = require("gettext")
local Screen = require("device").screen
local T = require("ffi/util").template
local OPDSCatalog = WidgetContainer:extend{
title = _("OPDS Catalog"),
}
function OPDSCatalog:init()
local opds_browser = OPDSBrowser:new{
title = self.title,
show_parent = self,
is_popout = false,
is_borderless = true,
close_callback = function() return self:onClose() end,
file_downloaded_callback = function(downloaded_file)
UIManager:show(ConfirmBox:new{
text = T(_("File saved to:\n%1\nWould you like to read the downloaded book now?"),
BD.filepath(downloaded_file)),
ok_text = _("Read now"),
cancel_text = _("Read later"),
ok_callback = function()
local Event = require("ui/event")
UIManager:broadcastEvent(Event:new("SetupShowReader"))
self:onClose()
local ReaderUI = require("apps/reader/readerui")
ReaderUI:showReader(downloaded_file)
end
})
end,
}
self[1] = FrameContainer:new{
padding = 0,
bordersize = 0,
background = Blitbuffer.COLOR_WHITE,
opds_browser,
}
end
function OPDSCatalog:onShow()
UIManager:setDirty(self, function()
return "ui", self[1].dimen -- i.e., FrameContainer
end)
end
function OPDSCatalog:onCloseWidget()
UIManager:setDirty(nil, function()
return "ui", self[1].dimen
end)
end
function OPDSCatalog:showCatalog()
logger.dbg("show OPDS catalog")
UIManager:show(OPDSCatalog:new{
dimen = Screen:getSize(),
covers_fullscreen = true, -- hint for UIManager:_repaint()
})
end
function OPDSCatalog:onClose()
logger.dbg("close OPDS catalog")
UIManager:close(self)
return true
end
return OPDSCatalog