From 9849d89f0e4ec59bf1b6a58276756b0bef72304d Mon Sep 17 00:00:00 2001 From: poire-z Date: Wed, 14 Mar 2018 18:14:52 +0100 Subject: [PATCH] coverbrowser: allow for batch metadata extraction (#3750) This adds a button to the Tap Plus menu, that allows extracting metadata and cover images for books in current directory. Info about the process and questions are initially shown and asked, and the process can be aborted at any moment. --- .../coverbrowser.koplugin/bookinfomanager.lua | 241 +++++++++++++++++- plugins/coverbrowser.koplugin/covermenu.lua | 49 ++++ plugins/coverbrowser.koplugin/listmenu.lua | 28 +- plugins/coverbrowser.koplugin/main.lua | 13 +- plugins/coverbrowser.koplugin/mosaicmenu.lua | 33 ++- 5 files changed, 340 insertions(+), 24 deletions(-) diff --git a/plugins/coverbrowser.koplugin/bookinfomanager.lua b/plugins/coverbrowser.koplugin/bookinfomanager.lua index bfa279467..6d19980f6 100644 --- a/plugins/coverbrowser.koplugin/bookinfomanager.lua +++ b/plugins/coverbrowser.koplugin/bookinfomanager.lua @@ -404,7 +404,6 @@ function BookInfoManager:extractBookInfo(filepath, cover_specs) local spec_max_cover_h = cover_specs.max_cover_h dbrow.cover_fetched = 'Y' -- we had a try at getting a cover - -- XXX make picdocument return a blitbuffer of the image local cover_bb = document:getCoverPageImage() if cover_bb then dbrow.has_cover = 'Y' @@ -455,6 +454,7 @@ function BookInfoManager:extractBookInfo(filepath, cover_specs) end self.set_stmt:step() self.set_stmt:clearbind():reset() -- get ready for next query + return loaded end function BookInfoManager:setBookInfoProperties(filepath, props) @@ -626,6 +626,245 @@ function BookInfoManager:cleanUp() end end +local function findFilesInDir(path, recursive) + local dirs = {path} + local files = {} + while #dirs ~= 0 do + local new_dirs = {} + -- handle each dir + for __, d in pairs(dirs) do + -- handle files in d + for f in lfs.dir(d) do + local fullpath = d.."/"..f + local attributes = lfs.attributes(fullpath) + if recursive and attributes.mode == "directory" and f ~= "." and f~=".." then + table.insert(new_dirs, fullpath) + elseif attributes.mode == "file" and DocumentRegistry:hasProvider(fullpath) then + table.insert(files, fullpath) + end + end + end + dirs = new_dirs + end + return files +end + +-- Batch extraction +function BookInfoManager:extractBooksInDirectory(path, cover_specs) + local Geom = require("ui/geometry") + local InfoMessage = require("ui/widget/infomessage") + local TopContainer = require("ui/widget/container/topcontainer") + local Trapper = require("ui/trapper") + local Screen = require("device").screen + + local go_on = Trapper:confirm(_([[ + +This will extract metadata and cover images for books in current directory. +Once extraction has started, you can abort at any moment by taping on the screen. + +Cover images will be saved with the adequate size for the current display mode. +If you later change display mode, they may need to be extracted again. + +This extraction may take time and use some battery power: you may wish to keep your device plugged in. +]]) , _("Cancel"), _("Continue")) + if not go_on then + return + end + + local recursive = Trapper:confirm(_([[ + +Do you want to extract book information for books in sub-directories too? +]]) , _("Here only"), _("Here and under")) + + local refresh_existing = Trapper:confirm(_([[ + +Do you want to refresh metadata and covers that have already been extracted? +]]) , _("Don't refresh"), _("Refresh")) + + local prune = Trapper:confirm(_([[ + +If you have removed many books, or have renamed some directories, it is good to remove them from the cache database. + +Do you want to prune cache of removed books? +]]) , _("Don't prune"), _("Prune")) + + Trapper:clear() + + local confirm_abort = function() + return Trapper:confirm(_("Do you want to abort extraction?"), _("Don't abort"), _("Abort")) + end + + -- Cancel any background job, before we launch new ones + self:terminateBackgroundJobs() + + local info, completed + if prune then + local summary + while true do + info = InfoMessage:new{text = _("Pruning cache of removed books…")} + UIManager:show(info) + UIManager:forceRePaint() + completed, summary = Trapper:dismissableRunInSubprocess(function() + return self:removeNonExistantEntries() + end, info) + if not completed then + if confirm_abort() then + return + end + else + UIManager:close(info) + info = InfoMessage:new{text = summary} + UIManager:show(info) + UIManager:forceRePaint() + util.sleep(2) -- Let the user see that + break + end + end + UIManager:close(info) + end + + local files + while true do + info = InfoMessage:new{text = _("Looking for books to index…")} + UIManager:show(info) + UIManager:forceRePaint() + completed, files = Trapper:dismissableRunInSubprocess(function() + local filepaths = findFilesInDir(path, recursive) + table.sort(filepaths) + return filepaths + end, info) + if not completed then + if confirm_abort() then + return + end + elseif not files or #files == 0 then + UIManager:close(info) + info = InfoMessage:new{text = _("No book were found.")} + UIManager:show(info) + return + else + break + end + end + UIManager:close(info) + + if refresh_existing then + info = InfoMessage:new{text = T(_("Found %1 books to index."), #files)} + UIManager:show(info) + UIManager:forceRePaint() + util.sleep(2) -- Let the user see that + else + local all_files = files + while true do + info = InfoMessage:new{text = T(_("Found %1 books.\nLooking for those not already present in cache database…"), #all_files)} + UIManager:show(info) + UIManager:forceRePaint() + util.sleep(2) -- Let the user see that + completed, files = Trapper:dismissableRunInSubprocess(function() + files = {} + for _, filepath in pairs(all_files) do + local bookinfo = self:getBookInfo(filepath) + local to_extract = not bookinfo + if bookinfo and cover_specs and not bookinfo.ignore_cover then + if bookinfo.cover_fetched then + if bookinfo.has_cover and cover_specs.sizetag ~= bookinfo.cover_sizetag then + if bookinfo.cover_sizetag ~= "M" then -- keep the bigger "M" + to_extract = true + end + end + else + to_extract = true + end + end + if to_extract then + table.insert(files, filepath) + end + end + return files + end, info) + if not completed then + if confirm_abort() then + return + end + elseif not files or #files == 0 then + UIManager:close(info) + info = InfoMessage:new{text = _("No books were found that need to be indexed.")} + UIManager:show(info) + return + else + break + end + end + UIManager:close(info) + info = InfoMessage:new{text = T(_("Found %1 books to index."), #files)} + UIManager:show(info) + UIManager:forceRePaint() + util.sleep(2) -- Let the user see that + end + UIManager:close(info) + + local nb_files = #files + local nb_done = 0 + local nb_success = 0 + local i = 1 + + -- We use a little hack to InfoMessage for a consistent height and + -- fast refresh to avoid flicking + info = InfoMessage:new{text = "dummy"} + UIManager:show(info) -- but not yet painted + local info_max_seen_height = 0 + local success + + while i <= nb_files do + local filepath = files[i] + local filename = util.basename(filepath) + + local orig_moved_offset = info.movable:getMovedOffset() + info:free() + info.text = T(_("Indexing %1 / %2…\n\n%3"), i, nb_files, filename) + info:init() + local text_widget = table.remove(info.movable[1][1], 3) + local text_widget_size = text_widget:getSize() + if text_widget_size.h > info_max_seen_height then + info_max_seen_height = text_widget_size.h + end + table.insert(info.movable[1][1], TopContainer:new{ + dimen = Geom:new{ + w = text_widget_size.w, + h = info_max_seen_height, + }, + text_widget + }) + info.movable:setMovedOffset(orig_moved_offset) + info:paintTo(Screen.bb, 0,0) + local d = info.movable[1].dimen + Screen.refreshUI(Screen, d.x, d.y, d.w, d.h) + + completed, success = Trapper:dismissableRunInSubprocess(function() + return self:extractBookInfo(filepath, cover_specs) + end, info) + if not completed then + if confirm_abort() then + break + end + -- Recreate the infomessage that was dismissed + info = InfoMessage:new{text = "dummy"} + info.movable:setMovedOffset(orig_moved_offset) + UIManager:show(info) -- but not yet painted + -- don't increment i, re-process the one we interrupted + else + nb_done = nb_done + 1 + if success then + nb_success = nb_success + 1 + end + i = i + 1 + end + end + UIManager:close(info) + info = InfoMessage:new{text = T(_("Processed %1 / %2 books.\n%3 extracted succesfully."), nb_done, nb_files, nb_success)} + UIManager:show(info) +end + BookInfoManager:init() return BookInfoManager diff --git a/plugins/coverbrowser.koplugin/covermenu.lua b/plugins/coverbrowser.koplugin/covermenu.lua index 08e7a72c5..4bf01d41e 100644 --- a/plugins/coverbrowser.koplugin/covermenu.lua +++ b/plugins/coverbrowser.koplugin/covermenu.lua @@ -28,6 +28,12 @@ local BookInfoManager = require("bookinfomanager") -- not found item to self.items_to_update for us to update() them -- regularly. +-- Store these as local, to be set by some object and re-used by +-- another object (as we plug the methods below to different objects, +-- we can't store them in 'self' if we want another one to use it) +local current_path = nil +local current_cover_specs = false + -- Simple holder of methods that will replace those -- in the real Menu class or instance local CoverMenu = {} @@ -76,6 +82,10 @@ function CoverMenu:updateItems(select_number) -- Specific UI building implementation (defined in some other module) self:_updateItemsBuildUI() + -- Set the local variables with the things we know + current_path = self.path + current_cover_specs = self.cover_specs + -- As done in Menu:updateItems() if self.item_group[1] then if not Device:isTouchDevice() then @@ -476,4 +486,43 @@ function CoverMenu:onSwipe(arg, ges_ev) end end +function CoverMenu:tapPlus() + -- Call original function: it will create a ButtonDialogTitle + -- and store it as self.file_dialog, and UIManager:show() it. + CoverMenu._FileManager_tapPlus_orig(self) + + -- Remember some of this original ButtonDialogTitle properties + local orig_title = self.file_dialog.title + local orig_title_align = self.file_dialog.title_align + local orig_buttons = self.file_dialog.buttons + -- Close original ButtonDialogTitle (it has not yet been painted + -- on screen, so we won't see it) + UIManager:close(self.file_dialog) + + -- Add a new button to original buttons set + table.insert(orig_buttons, {}) -- separator + table.insert(orig_buttons, { + { + text = _("Extract and cache book information"), + callback = function() + UIManager:close(self.file_dialog) + local Trapper = require("ui/trapper") + Trapper:wrap(function() + BookInfoManager:extractBooksInDirectory(current_path, current_cover_specs) + end) + end, + }, + }) + + -- Create the new ButtonDialogTitle, and let UIManager show it + local ButtonDialogTitle = require("ui/widget/buttondialogtitle") + self.file_dialog = ButtonDialogTitle:new{ + title = orig_title, + title_align = orig_title_align, + buttons = orig_buttons, + } + UIManager:show(self.file_dialog) + return true +end + return CoverMenu diff --git a/plugins/coverbrowser.koplugin/listmenu.lua b/plugins/coverbrowser.koplugin/listmenu.lua index 998ff6f5f..186f7a44b 100644 --- a/plugins/coverbrowser.koplugin/listmenu.lua +++ b/plugins/coverbrowser.koplugin/listmenu.lua @@ -188,6 +188,25 @@ function ListMenuItem:update() h = self.height - 2 * self.underline_h } + -- We'll draw a border around cover images, it may not be + -- needed with some covers, but it's nicer when cover is + -- a pure white background (like rendered text page) + local border_size = 1 + local max_img_w = dimen.h - 2*border_size -- width = height, squared + local max_img_h = dimen.h - 2*border_size + local cover_specs = { + sizetag = "s", + max_cover_w = max_img_w, + max_cover_h = max_img_h, + } + -- Make it available to our menu, for batch extraction + -- to know what size is needed for current view + if self.do_cover_image then + self.menu.cover_specs = cover_specs + else + self.menu.cover_specs = false + end + local file_mode = lfs.attributes(self.filepath, "mode") if file_mode == "directory" then self.is_directory = true @@ -226,9 +245,6 @@ function ListMenuItem:update() self.file_deleted = true end -- File - local border_size = 1 - local max_img_w = dimen.h - 2*border_size -- width = height, squared - local max_img_h = dimen.h - 2*border_size local bookinfo = BookInfoManager:getBookInfo(self.filepath, self.do_cover_image) if bookinfo and self.do_cover_image and not bookinfo.ignore_cover then @@ -550,11 +566,7 @@ function ListMenuItem:update() -- a new extraction will have to be made when one switch to image mode if self.do_cover_image then -- Not in db, we're going to fetch some cover - self.cover_specs = { - sizetag = "s", - max_cover_w = max_img_w, - max_cover_h = max_img_h, - } + self.cover_specs = cover_specs end -- if self.do_hint_opened and DocSettings:hasSidecarFile(self.filepath) then diff --git a/plugins/coverbrowser.koplugin/main.lua b/plugins/coverbrowser.koplugin/main.lua index c936c5fb7..bf81b04ad 100644 --- a/plugins/coverbrowser.koplugin/main.lua +++ b/plugins/coverbrowser.koplugin/main.lua @@ -23,6 +23,9 @@ local _FileChooser_onSwipe_orig = FileChooser.onSwipe local FileManagerHistory = require("apps/filemanager/filemanagerhistory") local _FileManagerHistory_updateItemTable_orig = FileManagerHistory.updateItemTable +local FileManager = require("apps/filemanager/filemanager") +local _FileManager_tapPlus_orig = FileManager.tapPlus + -- Available display modes local DISPLAY_MODES = { -- nil or "" -- classic : filename only @@ -120,11 +123,9 @@ function CoverBrowser:addToMainMenu(menu_items) callback = function() if G_reader_settings:readSetting("home_dir_display_name") then G_reader_settings:delSetting("home_dir_display_name") - local FileManager = require("apps/filemanager/filemanager") if FileManager.instance then FileManager.instance:reinit() end else G_reader_settings:saveSetting("home_dir_display_name", "~") - local FileManager = require("apps/filemanager/filemanager") if FileManager.instance then FileManager.instance:reinit() end end end, @@ -381,7 +382,6 @@ function CoverBrowser:addToMainMenu(menu_items) end function CoverBrowser:refreshFileManagerInstance(cleanup, post_init) - local FileManager = require("apps/filemanager/filemanager") local fm = FileManager.instance if fm then local fc = fm.file_chooser @@ -433,6 +433,7 @@ function CoverBrowser:setupFileManagerDisplayMode(display_mode) FileChooser.onCloseWidget = _FileChooser_onCloseWidget_orig FileChooser.onSwipe = _FileChooser_onSwipe_orig FileChooser._recalculateDimen = _FileChooser__recalculateDimen_orig + FileManager.tapPlus = _FileManager_tapPlus_orig -- Also clean-up what we added, even if it does not bother original code FileChooser._updateItemsBuildUI = nil FileChooser._do_cover_images = nil @@ -475,6 +476,12 @@ function CoverBrowser:setupFileManagerDisplayMode(display_mode) FileChooser._do_hint_opened = true -- dogear at bottom end + -- Replace this FileManager method with the one from CoverMenu + -- (but first, make the original method saved here as local available + -- to CoverMenu) + CoverMenu._FileManager_tapPlus_orig = _FileManager_tapPlus_orig + FileManager.tapPlus = CoverMenu.tapPlus + if init_done then self:refreshFileManagerInstance() else diff --git a/plugins/coverbrowser.koplugin/mosaicmenu.lua b/plugins/coverbrowser.koplugin/mosaicmenu.lua index 60633c27e..e24302c75 100644 --- a/plugins/coverbrowser.koplugin/mosaicmenu.lua +++ b/plugins/coverbrowser.koplugin/mosaicmenu.lua @@ -374,13 +374,32 @@ function MosaicMenuItem:update() h = self.height - self.underline_h } + -- We'll draw a border around cover images, it may not be + -- needed with some covers, but it's nicer when cover is + -- a pure white background (like rendered text page) + local border_size = 1 + local max_img_w = dimen.w - 2*border_size + local max_img_h = dimen.h - 2*border_size + local cover_specs = { + sizetag = "M", + max_cover_w = max_img_w, + max_cover_h = max_img_h, + } + -- Make it available to our menu, for batch extraction + -- to know what size is needed for current view + if self.do_cover_image then + self.menu.cover_specs = cover_specs + else + self.menu.cover_specs = false + end + local file_mode = lfs.attributes(self.filepath, "mode") if file_mode == "directory" then self.is_directory = true -- Directory : rounded corners local margin = Screen:scaleBySize(5) -- make directories less wide local padding = Screen:scaleBySize(5) - local border_size = Screen:scaleBySize(2) -- make directories bolder + border_size = Screen:scaleBySize(2) -- make directories bolder local dimen_in = Geom:new{ w = dimen.w - (margin + padding + border_size)*2, h = dimen.h - (margin + padding + border_size)*2 @@ -420,12 +439,6 @@ function MosaicMenuItem:update() self.file_deleted = true end -- File : various appearances - -- We'll draw a border around cover images, it may not be - -- needed with some covers, but it's nicer when cover is - -- a pure white background (like rendered text page) - local border_size = 1 - local max_img_w = dimen.w - 2*border_size - local max_img_h = dimen.h - 2*border_size if self.do_hint_opened and DocSettings:hasSidecarFile(self.filepath) then self.been_opened = true @@ -544,11 +557,7 @@ function MosaicMenuItem:update() -- a new extraction will have to be made when one switch to image mode if self.do_cover_image then -- Not in db, we're going to fetch some cover - self.cover_specs = { - sizetag = "M", - max_cover_w = max_img_w, - max_cover_h = max_img_h, - } + self.cover_specs = cover_specs end -- Same as real FakeCover, but let it be squared (like a file) local hint = "…" -- display hint it's being loaded