Cloud-based sync for 2 plugins: reading statistics and vocabulary builder (#9709)

This commit adds cross-device sync ability for two plugins: reading statistics and vocabulary builder. It relies on user setting up a Cloud server (DropBox and WebDAV but not FTP though) and designating a path. Behind the curtains sqlite databases are being passed around and updated.

UI-wise, for the statistics plugin, two new menu items Synchronize now and Cloud sync to set it up (might not be the best wording) are added. As for vocabulary builder, a similar Cloud sync button is added to the menu and a shortcut icon button to Synchronize now is pinned at the bottom corner.

CloudStorage new features: WebDAV creating folders and uploading files. And a new widget-like sync server chooser. In the end I decided not to add automatic sync, as the SQL commands part seem a bit much.

frontend/apps/cloudstorage/cloudstorage.lua

@@ -173,11 +173,11 @@ function CloudStorage:openCloudServer(url)
         if NetworkMgr:willRerunWhenConnected(function() self:openCloudServer(url) end) then
             return
         end
-        tbl, e = WebDav:run(self.address, self.username, self.password, url)
+        tbl, e = WebDav:run(self.address, self.username, self.password, url, self.choose_folder_mode)
     end
     if tbl then
         self:switchItemTable(url, tbl)
-        if self.type == "dropbox" then
+        if self.type == "dropbox" or self.type == "webdav" then
             self.onLeftButtonTap = function()
                 self:showPlusMenu(url)
             end
@@ -648,7 +648,11 @@ function CloudStorage:uploadFile(url)
                 end)
                 local url_base = url ~= "/" and url or ""
                 UIManager:tickAfterNext(function()
-                    DropBox:uploadFile(url_base, self.password, file_path, callback_close)
+                    if self.type == "dropbox" then
+                        DropBox:uploadFile(url_base, self.password, file_path, callback_close)
+                    elseif self.type == "webdav" then
+                        WebDav:uploadFile(url_base, self.address, self.username, self.password, file_path, callback_close)
+                    end
                 end)
             end
         end
@@ -686,7 +690,11 @@ function CloudStorage:createFolder(url)
                             end
                             self:openCloudServer(url)
                         end
-                        DropBox:createFolder(url_base, self.password, folder_name, callback_close)
+                        if self.type == "dropbox" then
+                            DropBox:createFolder(url_base, self.password, folder_name, callback_close)
+                        elseif self.type == "webdav" then
+                            WebDav:createFolder(url_base, self.address, self.username, self.password, folder_name, callback_close)
+                        end
                     end,
                 },
             }

frontend/apps/cloudstorage/dropboxapi.lua

@@ -117,7 +117,7 @@ end
 function DropBoxApi:downloadFile(path, token, local_path)
     local data1 = "{\"path\": \"" .. path .. "\"}"
     socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT)
-    local code, _, status = socket.skip(1, http.request{
+    local code, headers, status = socket.skip(1, http.request{
         url     = API_DOWNLOAD_FILE,
         method  = "GET",
         headers = {
@@ -130,12 +130,12 @@ function DropBoxApi:downloadFile(path, token, local_path)
     if code ~= 200 then
         logger.warn("DropBoxApi: Download failure:", status or code or "network unreachable")
     end
-    return code
+    return code, (headers or {}).etag
 end
 
-function DropBoxApi:uploadFile(path, token, file_path)
+function DropBoxApi:uploadFile(path, token, file_path, etag, overwrite)
     local data = "{\"path\": \"" .. path .. "/" .. BaseUtil.basename(file_path) ..
-        "\",\"mode\": \"add\",\"autorename\": true,\"mute\": false,\"strict_conflict\": false}"
+        "\",\"mode\":" .. (overwrite and "\"overwrite\"" or "\"add\"") .. ",\"autorename\": " .. (overwrite and "false" or "true") .. ",\"mute\": false,\"strict_conflict\": false}"
     socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT)
     local code, _, status = socket.skip(1, http.request{
         url     = API_UPLOAD_FILE,
@@ -145,6 +145,7 @@ function DropBoxApi:uploadFile(path, token, file_path)
             ["Dropbox-API-Arg"] = data,
             ["Content-Type"] = "application/octet-stream",
             ["Content-Length"] = lfs.attributes(file_path, "size"),
+            ["If-Match"] = etag,
         },
         source = ltn12.source.file(io.open(file_path, "r")),
     })

frontend/apps/cloudstorage/syncservice.lua

@@ -0,0 +1,175 @@
+local DataStorage = require("datastorage")
+local Font = require("ui/font")
+local InfoMessage = require("ui/widget/infomessage")
+local LuaSettings = require("luasettings")
+local Menu = require("ui/widget/menu")
+local Notification = require("ui/widget/notification")
+local Screen = require("device").screen
+local UIManager = require("ui/uimanager")
+local ffiutil = require("ffi/util")
+local util = require("util")
+
+local _ = require("gettext")
+
+local server_types = {
+    dropbox = _("Dropbox"),
+    webdav = _("WebDAV"),
+}
+local indent = ""
+
+local SyncService = Menu:extend{
+    no_title = false,
+    show_parent = nil,
+    is_popout = false,
+    is_borderless = true,
+    title = _("Cloud sync settings"),
+    title_face = Font:getFace("smallinfofontbold"),
+}
+
+function SyncService:init()
+    self.cs_settings = LuaSettings:open(DataStorage:getSettingsDir().."/cloudstorage.lua")
+    self.item_table = self:generateItemTable()
+    self.width = Screen:getWidth()
+    self.height = Screen:getHeight()
+    Menu.init(self)
+end
+
+function SyncService:generateItemTable()
+    local item_table = {}
+    -- select and/or add server
+    local added_servers = self.cs_settings:readSetting("cs_servers") or {}
+    for _, server in ipairs(added_servers) do
+        if server.type == "dropbox" or server.type == "webdav" then
+            local item = {
+                text = indent .. server.name,
+                address = server.address,
+                username = server.username,
+                password = server.password,
+                type = server.type,
+                url = server.url,
+                mandatory = server_types[server.type],
+            }
+            item.callback = function()
+                require("ui/downloadmgr"):new{
+                item = item,
+                onConfirm = function(path)
+                    server.url = path
+                    self.onConfirm(server)
+                    self:onClose()
+                end,
+                }:chooseCloudDir()
+            end
+            table.insert(item_table, item)
+        end
+    end
+    if #item_table > 0 then
+        table.insert(item_table, 1, {
+            text = _("Choose cloud service:"),
+            bold = true,
+        })
+    end
+    table.insert(item_table, {
+        text = _("Add service"),
+        bold = true,
+        callback = function()
+            local cloud_storage = require("apps/cloudstorage/cloudstorage"):new{}
+            UIManager:show(cloud_storage)
+        end
+    })
+    return item_table
+end
+
+function SyncService.getReadablePath(server)
+    local url = util.stringStartsWith(server.url, "/") and server.url:sub(2) or server.url
+    url = util.urlDecode(url) or url
+    url = util.stringEndsWith(url, "/") and url or url .. "/"
+    url = (server.address:sub(-1) == "/" and server.address or server.address .. "/") .. url
+    if url:sub(-2) == "//" then url = url:sub(1, -2) end
+    return url
+end
+
+-- Prepares three files for sync_cb to call to do the actual syncing:
+-- * local_file (one that is being used)
+-- * income_file (one that has just been downloaded from Cloud to be merged, then to be deleted)
+-- * cached_file (the one that was uploaded in the previous round of syncing)
+--
+-- How it works:
+--
+-- If we simply merge the local file with the income file (ignore duplicates), then items that have been deleted locally
+-- but not remotely (on other devices) will re-emerge in the result file. The same goes for items deleted remotely but
+-- not locally. To avoid this, we first need to delete them from both the income file and local file.
+--
+-- The problem is how to identify them, and that is when the cached file comes into play.
+-- The cached file represents what local and remote agreed on previously (was identical to local and remote after being uploaded
+-- the previous round), by comparing it with local file, items no longer in local file are ones being recently deleted.
+-- The same applies to income file. Then we can delete them from both local and income files to be ready for merging. (The actual
+-- deletion and merging procedures happen in sync_cb as users of this service will have different file specifications)
+--
+-- After merging, the income file is no longer needed and is deleted. The local file is uploaded and then a copy of it is saved
+-- and renamed to replace the old cached file (thus the naming). The cached file stays (in the same folder) till being replaced
+-- in the next round.
+function SyncService.sync(server, file_path, sync_cb, is_silent)
+    local file_name = ffiutil.basename(file_path)
+    local income_file_path = file_path .. ".temp" -- file downloaded from server
+    local cached_file_path = file_path .. ".sync" -- file uploaded to server last time
+
+    local fail_msg = _("Something went wrong when syncing, please check your network connection and try again later.")
+    local show_msg = function(msg)
+        if is_silent then return end
+        UIManager:show(InfoMessage:new{
+            text = msg or fail_msg,
+            timeout = 3,
+        })
+    end
+    if server.type ~= "dropbox" and server.type ~= "webdav" then
+        show_msg(_("Wrong server type."))
+        return
+    end
+    local code_response = 412 -- If-Match header failed
+    local etag
+    local api = server.type == "dropbox" and require("apps/cloudstorage/dropboxapi") or require("apps/cloudstorage/webdavapi")
+    while code_response == 412 do
+        os.remove(income_file_path)
+        if server.type == "dropbox" then
+            local url_base = server.url:sub(-1) == "/" and server.url or server.url.."/"
+            code_response, etag = api:downloadFile(url_base..file_name, server.password, income_file_path)
+        elseif server.type == "webdav" then
+            local path = api:getJoinedPath(server.address, server.url)
+            path = api:getJoinedPath(path, file_name)
+            code_response, etag = api:downloadFile(path, server.username, server.password, income_file_path)
+        end
+        if code_response ~= 200 and code_response ~= 404
+           and not (server.type == "dropbox" and code_response == 409) then
+            show_msg()
+            return
+        end
+        local ok, cb_return = pcall(sync_cb, file_path, cached_file_path, income_file_path)
+        if not ok or not cb_return then
+            show_msg()
+            if not ok then require("logger").err("sync service callback failed:", cb_return) end
+            return
+        end
+        if server.type == "dropbox" then
+            local url_base = server.url == "/" and "" or server.url
+            code_response = api:uploadFile(url_base, server.password, file_path, etag, true)
+        elseif server.type == "webdav" then
+            local path = api:getJoinedPath(server.address, server.url)
+            path = api:getJoinedPath(path, file_name)
+            code_response = api:uploadFile(path, server.username, server.password, file_path, etag)
+        end
+    end
+    os.remove(income_file_path)
+    if type(code_response) == "number" and code_response >= 200 and code_response < 300 then
+        os.remove(cached_file_path)
+        ffiutil.copyFile(file_path, cached_file_path)
+        UIManager:show(Notification:new{
+            text = _("Successfully synchronized."),
+            timeout = 2,
+        })
+    else
+        show_msg()
+    end
+end
+
+
+return SyncService

frontend/apps/cloudstorage/webdav.lua

@@ -7,13 +7,14 @@ local UIManager = require("ui/uimanager")
 local ReaderUI = require("apps/reader/readerui")
 local WebDavApi = require("apps/cloudstorage/webdavapi")
 local util = require("util")
+local ffiutil = require("ffi/util")
 local _ = require("gettext")
 local T = require("ffi/util").template
 
 local WebDav = {}
 
-function WebDav:run(address, user, pass, path)
-    return WebDavApi:listFolder(address, user, pass, path)
+function WebDav:run(address, user, pass, path, folder_mode)
+    return WebDavApi:listFolder(address, user, pass, path, folder_mode)
 end
 
 function WebDav:downloadFile(item, address, username, password, local_path, callback_close)
@@ -48,6 +49,36 @@ function WebDav:downloadFile(item, address, username, password, local_path, call
     end
 end
 
+function WebDav:uploadFile(url, address, username, password, local_path, callback_close)
+    local path = WebDavApi:getJoinedPath(address, url)
+    path = WebDavApi:getJoinedPath(path, ffiutil.basename(local_path))
+    local code_response = WebDavApi:uploadFile(path, username, password, local_path)
+    if code_response >= 200 and code_response < 300 then
+        UIManager:show(InfoMessage:new{
+            text = T(_("File uploaded:\n%1"), BD.filepath(address)),
+        })
+        if callback_close then callback_close() end
+    else
+        UIManager:show(InfoMessage:new{
+            text = T(_("Could not upload file:\n%1"), BD.filepath(address)),
+            timeout = 3,
+        })
+    end
+end
+
+function WebDav:createFolder(url, address, username, password, folder_name, callback_close)
+    local code_response = WebDavApi:createFolder(address .. WebDavApi:urlEncode(url .. "/" .. folder_name), username, password, folder_name)
+    if code_response == 201 then
+        if callback_close then
+            callback_close()
+        end
+    else
+        UIManager:show(InfoMessage:new{
+            text = T(_("Could not create folder:\n%1"), folder_name),
+        })
+    end
+end
+
 function WebDav:config(item, callback)
     local text_info = _([[Server address must be of the form http(s)://domain.name/path
 This can point to a sub-directory of the WebDAV server.

frontend/apps/cloudstorage/webdavapi.lua

@@ -59,7 +59,7 @@ function WebDavApi:urlEncode(url_data)
     return url_data
 end
 
-function WebDavApi:listFolder(address, user, pass, folder_path)
+function WebDavApi:listFolder(address, user, pass, folder_path, folder_mode)
     local path = self:urlEncode( folder_path )
     local webdav_list = {}
     local webdav_file = {}
@@ -169,6 +169,14 @@ function WebDavApi:listFolder(address, user, pass, folder_path)
             type = files.type,
         })
     end
+    if folder_mode then
+        table.insert(webdav_list, 1, {
+            text = _("Long-press to choose current folder"),
+            url = folder_path,
+            type = "folder_long_press",
+            bold = true
+        })
+    end
     return webdav_list
 end
 
@@ -187,7 +195,42 @@ function WebDavApi:downloadFile(file_url, user, pass, local_path)
         logger.warn("WebDavApi: Download failure:", status or code or "network unreachable")
         logger.dbg("WebDavApi: Response headers:", headers)
     end
+    return code, (headers or {}).etag
+end
+
+function WebDavApi:uploadFile(file_url, user, pass, local_path, etag)
+    socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT)
+    local code, _, status = socket.skip(1, http.request{
+        url      = file_url,
+        method   = "PUT",
+        source   = ltn12.source.file(io.open(local_path, "r")),
+        user     = user,
+        password = pass,
+        headers = {
+            ["If-Match"] = etag
+        }
+    })
+    socketutil:reset_timeout()
+    if code < 200 or code > 299 then
+        logger.warn("WebDavApi: upload failure:", status or code or "network unreachable")
+    end
     return code
 end
 
+function WebDavApi:createFolder(folder_url, user, pass, folder_name)
+    socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT)
+    local code, _, status = socket.skip(1, http.request{
+        url      = folder_url,
+        method   = "MKCOL",
+        user     = user,
+        password = pass,
+    })
+    socketutil:reset_timeout()
+    if code ~= 201 then
+        logger.warn("WebDavApi: create folder failure:", status or code or "network unreachable")
+    end
+    return code
+end
+
+
 return WebDavApi

plugins/statistics.koplugin/main.lua

@@ -14,6 +14,7 @@ local ReaderProgress = require("readerprogress")
 local ReadHistory = require("readhistory")
 local Screensaver = require("ui/screensaver")
 local SQ3 = require("lua-ljsqlite3/init")
+local SyncService = require("frontend/apps/cloudstorage/syncservice")
 local UIManager = require("ui/uimanager")
 local Widget = require("ui/widget/widget")
 local lfs = require("libs/libkoreader-lfs")
@@ -33,7 +34,7 @@ local DEFAULT_CALENDAR_START_DAY_OF_WEEK = 2 -- Monday
 local DEFAULT_CALENDAR_NB_BOOK_SPANS = 3
 
 -- Current DB schema version
-local DB_SCHEMA_VERSION = 20201022
+local DB_SCHEMA_VERSION = 20221111
 
 -- This is the query used to compute the total time spent reading distinct pages of the book,
 -- capped at self.settings.max_sec per distinct page.
@@ -381,6 +382,10 @@ Do you want to create an empty database?
                 self:upgradeDBto20201022(conn)
             end
 
+            if db_version < 20221111 then
+                self:upgradeDBto20221111(conn)
+            end
+
             -- Get back the space taken by the deleted page_stat table
             conn:exec("PRAGMA temp_store = 2;") -- use memory for temp files
             local ok, errmsg = pcall(conn.exec, conn, "VACUUM;") -- this may take some time
@@ -541,7 +546,7 @@ function ReaderStatistics:createDB(conn)
     conn:exec(sql_stmt)
     -- Index
     sql_stmt = [[
-        CREATE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5);
+        CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5);
     ]]
     conn:exec(sql_stmt)
 
@@ -598,6 +603,41 @@ function ReaderStatistics:upgradeDBto20201022(conn)
     conn:exec("PRAGMA user_version=20201022;")
 end
 
+function ReaderStatistics:upgradeDBto20221111(conn)
+    conn:exec([[
+        -- We make the index on book's (title, author, md5) unique in order to sync dbs
+        -- First we fill null authors with ''
+        UPDATE book SET authors = '' WHERE authors IS NULL;
+        -- Secondly, we unify the id_book in page_stat_data entries for duplicate books
+        -- to the smallest of each, so as to delete the others.
+        UPDATE page_stat_data SET id_book = (
+            SELECT map.min_id FROM (
+                SELECT id, (
+                    SELECT min(id) FROM book b2
+                    WHERE (book.title, book.authors, book.md5) = (b2.title, b2.authors, b2.md5)
+                ) as min_id
+                FROM book WHERE book.id >= min_id
+            ) as map WHERE page_stat_data.id_book = map.id
+        );
+        -- Delete duplicate books and keep the one with smallest id.
+        DELETE FROM book WHERE id > (
+            SELECT MIN(id) FROM book b2
+            WHERE (book.title, book.authors, book.md5) = (b2.title, b2.authors, b2.md5)
+        );
+        -- Then we recompute the book statistics based on merged books
+        UPDATE book SET (total_read_pages, total_read_time) =
+        (SELECT count(DISTINCT page),
+                sum(duration)
+         FROM   page_stat
+         WHERE  id_book = book.id);
+        -- Finally we update the index to be unique
+        DROP INDEX IF EXISTS book_title_authors_md5;
+        CREATE UNIQUE INDEX book_title_authors_md5 ON book(title, authors, md5);]])
+
+    -- Update DB schema version
+    conn:exec("PRAGMA user_version=20221111;")
+end
+
 function ReaderStatistics:addBookStatToDB(book_stats, conn)
     local id_book
     local last_open_book = 0
@@ -1046,6 +1086,71 @@ The max value ensures a page you stay on for a long time (because you fell aslee
                         callback = function()
                             self.settings.calendar_browse_future_months = not self.settings.calendar_browse_future_months
                         end,
+                        separator = true,
+                    },
+                    {
+                        text = _("Cloud sync"),
+                        callback = function(touchmenu_instance)
+                            local server = self.settings.sync_server
+                            local edit_cb = function()
+                                local sync_settings = SyncService:new{}
+                                sync_settings.onClose = function(this)
+                                    UIManager:close(this)
+                                end
+                                sync_settings.onConfirm = function(sv)
+                                    self.settings.sync_server = sv
+                                    touchmenu_instance:updateItems()
+                                end
+                                UIManager:show(sync_settings)
+                            end
+                            if not server then
+                                edit_cb()
+                                return
+                            end
+                            local dialogue
+                            local delete_button = {
+                                text = _("Delete"),
+                                callback = function()
+                                    UIManager:close(dialogue)
+                                    UIManager:show(ConfirmBox:new{
+                                        text = _("Delete server info?"),
+                                        cancel_text = _("Cancel"),
+                                        cancel_callback = function()
+                                            return
+                                        end,
+                                        ok_text = _("Delete"),
+                                        ok_callback = function()
+                                            self.settings.sync_server = nil
+                                            touchmenu_instance:updateItems()
+                                        end,
+                                    })
+                                end,
+                            }
+                            local edit_button = {
+                                text = _("Edit"),
+                                callback = function()
+                                    UIManager:close(dialogue)
+                                    edit_cb()
+                                end
+                            }
+                            local close_button = {
+                                text = _("Close"),
+                                callback = function()
+                                    UIManager:close(dialogue)
+                                end
+                            }
+                            local type = server.type == "dropbox" and " (DropBox)" or " (WebDAV)"
+                            dialogue = require("ui/widget/buttondialogtitle"):new{
+                                title = T(_("Cloud storage:\n%1\n\nFolder path:\n%2\n\nSet up the same cloud folder on each device to sync across your devices."),
+                                             server.name.." "..type, SyncService.getReadablePath(server)),
+                                buttons = {
+                                    {delete_button, edit_button, close_button}
+                                },
+                            }
+                            UIManager:show(dialogue)
+                        end,
+                        enabled_func = function() return self.settings.is_enabled end,
+                        keep_menu_open = true,
                     },
                 },
             },
@@ -1054,6 +1159,17 @@ The max value ensures a page you stay on for a long time (because you fell aslee
                 sub_item_table = self:genResetBookSubItemTable(),
                 separator = true,
             },
+            {
+                text = _("Synchronize now"),
+                callback = function()
+                    SyncService.sync(self.settings.sync_server, db_location, self.onSync )
+                end,
+                enabled_func = function()
+                    return self.settings.sync_server ~= nil and self.settings.is_enabled and require("ui/network/manager"):isWifiOn()
+                end,
+                keep_menu_open = true,
+                separator = true,
+            },
             {
                 text = _("Current book"),
                 keep_menu_open = true,
@@ -2636,4 +2752,124 @@ function ReaderStatistics:getCurrentBookReadPages()
     return read_pages
 end
 
+function ReaderStatistics.onSync(local_path, cached_path, income_path)
+    local conn_income = SQ3.open(income_path)
+    local ok1, v1 = pcall(conn_income.rowexec, conn_income, "PRAGMA schema_version")
+    if not ok1 or tonumber(v1) == 0 then
+        -- no income db or wrong db, first time sync
+        logger.warn("statistics open income DB failed", v1)
+        return true
+    end
+
+    local sql = "attach '" .. income_path:gsub("'", "''") .."' as income_db;"
+    -- then we try to open cached db
+    local conn_cached = SQ3.open(cached_path)
+    local ok2, v2 = pcall(conn_cached.rowexec, conn_cached, "PRAGMA schema_version")
+    local attached_cache
+    if not ok2 or tonumber(v2) == 0 then
+        -- no cached or error, no item to delete
+        logger.warn("statistics open cached DB failed", v2)
+    else
+        attached_cache = true
+        sql = sql .. "attach '" .. cached_path:gsub("'", "''") ..[[' as cached_db;
+            -- first we delete from income_db books that exist in cached_db but not in local_db,
+            -- namely the ones that were deleted since last sync
+            DELETE FROM income_db.page_stat_data WHERE id_book IN (
+                SELECT id FROM income_db.book WHERE (title, authors, md5) IN (
+                    SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
+                        SELECT title, authors, md5 FROM book
+                    )
+                )
+            );
+            DELETE FROM income_db.book WHERE (title, authors, md5) IN (
+                SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
+                    SELECT title, authors, md5 FROM book
+                )
+            );
+
+            -- then we delete books from local db that were present in last sync but
+            -- not any more (ie. deleted in other devices)
+            DELETE FROM page_stat_data WHERE id_book IN (
+                SELECT id FROM book WHERE (title, authors, md5) IN (
+                    SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
+                        SELECT title, authors, md5 FROM income_db.book
+                    )
+                )
+            );
+            DELETE FROM book WHERE (title, authors, md5) IN (
+                SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
+                    SELECT title, authors, md5 FROM income_db.book
+                )
+            );
+        ]]
+    end
+
+    conn_cached:close()
+    conn_income:close()
+    local conn = SQ3.open(local_path)
+    local ok3, v3 = pcall(conn.exec, conn, "PRAGMA schema_version")
+    if not ok3 or tonumber(v3) == 0 then
+        -- no local db, this is an error
+        logger.err("statistics open local DB", v3)
+        return false
+    end
+
+    sql = sql .. [[
+        -- We merge the local db with income db to form the synced db.
+        -- Do the books
+        INSERT INTO book (
+            title, authors, notes, last_open, highlights, pages, series, language, md5, total_read_time, total_read_pages
+        ) SELECT
+            title, authors, notes, last_open, highlights, pages, series, language, md5, total_read_time, total_read_pages
+        FROM income_db.book WHERE true ON CONFLICT (title, authors, md5) DO NOTHING;
+
+        -- We create a book_id mapping temp table (view not possible due to attached db)
+        CREATE TEMP TABLE book_id_map AS
+            SELECT m.id as mid, ifnull(i.id, m.id) as iid FROM book m --main
+            LEFT JOIN income_db.book i
+            ON (m.title, m.authors, m.md5) = (i.title, i.authors, i.md5);
+        ]]
+    if attached_cache then
+        -- more deletion needed
+        sql = sql .. [[
+        -- DELETE stat_data items
+        DELETE FROM income_db.page_stat_data WHERE (id_book, page, start_time) IN (
+            SELECT map.iid, page, start_time FROM cached_db.page_stat_data
+            LEFT JOIN book_id_map AS map ON id_book = map.mid
+            WHERE (id_book, page, start_time) NOT IN (
+                SELECT id_book, page, start_time FROM page_stat_data
+            )
+        );
+        DELETE FROM page_stat_data WHERE (id_book, page, start_time) IN (
+            SELECT id_book, page, start_time FROM cached_db.page_stat_data WHERE (id_book, page, start_time) NOT IN (
+                SELECT map.mid, page, start_time FROM income_db.page_stat_data
+                LEFT JOIN book_id_map AS map on id_book = map.iid
+            )
+        );]]
+    end
+    sql = sql .. [[
+        -- Then we merge the income_db's contents into the local db
+        INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages)
+            SELECT map.mid, page, start_time, duration, total_pages
+            FROM income_db.page_stat_data
+            LEFT JOIN book_id_map as map
+            ON id_book = map.iid
+            WHERE true
+        ON CONFLICT(id_book, page, start_time) DO UPDATE SET
+        duration = MAX(duration, excluded.duration);
+
+        -- finally we update the total numbers of book
+        UPDATE book SET (total_read_pages, total_read_time) =
+        (SELECT count(DISTINCT page),
+                sum(duration)
+         FROM   page_stat
+         WHERE  id_book = book.id);
+    ]]
+    conn:exec(sql)
+    pcall(conn.exec, conn, "COMMIT;")
+    conn:exec("DETACH income_db;"..(attached_cache and "DETACH cached_db;" or ""))
+    conn:close()
+    return true
+end
+
 return ReaderStatistics

plugins/vocabbuilder.koplugin/db.lua

@@ -2,6 +2,7 @@ local DataStorage = require("datastorage")
 local Device = require("device")
 local SQ3 = require("lua-ljsqlite3/init")
 local LuaData = require("luadata")
+local logger = require("logger")
 
 local db_location = DataStorage:getSettingsDir() .. "/vocabulary_builder.sqlite3"
 
@@ -30,7 +31,9 @@ local VOCABULARY_DB_SCHEMA = [[
     CREATE INDEX IF NOT EXISTS title_name_index ON title(name);
 ]]
 
-local VocabularyBuilder = {}
+local VocabularyBuilder = {
+    path = db_location
+}
 
 function VocabularyBuilder:init()
     VocabularyBuilder:createDB()
@@ -248,6 +251,7 @@ function VocabularyBuilder:batchUpdateItems(items)
             stmt:bind(item.review_count, item.streak_count, item.review_time, item.due_time, item.word)
             stmt:step()
             stmt:clearbind():reset()
+            item.review_time = nil
         end
     end
 
@@ -338,6 +342,104 @@ function VocabularyBuilder:purge()
     conn:close()
 end
 
+
+-- Synchronization
+function VocabularyBuilder.onSync(local_path, cached_path, income_path)
+    -- we try to open income db
+    local conn_income = SQ3.open(income_path)
+    local ok1, v1 = pcall(conn_income.rowexec, conn_income, "PRAGMA schema_version")
+    if not ok1 or tonumber(v1) == 0 then
+        -- no income db or wrong db, first time sync
+        logger.dbg("vocabbuilder open income DB failed", v1)
+        return true
+    end
+
+    local sql = "attach '" .. income_path:gsub("'", "''") .."' as income_db;"
+    -- then we try to open cached db
+    local conn_cached = SQ3.open(cached_path)
+    local ok2, v2 = pcall(conn_cached.rowexec, conn_cached, "PRAGMA schema_version")
+    local attached_cache
+    if not ok2 or tonumber(v2) == 0 then
+        -- no cached or error, no item to delete
+        logger.dbg("vocabbuilder open cached DB failed", v2)
+    else
+        attached_cache = true
+        sql = sql .. "attach '" .. cached_path:gsub("'", "''") ..[[' as cached_db;
+            -- first we delete from income_db words that exist in cached_db but not in local_db,
+            -- namely the ones that were deleted since last sync
+            DELETE FROM income_db.vocabulary WHERE word IN (
+                SELECT word FROM cached_db.vocabulary WHERE word NOT IN (
+                    SELECT word FROM vocabulary
+                )
+            );
+            -- We need to delete words that were delete in income_db since last sync
+            DELETE FROM vocabulary WHERE word IN (
+                SELECT word FROM cached_db.vocabulary WHERE word NOT IN (
+                    SELECT word FROM income_db.vocabulary
+                )
+            );
+        ]]
+    end
+
+    conn_cached:close()
+    conn_income:close()
+    local conn = SQ3.open(local_path)
+    local ok3, v3 = pcall(conn.exec, conn, "PRAGMA schema_version")
+    if not ok3 or tonumber(v3) == 0 then
+        -- no local db, this is an error
+        logger.err("vocabbuilder open local DB", v3)
+        return false
+    end
+
+    sql = sql .. [[
+        -- We merge the local db with income db to form the synced db.
+        -- First we do the books
+        INSERT OR IGNORE INTO title (name) SELECT name FROM income_db.title;
+
+        -- Then update income db's book title id references
+        UPDATE income_db.vocabulary SET title_id = ifnull(
+            (SELECT mid FROM (
+                SELECT m.id as mid, title_id as i_tid FROM title as m -- main db
+                INNER JOIN income_db.title as i -- income db
+                ON m.name = i.name
+                LEFT JOIN income_db.vocabulary
+                on title_id = i.id
+            ) WHERE income_db.vocabulary.title_id = i_tid
+        ) , title_id);
+
+        -- Then we merge the income_db's contents into the local db
+        INSERT INTO vocabulary
+              (word, create_time, review_time, due_time, review_count, prev_context, next_context, title_id, streak_count)
+        SELECT word, create_time, review_time, due_time, review_count, prev_context, next_context, title_id, streak_count
+        FROM income_db.vocabulary WHERE true
+        ON CONFLICT(word) DO UPDATE SET
+        due_time = MAX(due_time, excluded.due_time),
+        review_count = CASE
+            WHEN create_time = excluded.create_time THEN MAX(review_count, excluded.review_count)
+            ELSE review_count + excluded.review_count
+        END,
+        prev_context = ifnull(excluded.prev_context, prev_context),
+        next_context = ifnull(excluded.next_context, next_context),
+        streak_count = CASE
+            WHEN review_time > excluded.review_time THEN streak_count
+            ELSE excluded.streak_count
+        END,
+        review_time = MAX(review_time, excluded.review_time),
+        create_time = excluded.create_time, -- we always use the remote value to eliminate duplicate review_count sum
+        title_id = excluded.title_id -- use remote in case re-assignable book id be supported
+    ]]
+    conn:exec(sql)
+    pcall(conn.exec, conn, "COMMIT;")
+    conn:exec("DETACH income_db;"..(attached_cache and "DETACH cached_db;" or ""))
+    conn:exec("PRAGMA temp_store = 2;") -- use memory for temp files
+    local ok, errmsg = pcall(conn.exec, conn, "VACUUM;") -- we upload a compact file
+    if not ok then
+        logger.warn("Failed compacting vocab database:", errmsg)
+    end
+    conn:close()
+    return true
+end
+
 VocabularyBuilder:init()
 
 return VocabularyBuilder

plugins/vocabbuilder.koplugin/main.lua

@@ -9,6 +9,7 @@ local Blitbuffer = require("ffi/blitbuffer")
 local BottomContainer = require("ui/widget/container/bottomcontainer")
 local DB = require("db")
 local Button = require("ui/widget/button")
+local ButtonDialogTitle = require("ui/widget/buttondialogtitle")
 local ButtonTable = require("ui/widget/buttontable")
 local CenterContainer = require("ui/widget/container/centercontainer")
 local ConfirmBox = require("ui/widget/confirmbox")
@@ -35,6 +36,7 @@ local OverlapGroup = require("ui/widget/overlapgroup")
 local Screen = Device.screen
 local Size = require("ui/size")
 local SortWidget = require("ui/widget/sortwidget")
+local SyncService = require("frontend/apps/cloudstorage/syncservice")
 local TextWidget = require("ui/widget/textwidget")
 local TextBoxWidget = require("ui/widget/textboxwidget")
 local TitleBar = require("ui/widget/titlebar")
@@ -171,7 +173,7 @@ function MenuDialog:init()
     end
 
     local size = Screen:getSize()
-    local width = math.floor(size.w * 0.8)
+    local width = math.floor(size.w * 0.9)
 
     -- Switch text translations could be long
     local temp_text_widget = TextWidget:new{
@@ -277,14 +279,85 @@ function MenuDialog:init()
         end,
     }
 
+    local show_sync_settings = function()
+        if not settings.server then
+            local sync_settings = SyncService:new{}
+            sync_settings.onClose = function(this)
+                UIManager:close(this)
+            end
+            sync_settings.onConfirm = function(server)
+                settings.server = server
+                saveSettings()
+                DB:batchUpdateItems(self.show_parent.item_table)
+                SyncService.sync(server, DB.path, DB.onSync, false)
+                self.show_parent:reloadItems()
+            end
+            UIManager:close(self.sync_dialogue)
+            UIManager:close(self)
+            UIManager:show(sync_settings)
+            return
+        end
+        local server = settings.server
+        local buttons = {
+            {
+                {
+                    text = _("Delete"),
+                    callback = function()
+                        settings.server = nil
+                        UIManager:close(self.sync_dialogue)
+                    end
+                },
+                {
+                    text = _("Edit"),
+                    callback = function()
+                        UIManager:close(self.sync_dialogue)
+                        UIManager:close(self)
+                        local sync_settings = SyncService:new{}
+                        sync_settings.onClose = function(this)
+                            UIManager:close(this)
+                        end
+
+                        sync_settings.onConfirm = function(chosen_server)
+                            settings.server = chosen_server
+                        end
+                        UIManager:show(sync_settings)
+                    end
+                },
+                {
+                    text = _("Synchronize now"),
+                    callback = function()
+                        UIManager:close(self.sync_dialogue)
+                        UIManager:close(self)
+                        DB:batchUpdateItems(self.show_parent.item_table)
+                        SyncService.sync(server, DB.path, DB.onSync, false)
+                        self.show_parent:reloadItems()
+                    end
+                }
+            }
+        }
+        local type = server.type == "dropbox" and " (DropBox)" or " (WebDAV)"
+        self.sync_dialogue = ButtonDialogTitle:new{
+            title = T(_("Cloud storage:\n%1\n\nFolder path:\n%2\n\nSet up the same cloud folder on each device to sync across your devices"),
+                         server.name.." "..type, SyncService.getReadablePath(server)),
+            info_face = Font:getFace("smallinfofont"),
+            buttons = buttons,
+        }
+        UIManager:show(self.sync_dialogue)
+    end
+    local sync_button = {
+        text = _("Cloud sync"),
+        callback = function()
+            show_sync_settings()
+        end
+    }
+
     local buttons = ButtonTable:new{
         width = width,
         buttons = {
-            {filter_button},
             {reverse_button},
-            {edit_button},
-            {reset_button},
-            {clean_button}
+            {sync_button},
+            {filter_button, edit_button},
+            {reset_button, clean_button},
         },
         show_parent = self
     }
@@ -1045,6 +1118,8 @@ function VocabularyBuilderWidget:init()
     self.item_width = self.dimen.w - 2 * padding
     self.footer_center_width = math.floor(self.width_widget * (32/100))
     self.footer_button_width = math.floor(self.width_widget * (12/100))
+    self.footer_left_corner_width = math.floor(self.width_widget * (8/100))
+    self.footer_right_corner_width = math.floor(self.width_widget * (12/100))
     -- group for footer
     local chevron_left = "chevron.left"
     local chevron_right = "chevron.right"
@@ -1091,6 +1166,40 @@ function VocabularyBuilderWidget:init()
         show_parent = self,
     }
 
+    self.footer_sync = Button:new{
+        text = "⇅",
+        width = self.footer_left_corner_width,
+        text_font_size = 18,
+        bordersize = 0,
+        radius = 0,
+        padding = Size.padding.large,
+        show_parent = self,
+        callback = function()
+            if not settings.server then
+                local sync_settings = SyncService:new{}
+                sync_settings.onClose = function(this)
+                    UIManager:close(this)
+                end
+                sync_settings.onConfirm = function(server)
+                    settings.server = server
+                    saveSettings()
+                    DB:batchUpdateItems(self.item_table)
+                    SyncService.sync(server, DB.path, DB.onSync, false)
+                    self:reloadItems()
+                end
+                UIManager:show(sync_settings)
+            else
+                -- manual sync
+                DB:batchUpdateItems(self.item_table)
+                UIManager:nextTick(function()
+                    SyncService.sync(settings.server, DB.path, DB.onSync, false)
+                    self:reloadItems()
+                end)
+            end
+        end
+    }
+    self.footer_sync.label_widget.fgcolor = Blitbuffer.COLOR_GRAY_3
+
     self.footer_page = Button:new{
         text = "",
         hold_input = {
@@ -1117,11 +1226,13 @@ function VocabularyBuilderWidget:init()
         show_parent = self,
     }
     self.page_info = HorizontalGroup:new{
+        self.footer_sync,
         self.footer_first_up,
         self.footer_left,
         self.footer_page,
         self.footer_right,
         self.footer_last_down,
+        HorizontalSpan:new{ width = self.footer_right_corner_width }
     }
 
     local bottom_line = LineWidget:new{
@@ -1270,6 +1381,7 @@ function VocabularyBuilderWidget:_populateItems()
             item
         )
     end
+    table.insert(self.layout, #self.layout, {self.footer_sync})
     if #self.main_content == 0 then
         table.insert(self.main_content, HorizontalSpan:new{width = self.item_width})
     end