From eea6c5465d0f63145f8a743cddf210427281c5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Fern=C3=A1ndez?= <975883+pazos@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:28:55 +0100 Subject: [PATCH] Exporter: migrate to rapidjson (#13204) --- plugins/exporter.koplugin/base.lua | 62 +++++++++++++++++++ plugins/exporter.koplugin/target/joplin.lua | 52 +++------------- .../exporter.koplugin/target/nextcloud.lua | 50 +++------------ plugins/exporter.koplugin/target/readwise.lua | 45 +++----------- plugins/exporter.koplugin/target/xmnote.lua | 44 +------------ 5 files changed, 90 insertions(+), 163 deletions(-) diff --git a/plugins/exporter.koplugin/base.lua b/plugins/exporter.koplugin/base.lua index c1d77cffd..381ab0b41 100644 --- a/plugins/exporter.koplugin/base.lua +++ b/plugins/exporter.koplugin/base.lua @@ -8,6 +8,12 @@ Each target should inherit from this class and implement *at least* an `export` local DataStorage = require("datastorage") local Device = require("device") +local http = require("socket.http") +local ltn12 = require("ltn12") +local rapidjson = require("rapidjson") +local socket = require("socket") +local socketutil = require("socketutil") + local util = require("util") local _ = require("gettext") @@ -160,4 +166,60 @@ function BaseExporter:shareText(text, title) Device:doShareText(text, reason, title, self.mimetype) end +--[[-- +Makes a json request against a remote endpoint + +@param endpoint string url +@param method string method +@param body string json string to encode +@param headers table of additional headers + +@treturn response or nil, err +]] + +function BaseExporter:makeJsonRequest(endpoint, method, body, headers) + local sink = {} + local extra_headers = headers or {} + local body_json = rapidjson.encode(body) + if not body_json then + return nil, "Invalid JSON string" + end + local source = ltn12.source.string(body_json) + socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) + + local request = { + url = endpoint, + method = method, + sink = ltn12.sink.table(sink), + source = source, + headers = { + ["Content-Length"] = #body_json, + ["Content-Type"] = "application/json", + }, + } + + -- fill in extra headers + for k, v in pairs(extra_headers) do + request.headers[k] = v + end + + local code = socket.skip(1, http.request(request)) + socketutil:reset_timeout() + + if code ~= 200 then + return nil, "Server HTTP response code is not OK" + end + + if not sink[1] then + return nil, "No response from server" + end + + local response, err = rapidjson.decode(sink[1]) + if not response then + return nil, "Unable to decode JSON: " .. err + end + + return response +end + return BaseExporter diff --git a/plugins/exporter.koplugin/target/joplin.lua b/plugins/exporter.koplugin/target/joplin.lua index a03bdb83b..2b6163997 100644 --- a/plugins/exporter.koplugin/target/joplin.lua +++ b/plugins/exporter.koplugin/target/joplin.lua @@ -2,11 +2,9 @@ local InfoMessage = require("ui/widget/infomessage") local InputDialog = require("ui/widget/inputdialog") local UIManager = require("ui/uimanager") local http = require("socket.http") -local json = require("json") local logger = require("logger") local ltn12 = require("ltn12") local md = require("template/md") -local socketutil = require("socketutil") local T = require("ffi/util").template local _ = require("gettext") @@ -18,38 +16,6 @@ local JoplinExporter = require("base"):new { version = "1.1.0", } -local function makeRequest(url, method, request_body) - local sink = {} - local request_body_json = json.encode(request_body) - local source = ltn12.source.string(request_body_json) - socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) - http.request{ - url = url, - method = method, - sink = ltn12.sink.table(sink), - source = source, - headers = { - ["Content-Length"] = #request_body_json, - ["Content-Type"] = "application/json" - }, - } - socketutil:reset_timeout() - - if not sink[1] then - return nil, "No response from Joplin Server" - end - - local response = json.decode(sink[1]) - - if not response then - return nil, "Unknown response from Joplin Server" - elseif response.error then - return nil, response.error - end - - return response -end - local function ping(ip, port) local sink = {} http.request{ @@ -75,7 +41,7 @@ function JoplinExporter:findNoteByTitle(title, notebook_id) repeat url = url_base..page - local notes, err = makeRequest(url, "GET") + local notes, err = self:makeJsonRequest(url, "GET") if not notes then logger.warn("Joplin findNoteByTitle error", err) return @@ -101,7 +67,7 @@ function JoplinExporter:findNotebookByTitle(title) repeat url = url_base .. page - local folders, err = makeRequest(url, "GET") + local folders, err = self:makeJsonRequest(url, "GET") if not folders then logger.warn("Joplin findNotebookByTitle error", err) return @@ -121,7 +87,7 @@ end function JoplinExporter:notebookExist(title) local url = string.format("http://%s:%s/folders?token=%s", self.settings.ip, self.settings.port, self.settings.token) - local response, err = makeRequest(url, "GET") + local response, err = self:makeJsonRequest(url, "GET") if not response then logger.warn("Joplin notebookExist error", err) return false @@ -139,14 +105,14 @@ end -- If successful returns id of created notebook (folder). function JoplinExporter:createNotebook(title, created_time) - local request_body = { + local body = { title = title, created_time = created_time } local url = string.format("http://%s:%s/folders?token=%s", self.settings.ip, self.settings.port, self.settings.token) - local response, err = makeRequest(url, "POST", request_body) + local response, err = self:makeJsonRequest(url, "POST", body) if not response then logger.warn("Joplin createNotebook error", err) return @@ -156,7 +122,7 @@ end -- If successful returns id of created note. function JoplinExporter:createNote(title, note, parent_id, created_time) - local request_body = { + local body = { title = title, body = note, parent_id = parent_id, @@ -165,7 +131,7 @@ function JoplinExporter:createNote(title, note, parent_id, created_time) local url = string.format("http://%s:%s/notes?token=%s", self.settings.ip, self.settings.port, self.settings.token) - local response, err = makeRequest(url, "POST", request_body) + local response, err = self:makeJsonRequest(url, "POST", body) if not response then logger.warn("Joplin createNote error", err) return @@ -175,14 +141,14 @@ end -- If successful returns id of updated note. function JoplinExporter:updateNote(note, note_id) - local request_body = { + local body = { body = note } local url = string.format("http://%s:%s/notes/%s?token=%s", self.settings.ip, self.settings.port, note_id, self.settings.token) - local response, err = makeRequest(url, "PUT", request_body) + local response, err = self:makeJsonRequest(url, "PUT", body) if not response then logger.warn("Joplin updateNote error", err) return diff --git a/plugins/exporter.koplugin/target/nextcloud.lua b/plugins/exporter.koplugin/target/nextcloud.lua index 9e9c4a221..a93c252fc 100644 --- a/plugins/exporter.koplugin/target/nextcloud.lua +++ b/plugins/exporter.koplugin/target/nextcloud.lua @@ -3,11 +3,6 @@ local MultiInputDialog = require("ui/widget/multiinputdialog") local UIManager = require("ui/uimanager") local mime = require("mime") local md = require("template/md") -local json = require("json") -local http = require("socket.http") -local ltn12 = require("ltn12") -local socket = require("socket") -local socketutil = require("socketutil") local logger = require("logger") local T = require("ffi/util").template local _ = require("gettext") @@ -23,41 +18,6 @@ local NextcloudExporter = require("base"):new { -- while we determine wether to update existing or create a new note local notes_cache -local function makeRequest(url, auth, method, request_body) - local sink = {} - local request_body_json = json.encode(request_body) - local source = ltn12.source.string(request_body_json) - - local request = { - url = url, - method = method, - sink = ltn12.sink.table(sink), - source = source, - headers = { - ["Content-Length"] = #request_body_json, - ["Content-Type"] = "application/json", - ["Authorization"] = "Basic " .. auth, - ["OCS-APIRequest"] = "true", - }, - } - socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) - local code, headers, status = socket.skip(1, http.request(request)) - socketutil:reset_timeout() - - if code ~= 200 then - logger.warn("Nextcloud: HTTP response code <> 200. Response status:", status or code or "network unreachable") - logger.dbg("Response headers:", headers) - return nil, status - end - - if not sink[1] then - return nil, "No response from Nextcloud" - end - - local response = json.decode(sink[1]) - return response -end - function NextcloudExporter:isReadyToExport() return self.settings.host and self.settings.username and self.settings.password end @@ -171,9 +131,15 @@ function NextcloudExporter:export(t) local response local err + + local json_headers = { + ["Authorization"] = "Basic " .. auth, + ["OCS-APIRequest"] = "true", + } + -- fetch existing notes from Nextcloud local url = url_base .. "notes?category=" .. self.category - notes_cache, err = makeRequest(url, auth, "GET") + notes_cache, err = self:makeJsonRequest(url, "GET", nil, json_headers) if not notes_cache then logger.warn("Error fetching existing notes from Nextcloud", err) return false @@ -212,7 +178,7 @@ function NextcloudExporter:export(t) end -- save note in Nextcloud - response, err = makeRequest(url, auth, verb, request_body) + response, err = self:makeJsonRequest(url, verb, request_body, json_headers) if not response then logger.warn("Error saving note in Nextcloud", err) return false diff --git a/plugins/exporter.koplugin/target/readwise.lua b/plugins/exporter.koplugin/target/readwise.lua index cee58d5a1..1877ed2a0 100644 --- a/plugins/exporter.koplugin/target/readwise.lua +++ b/plugins/exporter.koplugin/target/readwise.lua @@ -1,11 +1,6 @@ local InputDialog = require("ui/widget/inputdialog") local UIManager = require("ui/uimanager") -local http = require("socket.http") -local json = require("json") local logger = require("logger") -local ltn12 = require("ltn12") -local socket = require("socket") -local socketutil = require("socketutil") local _ = require("gettext") -- readwise exporter @@ -14,35 +9,6 @@ local ReadwiseExporter = require("base"):new { is_remote = true, } -local function makeRequest(endpoint, method, request_body, token) - local sink = {} - local request_body_json = json.encode(request_body) - local source = ltn12.source.string(request_body_json) - socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) - local request = { - url = "https://readwise.io/api/v2/" .. endpoint, - method = method, - sink = ltn12.sink.table(sink), - source = source, - headers = { - ["Content-Length"] = #request_body_json, - ["Content-Type"] = "application/json", - ["Authorization"] = "Token " .. token - }, - } - local code, headers, status = socket.skip(1, http.request(request)) - socketutil:reset_timeout() - - if code ~= 200 then - logger.warn("Readwise: HTTP response code <> 200. Response status:", status or code or "network unreachable") - logger.dbg("Response headers:", headers) - return nil, status - end - - local response = json.decode(sink[1]) - return response -end - function ReadwiseExporter:isReadyToExport() if self.settings.token then return true end return false @@ -96,6 +62,11 @@ end function ReadwiseExporter:createHighlights(booknotes) local highlights = {} + + local json_headers = { + ["Authorization"] = "Token " .. self.settings.token, + } + for _, chapter in ipairs(booknotes) do for _, clipping in ipairs(chapter) do local highlight = { @@ -113,13 +84,13 @@ function ReadwiseExporter:createHighlights(booknotes) end end - local result, err = makeRequest("highlights", "POST", { highlights = highlights }, self.settings.token) + local result, err = self:makeJsonRequest("https://readwise.io/api/v2/highlights", "POST", + { highlights = highlights }, json_headers) + if not result then logger.warn("error creating highlights", err) return false end - - logger.dbg("createHighlights result", result) return true end diff --git a/plugins/exporter.koplugin/target/xmnote.lua b/plugins/exporter.koplugin/target/xmnote.lua index 442519b00..9e5c8c785 100644 --- a/plugins/exporter.koplugin/target/xmnote.lua +++ b/plugins/exporter.koplugin/target/xmnote.lua @@ -3,12 +3,7 @@ local UIManager = require("ui/uimanager") local InfoMessage = require("ui/widget/infomessage") local BD = require("ui/bidi") local DataStorage = require("datastorage") -local http = require("socket.http") -local json = require("json") local logger = require("logger") -local ltn12 = require("ltn12") -local socket = require("socket") -local socketutil = require("socketutil") local util = require("ffi/util") local T = util.template local _ = require("gettext") @@ -103,44 +98,11 @@ function XMNoteExporter:createRequestBody(booknotes) return book end -function XMNoteExporter:makeRequest(endpoint, method, request_body) - local sink = {} - local request_body_json = json.encode(request_body) - local source = ltn12.source.string(request_body_json) - socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) - local url = "http://".. self.settings.ip .. ":" .. self.server_port .. endpoint - local request = { - url = url, - method = method, - sink = ltn12.sink.table(sink), - source = source, - headers = { - ["Content-Length"] = #request_body_json, - ["Content-Type"] = "application/json" - }, - } - local code, headers, status = socket.skip(1, http.request(request)) - socketutil:reset_timeout() - - if code ~= 200 then - logger.warn("XMNoteClient: HTTP response code <> 200. Response status: ", status) - logger.dbg("Response headers:", headers) - return nil, status - end - - local response = json.decode(sink[1]) - local api_code = response["code"] - if api_code ~= nil and api_code ~= 200 then - logger.warn("XMNoteClient: response code <> 200. message: ", response["message"]) - logger.dbg("Response headers:", headers) - return nil, status - end - return response -end - function XMNoteExporter:createHighlights(booknotes) local body = self:createRequestBody(booknotes) - local result, err = self:makeRequest("/send", "POST", body) + local url = "http://".. self.settings.ip .. ":" .. self.server_port .. "/send" + + local result, err = self:makeJsonRequest(url, "POST", body) if not result then logger.warn("error creating highlights", err) return false