Exporter: migrate to rapidjson (#13204)

plugins/exporter.koplugin/base.lua

@@ -8,6 +8,12 @@ Each target should inherit from this class and implement *at least* an `export`
 
 local DataStorage = require("datastorage")
 local Device = require("device")
+local http = require("socket.http")
+local ltn12 = require("ltn12")
+local rapidjson = require("rapidjson")
+local socket = require("socket")
+local socketutil = require("socketutil")
+
 local util = require("util")
 local _ = require("gettext")
 
@@ -160,4 +166,60 @@ function BaseExporter:shareText(text, title)
     Device:doShareText(text, reason, title, self.mimetype)
 end
 
+--[[--
+Makes a json request against a remote endpoint
+
+@param endpoint string url
+@param method string method
+@param body string json string to encode
+@param headers table of additional headers
+
+@treturn response or nil, err
+]]
+
+function BaseExporter:makeJsonRequest(endpoint, method, body, headers)
+    local sink = {}
+    local extra_headers = headers or {}
+    local body_json = rapidjson.encode(body)
+    if not body_json then
+        return nil, "Invalid JSON string"
+    end
+    local source = ltn12.source.string(body_json)
+    socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
+
+    local request = {
+        url = endpoint,
+        method = method,
+        sink = ltn12.sink.table(sink),
+        source = source,
+        headers = {
+            ["Content-Length"] = #body_json,
+            ["Content-Type"] = "application/json",
+        },
+    }
+
+    -- fill in extra headers
+    for k, v in pairs(extra_headers) do
+        request.headers[k] = v
+    end
+
+    local code = socket.skip(1, http.request(request))
+    socketutil:reset_timeout()
+
+    if code ~= 200 then
+        return nil, "Server HTTP response code is not OK"
+    end
+
+    if not sink[1] then
+        return nil, "No response from server"
+    end
+
+    local response, err = rapidjson.decode(sink[1])
+    if not response then
+        return nil, "Unable to decode JSON: " .. err
+    end
+
+    return response
+end
+
 return BaseExporter

plugins/exporter.koplugin/target/joplin.lua

@@ -2,11 +2,9 @@ local InfoMessage = require("ui/widget/infomessage")
 local InputDialog = require("ui/widget/inputdialog")
 local UIManager = require("ui/uimanager")
 local http = require("socket.http")
-local json = require("json")
 local logger = require("logger")
 local ltn12 = require("ltn12")
 local md = require("template/md")
-local socketutil = require("socketutil")
 local T = require("ffi/util").template
 local _ = require("gettext")
 
@@ -18,38 +16,6 @@ local JoplinExporter = require("base"):new {
     version = "1.1.0",
 }
 
-local function makeRequest(url, method, request_body)
-    local sink = {}
-    local request_body_json = json.encode(request_body)
-    local source = ltn12.source.string(request_body_json)
-    socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
-    http.request{
-        url     = url,
-        method  = method,
-        sink    = ltn12.sink.table(sink),
-        source  = source,
-        headers = {
-            ["Content-Length"] = #request_body_json,
-            ["Content-Type"] = "application/json"
-        },
-    }
-    socketutil:reset_timeout()
-
-    if not sink[1] then
-        return nil, "No response from Joplin Server"
-    end
-
-    local response = json.decode(sink[1])
-
-    if not response then
-        return nil, "Unknown response from Joplin Server"
-    elseif response.error then
-        return nil, response.error
-    end
-
-    return response
-end
-
 local function ping(ip, port)
     local sink = {}
     http.request{
@@ -75,7 +41,7 @@ function JoplinExporter:findNoteByTitle(title, notebook_id)
 
     repeat
         url = url_base..page
-        local notes, err = makeRequest(url, "GET")
+        local notes, err = self:makeJsonRequest(url, "GET")
         if not notes then
             logger.warn("Joplin findNoteByTitle error", err)
             return
@@ -101,7 +67,7 @@ function JoplinExporter:findNotebookByTitle(title)
 
     repeat
         url = url_base .. page
-        local folders, err = makeRequest(url, "GET")
+        local folders, err = self:makeJsonRequest(url, "GET")
         if not folders then
             logger.warn("Joplin findNotebookByTitle error", err)
             return
@@ -121,7 +87,7 @@ end
 function JoplinExporter:notebookExist(title)
     local url = string.format("http://%s:%s/folders?token=%s",
         self.settings.ip, self.settings.port, self.settings.token)
-    local response, err = makeRequest(url, "GET")
+    local response, err = self:makeJsonRequest(url, "GET")
     if not response then
         logger.warn("Joplin notebookExist error", err)
         return false
@@ -139,14 +105,14 @@ end
 
 -- If successful returns id of created notebook (folder).
 function JoplinExporter:createNotebook(title, created_time)
-    local request_body = {
+    local body = {
         title = title,
         created_time = created_time
     }
     local url = string.format("http://%s:%s/folders?token=%s",
         self.settings.ip, self.settings.port, self.settings.token)
 
-    local response, err = makeRequest(url, "POST", request_body)
+    local response, err = self:makeJsonRequest(url, "POST", body)
     if not response then
         logger.warn("Joplin createNotebook error", err)
         return
@@ -156,7 +122,7 @@ end
 
 -- If successful returns id of created note.
 function JoplinExporter:createNote(title, note, parent_id, created_time)
-    local request_body = {
+    local body = {
         title = title,
         body = note,
         parent_id = parent_id,
@@ -165,7 +131,7 @@ function JoplinExporter:createNote(title, note, parent_id, created_time)
     local url = string.format("http://%s:%s/notes?token=%s",
         self.settings.ip, self.settings.port, self.settings.token)
 
-    local response, err = makeRequest(url, "POST", request_body)
+    local response, err = self:makeJsonRequest(url, "POST", body)
     if not response then
         logger.warn("Joplin createNote error", err)
         return
@@ -175,14 +141,14 @@ end
 
 -- If successful returns id of updated note.
 function JoplinExporter:updateNote(note, note_id)
-    local request_body = {
+    local body = {
         body = note
     }
 
     local url = string.format("http://%s:%s/notes/%s?token=%s",
         self.settings.ip, self.settings.port, note_id, self.settings.token)
 
-    local response, err = makeRequest(url, "PUT", request_body)
+    local response, err = self:makeJsonRequest(url, "PUT", body)
     if not response then
         logger.warn("Joplin updateNote error", err)
         return

plugins/exporter.koplugin/target/nextcloud.lua

@@ -3,11 +3,6 @@ local MultiInputDialog = require("ui/widget/multiinputdialog")
 local UIManager = require("ui/uimanager")
 local mime = require("mime")
 local md = require("template/md")
-local json = require("json")
-local http = require("socket.http")
-local ltn12 = require("ltn12")
-local socket = require("socket")
-local socketutil = require("socketutil")
 local logger = require("logger")
 local T = require("ffi/util").template
 local _ = require("gettext")
@@ -23,41 +18,6 @@ local NextcloudExporter = require("base"):new {
 -- while we determine wether to update existing or create a new note
 local notes_cache
 
-local function makeRequest(url, auth, method, request_body)
-    local sink = {}
-    local request_body_json = json.encode(request_body)
-    local source = ltn12.source.string(request_body_json)
-
-    local request = {
-        url     = url,
-        method  = method,
-        sink    = ltn12.sink.table(sink),
-        source  = source,
-        headers = {
-            ["Content-Length"] = #request_body_json,
-            ["Content-Type"] = "application/json",
-            ["Authorization"] = "Basic " .. auth,
-            ["OCS-APIRequest"] = "true",
-        },
-    }
-    socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
-    local code, headers, status = socket.skip(1, http.request(request))
-    socketutil:reset_timeout()
-
-    if code ~= 200 then
-        logger.warn("Nextcloud: HTTP response code <> 200. Response status:", status or code or "network unreachable")
-        logger.dbg("Response headers:", headers)
-        return nil, status
-    end
-
-    if not sink[1] then
-        return nil, "No response from Nextcloud"
-    end
-
-    local response = json.decode(sink[1])
-    return response
-end
-
 function NextcloudExporter:isReadyToExport()
     return self.settings.host and self.settings.username and self.settings.password
 end
@@ -171,9 +131,15 @@ function NextcloudExporter:export(t)
     local response
     local err
 
+
+    local json_headers = {
+        ["Authorization"] = "Basic " .. auth,
+        ["OCS-APIRequest"] = "true",
+    }
+
     -- fetch existing notes from Nextcloud
     local url = url_base .. "notes?category=" .. self.category
-    notes_cache, err = makeRequest(url, auth, "GET")
+    notes_cache, err = self:makeJsonRequest(url, "GET", nil, json_headers)
     if not notes_cache then
         logger.warn("Error fetching existing notes from Nextcloud", err)
         return false
@@ -212,7 +178,7 @@ function NextcloudExporter:export(t)
         end
 
         -- save note in Nextcloud
-        response, err = makeRequest(url, auth, verb, request_body)
+        response, err = self:makeJsonRequest(url, verb, request_body, json_headers)
         if not response then
             logger.warn("Error saving note in Nextcloud", err)
             return false

plugins/exporter.koplugin/target/readwise.lua

@@ -1,11 +1,6 @@
 local InputDialog = require("ui/widget/inputdialog")
 local UIManager = require("ui/uimanager")
-local http = require("socket.http")
-local json = require("json")
 local logger = require("logger")
-local ltn12 = require("ltn12")
-local socket = require("socket")
-local socketutil = require("socketutil")
 local _ = require("gettext")
 
 -- readwise exporter
@@ -14,35 +9,6 @@ local ReadwiseExporter = require("base"):new {
     is_remote = true,
 }
 
-local function makeRequest(endpoint, method, request_body, token)
-    local sink = {}
-    local request_body_json = json.encode(request_body)
-    local source = ltn12.source.string(request_body_json)
-    socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
-    local request = {
-        url     = "https://readwise.io/api/v2/" .. endpoint,
-        method  = method,
-        sink    = ltn12.sink.table(sink),
-        source  = source,
-        headers = {
-            ["Content-Length"] = #request_body_json,
-            ["Content-Type"] = "application/json",
-            ["Authorization"] = "Token " .. token
-        },
-    }
-    local code, headers, status = socket.skip(1, http.request(request))
-    socketutil:reset_timeout()
-
-    if code ~= 200 then
-        logger.warn("Readwise: HTTP response code <> 200. Response status:", status or code or "network unreachable")
-        logger.dbg("Response headers:", headers)
-        return nil, status
-    end
-
-    local response = json.decode(sink[1])
-    return response
-end
-
 function ReadwiseExporter:isReadyToExport()
     if self.settings.token then return true end
     return false
@@ -96,6 +62,11 @@ end
 
 function ReadwiseExporter:createHighlights(booknotes)
     local highlights = {}
+
+    local json_headers = {
+        ["Authorization"] = "Token " .. self.settings.token,
+    }
+
     for _, chapter in ipairs(booknotes) do
         for _, clipping in ipairs(chapter) do
             local highlight = {
@@ -113,13 +84,13 @@ function ReadwiseExporter:createHighlights(booknotes)
         end
     end
 
-    local result, err = makeRequest("highlights", "POST", { highlights = highlights }, self.settings.token)
+    local result, err = self:makeJsonRequest("https://readwise.io/api/v2/highlights", "POST",
+         { highlights = highlights }, json_headers)
+
     if not result then
         logger.warn("error creating highlights", err)
         return false
     end
-
-    logger.dbg("createHighlights result", result)
     return true
 end
 

plugins/exporter.koplugin/target/xmnote.lua

@@ -3,12 +3,7 @@ local UIManager = require("ui/uimanager")
 local InfoMessage = require("ui/widget/infomessage")
 local BD = require("ui/bidi")
 local DataStorage = require("datastorage")
-local http = require("socket.http")
-local json = require("json")
 local logger = require("logger")
-local ltn12 = require("ltn12")
-local socket = require("socket")
-local socketutil = require("socketutil")
 local util = require("ffi/util")
 local T = util.template
 local _ = require("gettext")
@@ -103,44 +98,11 @@ function XMNoteExporter:createRequestBody(booknotes)
     return book
 end
 
-function XMNoteExporter:makeRequest(endpoint, method, request_body)
-    local sink = {}
-    local request_body_json = json.encode(request_body)
-    local source = ltn12.source.string(request_body_json)
-    socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
-    local url = "http://".. self.settings.ip .. ":" .. self.server_port .. endpoint
-    local request = {
-        url     = url,
-        method  = method,
-        sink    = ltn12.sink.table(sink),
-        source  = source,
-        headers = {
-            ["Content-Length"] = #request_body_json,
-            ["Content-Type"] = "application/json"
-        },
-    }
-    local code, headers, status = socket.skip(1, http.request(request))
-    socketutil:reset_timeout()
-
-    if code ~= 200 then
-        logger.warn("XMNoteClient: HTTP response code <> 200. Response status: ", status)
-        logger.dbg("Response headers:", headers)
-        return nil, status
-    end
-
-    local response = json.decode(sink[1])
-    local api_code = response["code"]
-    if api_code ~= nil and api_code ~= 200 then
-        logger.warn("XMNoteClient: response code <> 200. message: ", response["message"])
-        logger.dbg("Response headers:", headers)
-        return nil, status
-    end
-    return response
-end
-
 function XMNoteExporter:createHighlights(booknotes)
     local body = self:createRequestBody(booknotes)
-    local result, err = self:makeRequest("/send", "POST", body)
+    local url = "http://".. self.settings.ip .. ":" .. self.server_port .. "/send"
+
+    local result, err = self:makeJsonRequest(url, "POST", body)
     if not result then
         logger.warn("error creating highlights", err)
         return false