NewsDownloader externalize download engine (#3618)

plugins/newsdownloader.koplugin/internaldownloadbackend.lua

@@ -0,0 +1,24 @@
+local http = require("socket.http")
+local https = require("ssl.https")
+local logger = require("logger")
+local ltn12 = require("ltn12")
+local socket_url = require("socket.url")
+
+local InternalDownloadBackend = {}
+
+function InternalDownloadBackend:getResponseAsString(url)
+    local resp_lines = {}
+    local parsed = socket_url.parse(url)
+    local httpRequest = parsed.scheme == 'http' and http.request or https.request
+    httpRequest({ url = url, sink = ltn12.sink.table(resp_lines), })
+    return table.concat(resp_lines)
+end
+
+function InternalDownloadBackend:download(link, path)
+    logger.dbg("InternalDownloadBackend: News file will be stored to :", path)
+    local parsed = socket_url.parse(link)
+    local httpRequest = parsed.scheme == 'http' and http.request or https.request
+    httpRequest({ url = link, sink = ltn12.sink.file(io.open(path, "w")), })
+end
+
+return InternalDownloadBackend

plugins/newsdownloader.koplugin/luahttpdownloadbackend.lua

@@ -0,0 +1,20 @@
+local logger = require("logger")
+local http_request = require "http.request"
+
+-- Currently unused. TODO @mwoz123 ADD LUA-HTTP AS LIBRARY
+local LuaHttpDownloadBackend = {}
+
+function LuaHttpDownloadBackend:getResponseAsString(url)
+   local _, stream = assert(http_request.new_from_uri(url):go())
+   local body = assert(stream:get_body_as_string())
+   logger.dbg("Response body :", body)
+   return body
+end
+
+function LuaHttpDownloadBackend:download(link, path)
+   local _, stream = assert(http_request.new_from_uri(link):go())
+   stream:save_body_to_file(path)
+   stream:shutdown()
+end
+
+return LuaHttpDownloadBackend

plugins/newsdownloader.koplugin/main.lua

@@ -1,4 +1,6 @@
 local DataStorage = require("datastorage")
+local DownloadBackend = require("internaldownloadbackend")
+--local DownloadBackend = require("luahttpdownloadbackend")
 local ReadHistory = require("readhistory")
 local FFIUtil = require("ffi/util")
 local InfoMessage = require("ui/widget/infomessage")
@@ -7,11 +9,7 @@ local UIManager = require("ui/uimanager")
 local NetworkMgr = require("ui/network/manager")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
 local ffi = require("ffi")
-local http = require("socket.http")
-local https = require("ssl.https")
 local logger = require("logger")
-local ltn12 = require("ltn12")
-local socket_url = require("socket.url")
 local util = require("util")
 local _ = require("gettext")
 local T = FFIUtil.template
@@ -66,7 +64,6 @@ function NewsDownloader:init()
     self.ui.menu:registerToMainMenu(self)
 end
 
-
 function NewsDownloader:addToMainMenu(menu_items)
     self:lazyInitialization()
     menu_items.news_downloader = {
@@ -212,13 +209,16 @@ function NewsDownloader:loadConfigAndProcessFeeds()
 end
 
 function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls, download_full_article)
-    local resp_lines = {}
-    local parsed = socket_url.parse(url)
-    local httpRequest = parsed.scheme == 'http' and http.request or https.request
-    httpRequest({ url = url, sink = ltn12.sink.table(resp_lines), })
-    local feeds = self:deserializeXMLString(table.concat(resp_lines))
 
-    if not feeds then
+    local ok, response = pcall(function()
+        return DownloadBackend:getResponseAsString(url)
+    end)
+    local feeds
+    if ok then
+        feeds = self:deserializeXMLString(response)
+    end
+
+    if not ok or not feeds then
         table.insert(unsupported_feeds_urls, url)
         return
     end
@@ -227,12 +227,16 @@ function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls, do
     local is_atom = feeds.feed and feeds.feed.title and feeds.feed.entry[1] and feeds.feed.entry[1].title and feeds.feed.entry[1].link
 
     if is_atom then
-        self:processAtom(feeds, limit, download_full_article)
+        ok = pcall(function()
+            return self:processAtom(feeds, limit, download_full_article)
+        end)
     elseif is_rss then
-        self:processRSS(feeds, limit, download_full_article)
-    else
+        ok = pcall(function()
+            return self:processRSS(feeds, limit, download_full_article)
+        end)
+    end
+    if not ok or (not is_rss and not is_atom) then
         table.insert(unsupported_feeds_urls, url)
-        return
     end
 end
 
@@ -299,9 +303,7 @@ function NewsDownloader:downloadFeed(feed, feed_output_dir)
                                                file_extension)
     logger.dbg("NewsDownloader: News file will be stored to :", news_dl_path)
 
-    local parsed = socket_url.parse(link)
-    local httpRequest = parsed.scheme == 'http' and http.request or https.request
-    httpRequest({ url = link, sink = ltn12.sink.file(io.open(news_dl_path, 'w')), })
+    DownloadBackend:download(link, news_dl_path)
 end
 
 function NewsDownloader:createFromDescription(feed, context, feed_output_dir)