renun/koreader
1
0
Fork 0
mirror of https://github.com/koreader/koreader.git synced 2025-08-10 00:52:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

NewsDownloader: add support for cache-control and retry-after header
Some checks are pending
macos / macOS ${{ matrix.image }} ${{ matrix.platform }} 🔨${{ matrix.xcode_version }} 🎯${{ matrix.deployment_target }} (10.15, 13, x86-64, 15.2) (push) Waiting to run
Details
macos / macOS ${{ matrix.image }} ${{ matrix.platform }} 🔨${{ matrix.xcode_version }} 🎯${{ matrix.deployment_target }} (11.0, 14, ARM64, 15.4) (push) Waiting to run
Details

Browse Source
This commit is contained in:
Frans de Jonge
2025-01-31 23:04:03 +01:00
committed by Frans de Jonge
parent 7bc8f9bd3d
commit 2604f7da8d
1 changed files with 57 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
plugins/newsdownloader.koplugin/main.lua
View File

@@ -354,41 +354,67 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
end
if cached_response then
logger.dbg("NewsDownloader: Using cached response for ", url)
logger.dbg("NewsDownloader: Checking cache validity for:", url)
local headers_cached = cached_response.headers
logger.dbg("NewsDownloader: Cached response headers", headers_cached)
local etag = headers_cached["etag"]
local last_modified = headers_cached["last-modified"]
if etag or last_modified then
logger.dbg("NewsDownloader: requesting with If-Modified-Since:", last_modified, "If-None-Match:", etag, url)
local response_body = {}
local headers = {
["If-Modified-Since"] = last_modified,
["If-None-Match"] = etag,
}
if cookies then
headers["Cookie"] = cookies
local cache_control = headers_cached["cache-control"]
local retry_after = headers_cached["retry-after"]
if (cache_control and cache_control:find("max%-age")) or retry_after then
local max_age = cache_control and tonumber(cache_control:match("max%-age=(%d+)")) or 0
local retry = retry_after and tonumber(retry_after) or 0
local timeout = math.min(43200, math.max(max_age, retry)) -- Limit to 12 hours.
if timeout then
local last_access = headers_cached["date"]
if last_access then
logger.dbg("NewsDownloader: Checking cache validity for:", url, "last_access", last_access, "timeout", timeout)
local last_access_time = dateparser.parse(last_access)
if last_access_time then
local now = os.time()
local diff = now - last_access_time
if diff < timeout then
logger.dbg("NewsDownloader: Using cached response for:", url, "max-age:", max_age, "retry-after:", retry_after, "timeout:", timeout, "diff:", diff)
response = cached_response.content
ok = true
end
end
end
end
local code, response_headers = socket.skip(1, http.request{
url = url,
headers = headers,
sink = ltn12.sink.table(response_body)
})
logger.dbg("NewsDownloader: If-Modified-Since response", code, response_headers)
if code == 304 then
ok = true
response = cached_response.content
-- Update cached headers.
cached_response.headers = response_headers
cache:insert(url, cached_response)
elseif code == 200 then
ok = true
response = table.concat(response_body)
-- Update cached response.
cached_response.headers = response_headers
cached_response.content = response
cache:insert(url, cached_response)
end
if not ok then
local etag = headers_cached["etag"]
local last_modified = headers_cached["last-modified"]
if etag or last_modified then
logger.dbg("NewsDownloader: requesting with If-Modified-Since:", last_modified, "If-None-Match:", etag, url)
local response_body = {}
local headers = {
["If-Modified-Since"] = last_modified,
["If-None-Match"] = etag,
}
if cookies then
headers["Cookie"] = cookies
end
local code, response_headers = socket.skip(1, http.request{
url = url,
headers = headers,
sink = ltn12.sink.table(response_body)
})
logger.dbg("NewsDownloader: If-Modified-Since/If-None-Match response", code, response_headers)
if code == 304 then
ok = true
response = cached_response.content
-- Update cached headers.
cached_response.headers = response_headers
cache:insert(url, cached_response)
elseif code == 200 then
ok = true
response = table.concat(response_body)
-- Update cached response.
cached_response.headers = response_headers
cached_response.content = response
cache:insert(url, cached_response)
end
end
end
end