diff --git a/RSCore/Sources/RSCore/Shared/Array+RSCore.swift b/RSCore/Sources/RSCore/Array+RSCore.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Array+RSCore.swift rename to RSCore/Sources/RSCore/Array+RSCore.swift diff --git a/RSCore/Sources/RSCore/Shared/BatchUpdate.swift b/RSCore/Sources/RSCore/BatchUpdate.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/BatchUpdate.swift rename to RSCore/Sources/RSCore/BatchUpdate.swift diff --git a/RSCore/Sources/RSCore/Shared/BinaryDiskCache.swift b/RSCore/Sources/RSCore/BinaryDiskCache.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/BinaryDiskCache.swift rename to RSCore/Sources/RSCore/BinaryDiskCache.swift diff --git a/RSCore/Sources/RSCore/Shared/Blocks.swift b/RSCore/Sources/RSCore/Blocks.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Blocks.swift rename to RSCore/Sources/RSCore/Blocks.swift diff --git a/RSCore/Sources/RSCore/Cache.swift b/RSCore/Sources/RSCore/Cache.swift new file mode 100644 index 000000000..33ee208b5 --- /dev/null +++ b/RSCore/Sources/RSCore/Cache.swift @@ -0,0 +1,85 @@ +// +// Cache.swift +// +// +// Created by Brent Simmons on 10/12/24. +// + +import Foundation +import os + +public protocol CacheRecord: Sendable { + var dateCreated: Date { get } +} + +public final class Cache: Sendable { + + public let timeToLive: TimeInterval + public let timeBetweenCleanups: TimeInterval + + private struct State: Sendable { + var lastCleanupDate = Date() + var cache = [String: T]() + } + + private let stateLock = OSAllocatedUnfairLock(initialState: State()) + + public init(timeToLive: TimeInterval, timeBetweenCleanups: TimeInterval) { + self.timeToLive = timeToLive + self.timeBetweenCleanups = timeBetweenCleanups + } + + public subscript(_ key: String) -> T? { + get { + stateLock.withLock { state in + + cleanupIfNeeded(&state) + + guard let value = state.cache[key] else { + return nil + } + if value.dateCreated.timeIntervalSinceNow < -timeToLive { + state.cache[key] = nil + return nil + } + + return value + } + } + set { + stateLock.withLock { state in + state.cache[key] = newValue + } + } + } + + public func cleanup() { + stateLock.withLock { state in + cleanupIfNeeded(&state) + } + } +} + +extension Cache { + + private func cleanupIfNeeded(_ state: inout State) { + + let currentDate = Date() + guard state.lastCleanupDate.timeIntervalSince(currentDate) < -timeBetweenCleanups else { + return + } + + var keysToDelete = [String]() + for (key, value) in state.cache { + if value.dateCreated.timeIntervalSince(currentDate) < -timeToLive { + keysToDelete.append(key) + } + } + + for key in keysToDelete { + state.cache[key] = nil + } + + state.lastCleanupDate = Date() + } +} diff --git a/RSCore/Sources/RSCore/Shared/Calendar+RSCore.swift b/RSCore/Sources/RSCore/Calendar+RSCore.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Calendar+RSCore.swift rename to RSCore/Sources/RSCore/Calendar+RSCore.swift diff --git a/RSCore/Sources/RSCore/Shared/Character+RSCore.swift b/RSCore/Sources/RSCore/Character+RSCore.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Character+RSCore.swift rename to RSCore/Sources/RSCore/Character+RSCore.swift diff --git a/RSCore/Sources/RSCore/Shared/CoalescingQueue.swift b/RSCore/Sources/RSCore/CoalescingQueue.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/CoalescingQueue.swift rename to RSCore/Sources/RSCore/CoalescingQueue.swift diff --git a/RSCore/Sources/RSCore/Shared/Data+RSCore.swift b/RSCore/Sources/RSCore/Data+RSCore.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Data+RSCore.swift rename to RSCore/Sources/RSCore/Data+RSCore.swift diff --git a/RSCore/Sources/RSCore/Shared/Date+Extensions.swift b/RSCore/Sources/RSCore/Date+Extensions.swift similarity index 73% rename from RSCore/Sources/RSCore/Shared/Date+Extensions.swift rename to RSCore/Sources/RSCore/Date+Extensions.swift index 48421cbe5..f1ef88f01 100755 --- a/RSCore/Sources/RSCore/Shared/Date+Extensions.swift +++ b/RSCore/Sources/RSCore/Date+Extensions.swift @@ -16,14 +16,22 @@ public extension Date { return addingTimeInterval(0.0 - TimeInterval(days: days)) } + func bySubtracting(hours: Int) -> Date { + return addingTimeInterval(0.0 - TimeInterval(hours: hours)) + } + func byAdding(days: Int) -> Date { return addingTimeInterval(TimeInterval(days: days)) } } -private extension TimeInterval { +public extension TimeInterval { init(days: Int) { self.init(days * 24 * 60 * 60) } + + init(hours: Int) { + self.init(hours * 60 * 60) + } } diff --git a/RSCore/Sources/RSCore/Shared/Dictionary+RSCore.swift b/RSCore/Sources/RSCore/Dictionary+RSCore.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Dictionary+RSCore.swift rename to RSCore/Sources/RSCore/Dictionary+RSCore.swift diff --git a/RSCore/Sources/RSCore/Shared/DisplayNameProvider.swift b/RSCore/Sources/RSCore/DisplayNameProvider.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/DisplayNameProvider.swift rename to RSCore/Sources/RSCore/DisplayNameProvider.swift diff --git a/RSCore/Sources/RSCore/Shared/FileManager+RSCore.swift b/RSCore/Sources/RSCore/FileManager+RSCore.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/FileManager+RSCore.swift rename to RSCore/Sources/RSCore/FileManager+RSCore.swift diff --git a/RSCore/Sources/RSCore/Shared/Geometry.swift b/RSCore/Sources/RSCore/Geometry.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Geometry.swift rename to RSCore/Sources/RSCore/Geometry.swift diff --git a/RSCore/Sources/RSCore/Shared/MacroProcessor.swift b/RSCore/Sources/RSCore/MacroProcessor.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/MacroProcessor.swift rename to RSCore/Sources/RSCore/MacroProcessor.swift diff --git a/RSCore/Sources/RSCore/Shared/MainThreadBlockOperation.swift b/RSCore/Sources/RSCore/MainThreadBlockOperation.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/MainThreadBlockOperation.swift rename to RSCore/Sources/RSCore/MainThreadBlockOperation.swift diff --git a/RSCore/Sources/RSCore/Shared/MainThreadOperation.swift b/RSCore/Sources/RSCore/MainThreadOperation.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/MainThreadOperation.swift rename to RSCore/Sources/RSCore/MainThreadOperation.swift diff --git a/RSCore/Sources/RSCore/Shared/MainThreadOperationQueue.swift b/RSCore/Sources/RSCore/MainThreadOperationQueue.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/MainThreadOperationQueue.swift rename to RSCore/Sources/RSCore/MainThreadOperationQueue.swift diff --git a/RSCore/Sources/RSCore/Shared/ManagedResourceFile.swift b/RSCore/Sources/RSCore/ManagedResourceFile.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/ManagedResourceFile.swift rename to RSCore/Sources/RSCore/ManagedResourceFile.swift diff --git a/RSCore/Sources/RSCore/Shared/OPMLRepresentable.swift b/RSCore/Sources/RSCore/OPMLRepresentable.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/OPMLRepresentable.swift rename to RSCore/Sources/RSCore/OPMLRepresentable.swift diff --git a/RSCore/Sources/RSCore/Shared/Platform.swift b/RSCore/Sources/RSCore/Platform.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Platform.swift rename to RSCore/Sources/RSCore/Platform.swift diff --git a/RSCore/Sources/RSCore/Shared/PropertyList.swift b/RSCore/Sources/RSCore/PropertyList.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/PropertyList.swift rename to RSCore/Sources/RSCore/PropertyList.swift diff --git a/RSCore/Sources/RSCore/RSCore.swift b/RSCore/Sources/RSCore/RSCore.swift deleted file mode 100644 index a3b8373de..000000000 --- a/RSCore/Sources/RSCore/RSCore.swift +++ /dev/null @@ -1,3 +0,0 @@ -struct RSCore { - var text = "Hello, World!" -} diff --git a/RSCore/Sources/RSCore/Shared/RSImage.swift b/RSCore/Sources/RSCore/RSImage.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/RSImage.swift rename to RSCore/Sources/RSCore/RSImage.swift diff --git a/RSCore/Sources/RSCore/Shared/RSScreen.swift b/RSCore/Sources/RSCore/RSScreen.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/RSScreen.swift rename to RSCore/Sources/RSCore/RSScreen.swift diff --git a/RSCore/Sources/RSCore/Shared/Renamable.swift b/RSCore/Sources/RSCore/Renamable.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Renamable.swift rename to RSCore/Sources/RSCore/Renamable.swift diff --git a/RSCore/Sources/RSCore/Shared/SendToBlogEditorApp.swift b/RSCore/Sources/RSCore/SendToBlogEditorApp.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/SendToBlogEditorApp.swift rename to RSCore/Sources/RSCore/SendToBlogEditorApp.swift diff --git a/RSCore/Sources/RSCore/Shared/SendToCommand.swift b/RSCore/Sources/RSCore/SendToCommand.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/SendToCommand.swift rename to RSCore/Sources/RSCore/SendToCommand.swift diff --git a/RSCore/Sources/RSCore/Shared/Set+Extensions.swift b/RSCore/Sources/RSCore/Set+Extensions.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/Set+Extensions.swift rename to RSCore/Sources/RSCore/Set+Extensions.swift diff --git a/RSCore/Sources/RSCore/Shared/String+RSCore.swift b/RSCore/Sources/RSCore/String+RSCore.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/String+RSCore.swift rename to RSCore/Sources/RSCore/String+RSCore.swift diff --git a/RSCore/Sources/RSCore/Shared/UndoableCommand.swift b/RSCore/Sources/RSCore/UndoableCommand.swift similarity index 100% rename from RSCore/Sources/RSCore/Shared/UndoableCommand.swift rename to RSCore/Sources/RSCore/UndoableCommand.swift diff --git a/RSWeb/Package.swift b/RSWeb/Package.swift index 6c3645ba4..55fba142c 100644 --- a/RSWeb/Package.swift +++ b/RSWeb/Package.swift @@ -12,10 +12,16 @@ let package = Package( targets: ["RSWeb"]), ], dependencies: [ + .package(path: "../Parser"), + .package(path: "../RSCore"), ], targets: [ .target( name: "RSWeb", + dependencies: [ + "Parser", + "RSCore" + ], swiftSettings: [.unsafeFlags(["-warnings-as-errors"])] ), .testTarget( diff --git a/RSWeb/Sources/RSWeb/CacheControlInfo.swift b/RSWeb/Sources/RSWeb/CacheControlInfo.swift new file mode 100644 index 000000000..d6de46d38 --- /dev/null +++ b/RSWeb/Sources/RSWeb/CacheControlInfo.swift @@ -0,0 +1,66 @@ +// +// CacheControl.swift +// RSWeb +// +// Created by Brent Simmons on 11/30/24. +// + +import Foundation + +/// Basic Cache-Control handling — just the part we need, +/// which is to know when we got the response (dateCreated) +/// and when we can ask again (canResume). +public struct CacheControlInfo: Codable, Equatable { + + let dateCreated: Date + let maxAge: TimeInterval + + var resumeDate: Date { + dateCreated + maxAge + } + public var canResume: Bool { + Date() >= resumeDate + } + + public init?(urlResponse: HTTPURLResponse) { + guard let cacheControlValue = urlResponse.valueForHTTPHeaderField(HTTPResponseHeader.cacheControl) else { + return nil + } + self.init(value: cacheControlValue) + } + + /// Returns nil if there’s no max-age or it’s < 1. + public init?(value: String) { + + guard let maxAge = Self.parseMaxAge(value) else { + return nil + } + + let d = Date() + self.dateCreated = d + self.maxAge = maxAge + } +} + +private extension CacheControlInfo { + + static let maxAgePrefix = "max-age=" + static let maxAgePrefixCount = maxAgePrefix.count + + static func parseMaxAge(_ s: String) -> TimeInterval? { + + let components = s.components(separatedBy: ",") + let trimmedComponents = components.map { $0.trimmingCharacters(in: .whitespaces) } + + for component in trimmedComponents { + if component.hasPrefix(Self.maxAgePrefix) { + let maxAgeStringValue = component.dropFirst(maxAgePrefixCount) + if let timeInterval = TimeInterval(maxAgeStringValue), timeInterval > 0 { + return timeInterval + } + } + } + + return nil + } +} diff --git a/RSWeb/Sources/RSWeb/DownloadObject.swift b/RSWeb/Sources/RSWeb/DownloadObject.swift deleted file mode 100755 index bc8b3f73b..000000000 --- a/RSWeb/Sources/RSWeb/DownloadObject.swift +++ /dev/null @@ -1,32 +0,0 @@ -// -// DownloadObject.swift -// RSWeb -// -// Created by Brent Simmons on 8/3/16. -// Copyright © 2016 Ranchero Software, LLC. All rights reserved. -// - -import Foundation - -public final class DownloadObject: Hashable { - - public let url: URL - public var data = Data() - - public init(url: URL) { - self.url = url - } - - // MARK: - Hashable - - public func hash(into hasher: inout Hasher) { - hasher.combine(url) - } - - // MARK: - Equatable - - public static func ==(lhs: DownloadObject, rhs: DownloadObject) -> Bool { - return lhs.url == rhs.url && lhs.data == rhs.data - } -} - diff --git a/RSWeb/Sources/RSWeb/Downloader.swift b/RSWeb/Sources/RSWeb/Downloader.swift new file mode 100755 index 000000000..c0dc12ca8 --- /dev/null +++ b/RSWeb/Sources/RSWeb/Downloader.swift @@ -0,0 +1,56 @@ +// +// Downloader.swift +// RSWeb +// +// Created by Brent Simmons on 8/27/16. +// Copyright © 2016 Ranchero Software, LLC. All rights reserved. +// + +import Foundation + +public typealias DownloadCallback = (Data?, URLResponse?, Error?) -> Swift.Void + +/// Simple downloader, for a one-shot download like an image +/// or a web page. For a download-feeds session, see DownloadSession. +public final class Downloader { + + public static let shared = Downloader() + private let urlSession: URLSession + + private init() { + + let sessionConfiguration = URLSessionConfiguration.ephemeral + sessionConfiguration.requestCachePolicy = .reloadIgnoringLocalCacheData + sessionConfiguration.httpShouldSetCookies = false + sessionConfiguration.httpCookieAcceptPolicy = .never + sessionConfiguration.httpMaximumConnectionsPerHost = 1 + sessionConfiguration.httpCookieStorage = nil + + if let userAgentHeaders = UserAgent.headers() { + sessionConfiguration.httpAdditionalHeaders = userAgentHeaders + } + + urlSession = URLSession(configuration: sessionConfiguration) + } + + deinit { + urlSession.invalidateAndCancel() + } + + public func download(_ url: URL, _ completion: DownloadCallback? = nil) { + download(URLRequest(url: url), completion) + } + + public func download(_ urlRequest: URLRequest, _ completion: DownloadCallback? = nil) { + + var urlRequestToUse = urlRequest + urlRequestToUse.addSpecialCaseUserAgentIfNeeded() + + let task = urlSession.dataTask(with: urlRequestToUse) { (data, response, error) in + DispatchQueue.main.async() { + completion?(data, response, error) + } + } + task.resume() + } +} diff --git a/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift b/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift new file mode 100644 index 000000000..b213212d2 --- /dev/null +++ b/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift @@ -0,0 +1,47 @@ +// +// HTMLMetadataCache.swift +// +// +// Created by Brent Simmons on 10/13/24. +// + +import Foundation +import Parser +import RSCore + +extension Notification.Name { + // Sent when HTMLMetadata is cached. Posted on any thread. + static let htmlMetadataAvailable = Notification.Name("htmlMetadataAvailable") +} + +final class HTMLMetadataCache: Sendable { + + static let shared = HTMLMetadataCache() + + // Sent along with .htmlMetadataAvailable notification + struct UserInfoKey { + static let htmlMetadata = "htmlMetadata" + static let url = "url" // String value + } + + private struct HTMLMetadataCacheRecord: CacheRecord { + let metadata: HTMLMetadata + let dateCreated = Date() + } + + private let cache = Cache(timeToLive: TimeInterval(hours: 21), timeBetweenCleanups: TimeInterval(hours: 10)) + + subscript(_ url: String) -> HTMLMetadata? { + get { + return cache[url]?.metadata + } + set { + guard let htmlMetadata = newValue else { + return + } + let cacheRecord = HTMLMetadataCacheRecord(metadata: htmlMetadata) + cache[url] = cacheRecord + NotificationCenter.default.post(name: .htmlMetadataAvailable, object: self, userInfo: [UserInfoKey.htmlMetadata: htmlMetadata, UserInfoKey.url: url]) + } + } +} diff --git a/RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift b/RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift new file mode 100644 index 000000000..c113f6c89 --- /dev/null +++ b/RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift @@ -0,0 +1,120 @@ +// +// HTMLMetadataDownloader.swift +// NetNewsWire +// +// Created by Brent Simmons on 11/26/17. +// Copyright © 2017 Ranchero Software. All rights reserved. +// + +import Foundation +import os +import Parser +import RSCore + +public final class HTMLMetadataDownloader: Sendable { + + public static let shared = HTMLMetadataDownloader() + + private static let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "HTMLMetadataDownloader") + private static let debugLoggingEnabled = false + + private let cache = HTMLMetadataCache() + private let attemptDatesLock = OSAllocatedUnfairLock(initialState: [String: Date]()) + private let urlsReturning4xxsLock = OSAllocatedUnfairLock(initialState: Set()) + + public func cachedMetadata(for url: String) -> HTMLMetadata? { + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader requested cached metadata for \(url)") + } + + guard let htmlMetadata = cache[url] else { + downloadMetadataIfNeeded(url) + return nil + } + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader returning cached metadata for \(url)") + } + return htmlMetadata + } +} + +private extension HTMLMetadataDownloader { + + func downloadMetadataIfNeeded(_ url: String) { + + if urlShouldBeSkippedDueToPrevious4xxResponse(url) { + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an earlier request returned a 4xx response.") + } + return + } + + // Limit how often a download should be attempted. + let shouldDownload = attemptDatesLock.withLock { attemptDates in + + let currentDate = Date() + + let hoursBetweenAttempts = 3 // arbitrary + if let attemptDate = attemptDates[url], attemptDate > currentDate.bySubtracting(hours: hoursBetweenAttempts) { + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an attempt was made less than an hour ago.") + } + return false + } + + attemptDates[url] = currentDate + return true + } + + if shouldDownload { + downloadMetadata(url) + } + } + + func downloadMetadata(_ url: String) { + + guard let actualURL = URL(string: url) else { + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because it couldn’t construct a URL.") + } + return + } + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader downloading for \(url)") + } + + Downloader.shared.download(actualURL) { data, response, error in + if let data, !data.isEmpty, let response, response.statusIsOK { + let urlToUse = response.url ?? actualURL + let parserData = ParserData(url: urlToUse.absoluteString, data: data) + let htmlMetadata = HTMLMetadataParser.metadata(with: parserData) + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader caching parsed metadata for \(url)") + } + self.cache[url] = htmlMetadata + return + } + + if let statusCode = response?.forcedStatusCode, (400...499).contains(statusCode) { + self.noteURLDidReturn4xx(url) + } + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader failed download for \(url)") + } + } + } + + func urlShouldBeSkippedDueToPrevious4xxResponse(_ url: String) -> Bool { + + urlsReturning4xxsLock.withLock { $0.contains(url) } + } + + func noteURLDidReturn4xx(_ url: String) { + + _ = urlsReturning4xxsLock.withLock { $0.insert(url) } + } +} diff --git a/RSWeb/Sources/RSWeb/HTTPResponse429.swift b/RSWeb/Sources/RSWeb/HTTPResponse429.swift new file mode 100644 index 000000000..e2bdd50b7 --- /dev/null +++ b/RSWeb/Sources/RSWeb/HTTPResponse429.swift @@ -0,0 +1,37 @@ +// +// File.swift +// RSWeb +// +// Created by Brent Simmons on 11/24/24. +// + +import Foundation + +// 429 Too Many Requests + +struct HTTPResponse429 { + + let url: URL + let host: String // lowercased + let dateCreated: Date + let retryAfter: TimeInterval + + var resumeDate: Date { + dateCreated + TimeInterval(retryAfter) + } + var canResume: Bool { + Date() >= resumeDate + } + + init?(url: URL, retryAfter: TimeInterval) { + + guard let host = url.host() else { + return nil + } + + self.url = url + self.host = host.lowercased() + self.retryAfter = retryAfter + self.dateCreated = Date() + } +} diff --git a/RSWeb/Sources/RSWeb/OneShotDownload.swift b/RSWeb/Sources/RSWeb/OneShotDownload.swift deleted file mode 100755 index 659dfc84b..000000000 --- a/RSWeb/Sources/RSWeb/OneShotDownload.swift +++ /dev/null @@ -1,192 +0,0 @@ -// -// OneShotDownload.swift -// RSWeb -// -// Created by Brent Simmons on 8/27/16. -// Copyright © 2016 Ranchero Software, LLC. All rights reserved. -// - -import Foundation - -// Main thread only. - -public typealias OneShotDownloadCallback = (Data?, URLResponse?, Error?) -> Swift.Void - -private final class OneShotDownloadManager { - - private let urlSession: URLSession - fileprivate static let shared = OneShotDownloadManager() - - public init() { - - let sessionConfiguration = URLSessionConfiguration.ephemeral - sessionConfiguration.requestCachePolicy = .reloadIgnoringLocalCacheData - sessionConfiguration.httpShouldSetCookies = false - sessionConfiguration.httpCookieAcceptPolicy = .never - sessionConfiguration.httpMaximumConnectionsPerHost = 2 - sessionConfiguration.httpCookieStorage = nil - sessionConfiguration.urlCache = nil - sessionConfiguration.timeoutIntervalForRequest = 30 - - if let userAgentHeaders = UserAgent.headers() { - sessionConfiguration.httpAdditionalHeaders = userAgentHeaders - } - - urlSession = URLSession(configuration: sessionConfiguration) - } - - deinit { - urlSession.invalidateAndCancel() - } - - public func download(_ url: URL, _ completion: @escaping OneShotDownloadCallback) { - let task = urlSession.dataTask(with: url) { (data, response, error) in - DispatchQueue.main.async() { - completion(data, response, error) - } - } - task.resume() - } - - public func download(_ urlRequest: URLRequest, _ completion: @escaping OneShotDownloadCallback) { - let task = urlSession.dataTask(with: urlRequest) { (data, response, error) in - DispatchQueue.main.async() { - completion(data, response, error) - } - } - task.resume() - } -} - -// Call one of these. It’s easier than referring to OneShotDownloadManager. -// callback is called on the main queue. - -public func download(_ url: URL, _ completion: @escaping OneShotDownloadCallback) { - precondition(Thread.isMainThread) - OneShotDownloadManager.shared.download(url, completion) -} - -public func download(_ urlRequest: URLRequest, _ completion: @escaping OneShotDownloadCallback) { - precondition(Thread.isMainThread) - OneShotDownloadManager.shared.download(urlRequest, completion) -} - -// MARK: - Downloading using a cache - -private struct WebCacheRecord { - - let url: URL - let dateDownloaded: Date - let data: Data - let response: URLResponse -} - -private final class WebCache { - - private var cache = [URL: WebCacheRecord]() - - func cleanup(_ cleanupInterval: TimeInterval) { - - let cutoffDate = Date(timeInterval: -cleanupInterval, since: Date()) - for key in cache.keys { - let cacheRecord = self[key]! - if shouldDelete(cacheRecord, cutoffDate) { - cache[key] = nil - } - } - } - - private func shouldDelete(_ cacheRecord: WebCacheRecord, _ cutoffDate: Date) -> Bool { - - return cacheRecord.dateDownloaded < cutoffDate - } - - subscript(_ url: URL) -> WebCacheRecord? { - get { - return cache[url] - } - set { - if let cacheRecord = newValue { - cache[url] = cacheRecord - } - else { - cache[url] = nil - } - } - } -} - -// URLSessionConfiguration has a cache policy. -// But we don’t know how it works, and the unimplemented parts spook us a bit. -// So we use a cache that works exactly as we want it to work. -// It also makes sure we don’t have multiple requests for the same URL at the same time. - -private struct CallbackRecord { - let url: URL - let completion: OneShotDownloadCallback -} - -private final class DownloadWithCacheManager { - - static let shared = DownloadWithCacheManager() - private var cache = WebCache() - private static let timeToLive: TimeInterval = 10 * 60 // 10 minutes - private static let cleanupInterval: TimeInterval = 5 * 60 // clean up the cache at most every 5 minutes - private var lastCleanupDate = Date() - private var pendingCallbacks = [CallbackRecord]() - private var urlsInProgress = Set() - - func download(_ url: URL, _ completion: @escaping OneShotDownloadCallback, forceRedownload: Bool = false) { - - if lastCleanupDate.timeIntervalSinceNow < -DownloadWithCacheManager.cleanupInterval { - lastCleanupDate = Date() - cache.cleanup(DownloadWithCacheManager.timeToLive) - } - - if !forceRedownload { - let cacheRecord: WebCacheRecord? = cache[url] - if let cacheRecord = cacheRecord { - completion(cacheRecord.data, cacheRecord.response, nil) - return - } - } - - let callbackRecord = CallbackRecord(url: url, completion: completion) - pendingCallbacks.append(callbackRecord) - if urlsInProgress.contains(url) { - return // The completion handler will get called later. - } - urlsInProgress.insert(url) - - OneShotDownloadManager.shared.download(url) { (data, response, error) in - - self.urlsInProgress.remove(url) - - if let data = data, let response = response, response.statusIsOK, error == nil { - let cacheRecord = WebCacheRecord(url: url, dateDownloaded: Date(), data: data, response: response) - self.cache[url] = cacheRecord - } - - var callbackCount = 0 - for callbackRecord in self.pendingCallbacks { - if url == callbackRecord.url { - callbackRecord.completion(data, response, error) - callbackCount += 1 - } - } - self.pendingCallbacks.removeAll(where: { (callbackRecord) -> Bool in - return callbackRecord.url == url - }) - } - } -} - -public func downloadUsingCache(_ url: URL, _ completion: @escaping OneShotDownloadCallback) { - precondition(Thread.isMainThread) - DownloadWithCacheManager.shared.download(url, completion) -} - -public func downloadAddingToCache(_ url: URL, _ completion: @escaping OneShotDownloadCallback) { - precondition(Thread.isMainThread) - DownloadWithCacheManager.shared.download(url, completion, forceRedownload: true) -} diff --git a/RSWeb/Sources/RSWeb/SpecialCases.swift b/RSWeb/Sources/RSWeb/SpecialCases.swift new file mode 100644 index 000000000..67c70f6ed --- /dev/null +++ b/RSWeb/Sources/RSWeb/SpecialCases.swift @@ -0,0 +1,108 @@ +// +// SpecialCases.swift +// RSWeb +// +// Created by Brent Simmons on 12/12/24. +// + +import Foundation +import os + +extension URL { + + private static let openRSSOrgURLCache = OSAllocatedUnfairLock(initialState: [URL: Bool]()) + + public var isOpenRSSOrgURL: Bool { + + Self.openRSSOrgURLCache.withLock { cache in + if let cachedResult = cache[self] { + return cachedResult + } + + let result: Bool + if let host = host(), host.contains("openrss.org") { + result = true + } + else { + result = false + } + + cache[self] = result + return result + } + } +} + +extension Set where Element == URL { + + func byRemovingOpenRSSOrgURLs() -> Set { + + filter { !$0.isOpenRSSOrgURL } + } + + func openRSSOrgURLs() -> Set { + + filter { $0.isOpenRSSOrgURL } + } + + func byRemovingAllButOneRandomOpenRSSOrgURL() -> Set { + + if self.isEmpty || self.count == 1 { + return self + } + + let openRSSOrgURLs = openRSSOrgURLs() + if openRSSOrgURLs.isEmpty || openRSSOrgURLs.count == 1 { + return self + } + + let randomIndex = Int.random(in: 0.. String? { + + guard let s = Bundle.main.object(forInfoDictionaryKey: key) as? String else { + assertionFailure("Expected to get \(key) from infoDictionary.") + return nil + } + return s + } +} diff --git a/SyncDatabase/Package.swift b/SyncDatabase/Package.swift index a5ee79444..4b30bc82d 100644 --- a/SyncDatabase/Package.swift +++ b/SyncDatabase/Package.swift @@ -14,7 +14,7 @@ let package = Package( dependencies: [ .package(path: "../RSCore"), .package(path: "../Articles"), - .package(path: "../RSDatabase.git"), + .package(path: "../RSDatabase"), ], targets: [ .target(