Move modules to Modules folder.

Modules/RSWeb/.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:">
+   </FileRef>
+</Workspace>

Modules/RSWeb/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Brent Simmons
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Modules/RSWeb/Package.swift

@@ -0,0 +1,31 @@
+// swift-tools-version:5.10
+
+import PackageDescription
+
+let package = Package(
+	name: "RSWeb",
+	platforms: [.macOS(.v14), .iOS(.v17)],
+	products: [
+		.library(
+			name: "RSWeb",
+			type: .dynamic,
+			targets: ["RSWeb"]),
+	],
+	dependencies: [
+		.package(path: "../Parser"),
+		.package(path: "../RSCore"),
+	],
+	targets: [
+		.target(
+			name: "RSWeb",
+			dependencies: [
+				"Parser",
+				"RSCore"
+			],
+			swiftSettings: [.unsafeFlags(["-warnings-as-errors"])]
+		),
+		.testTarget(
+			name: "RSWebTests",
+			dependencies: ["RSWeb"]),
+	]
+)

Modules/RSWeb/README.md

@@ -0,0 +1,19 @@
+# RSWeb
+
+RSWeb is utility code — all Swift — for downloading things from the web. It builds a Mac framework and an iOS framework.
+
+#### Easy way
+
+See `OneShotDownload` for a top-level `download` function that takes a URL and a callback. The callback takes `Data`, `URLResponse`, and `Error` parameters. It’s easy.
+
+#### Slightly less easy way
+
+See `DownloadSession` and `DownloadSessionDelegate` for when you’re doing a bunch of downloads and you need to track progress.
+
+#### Extras
+
+`HTTPConditionalGetInfo` helps with supporting conditional GET, for when you’re downloading things that may not have changed. See [HTTP Conditional Get for RSS Hackers](http://fishbowl.pastiche.org/2002/10/21/http_conditional_get_for_rss_hackers/) for more about conditional GET. This is especially critical when polling for changes, such as with an RSS reader.
+
+`MimeType` could use expansion, but is useful for some cases right now.
+
+`MacWebBrowser` makes it easy to open a URL in the default browser. You can specify whether or not to open in background.

Modules/RSWeb/Sources/RSWeb/CacheControlInfo.swift

@@ -0,0 +1,66 @@
+//
+//  CacheControl.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 11/30/24.
+//
+
+import Foundation
+
+/// Basic Cache-Control handling — just the part we need,
+/// which is to know when we got the response (dateCreated)
+/// and when we can ask again (canResume).
+public struct CacheControlInfo: Codable, Equatable {
+
+	let dateCreated: Date
+	let maxAge: TimeInterval
+
+	var resumeDate: Date {
+		dateCreated + maxAge
+	}
+	public var canResume: Bool {
+		Date() >= resumeDate
+	}
+	
+	public init?(urlResponse: HTTPURLResponse) {
+		guard let cacheControlValue = urlResponse.valueForHTTPHeaderField(HTTPResponseHeader.cacheControl) else {
+			return nil
+		}
+		self.init(value: cacheControlValue)
+	}
+
+	/// Returns nil if there’s no max-age or it’s < 1.
+	public init?(value: String) {
+
+		guard let maxAge = Self.parseMaxAge(value) else {
+			return nil
+		}
+		
+		let d = Date()
+		self.dateCreated = d
+		self.maxAge = maxAge
+	}
+}
+
+private extension CacheControlInfo {
+
+	static let maxAgePrefix = "max-age="
+	static let maxAgePrefixCount = maxAgePrefix.count
+
+	static func parseMaxAge(_ s: String) -> TimeInterval? {
+
+		let components = s.components(separatedBy: ",")
+		let trimmedComponents = components.map { $0.trimmingCharacters(in: .whitespaces) }
+
+		for component in trimmedComponents {
+			if component.hasPrefix(Self.maxAgePrefix) {
+				let maxAgeStringValue = component.dropFirst(maxAgePrefixCount)
+				if let timeInterval = TimeInterval(maxAgeStringValue), timeInterval > 0 {
+					return timeInterval
+				}
+			}
+		}
+
+		return nil
+	}
+}

Modules/RSWeb/Sources/RSWeb/Dictionary+RSWeb.swift

@@ -0,0 +1,26 @@
+//
+//  Dictionary+RSWeb.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 1/13/18.
+//  Copyright © 2018 Ranchero Software. All rights reserved.
+//
+
+import Foundation
+
+public extension Dictionary where Key == String, Value == String  {
+
+	/// Translates a dictionary into a string like `foo=bar&param2=some%20thing`.
+	var urlQueryString: String? {
+
+		var components = URLComponents()
+
+		components.queryItems = self.reduce(into: [URLQueryItem]()) {
+			$0.append(URLQueryItem(name: $1.key, value: $1.value))
+		}
+
+		let s = components.percentEncodedQuery
+
+		return s == nil || s!.isEmpty ? nil : s
+	}
+}

Modules/RSWeb/Sources/RSWeb/DownloadProgress.swift

@@ -0,0 +1,101 @@
+//
+//  DownloadProgress.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 9/17/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+// Main thread only.
+
+public extension Notification.Name {
+	
+	static let DownloadProgressDidChange = Notification.Name(rawValue: "DownloadProgressDidChange")
+}
+
+public final class DownloadProgress {
+	
+	public var numberOfTasks = 0 {
+		didSet {
+			if numberOfTasks == 0 && numberRemaining != 0 {
+				numberRemaining = 0
+			}
+			if numberOfTasks != oldValue {
+				postDidChangeNotification()
+			}
+		}
+	}
+	
+	public var numberRemaining = 0 {
+		didSet {
+			if numberRemaining != oldValue {
+				postDidChangeNotification()
+			}
+		}
+	}
+
+	public var numberCompleted: Int {
+		var n = numberOfTasks - numberRemaining
+		if n < 0 {
+			n = 0
+		}
+		if n > numberOfTasks {
+			n = numberOfTasks
+		}
+		return n
+	}
+	
+	public var isComplete: Bool {
+		assert(Thread.isMainThread)
+		return numberRemaining < 1
+	}
+	
+	public init(numberOfTasks: Int) {
+		assert(Thread.isMainThread)
+		self.numberOfTasks = numberOfTasks
+	}
+	
+	public func addToNumberOfTasks(_ n: Int) {
+		assert(Thread.isMainThread)
+		numberOfTasks = numberOfTasks + n
+	}
+	
+	public func addToNumberOfTasksAndRemaining(_ n: Int) {
+		assert(Thread.isMainThread)
+		numberOfTasks = numberOfTasks + n
+		numberRemaining = numberRemaining + n
+	}
+
+	public func completeTask() {
+		assert(Thread.isMainThread)
+		if numberRemaining > 0 {
+			numberRemaining = numberRemaining - 1
+		}
+	}
+	
+	public func completeTasks(_ tasks: Int) {
+		assert(Thread.isMainThread)
+		if numberRemaining >= tasks {
+			numberRemaining = numberRemaining - tasks
+		}
+	}
+	
+	public func reset() {
+		assert(Thread.isMainThread)
+		numberRemaining = 0
+		numberOfTasks = 0
+	}
+}
+
+// MARK: - Private
+
+private extension DownloadProgress {
+	
+	func postDidChangeNotification() {
+		DispatchQueue.main.async {
+			NotificationCenter.default.post(name: .DownloadProgressDidChange, object: self)
+		}
+	}
+}

Modules/RSWeb/Sources/RSWeb/DownloadSession.swift

@@ -0,0 +1,466 @@
+//
+//  DownloadSession.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 3/12/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+import os
+
+// Create a DownloadSessionDelegate, then create a DownloadSession.
+// To download things: call download with a set of URLs. DownloadSession will call the various delegate methods.
+
+public protocol DownloadSessionDelegate {
+
+	func downloadSession(_ downloadSession: DownloadSession, conditionalGetInfoFor: URL) -> HTTPConditionalGetInfo?
+	func downloadSession(_ downloadSession: DownloadSession, downloadDidComplete: URL, response: URLResponse?, data: Data, error: NSError?)
+	func downloadSession(_ downloadSession: DownloadSession, shouldContinueAfterReceivingData: Data, url: URL) -> Bool
+	func downloadSessionDidComplete(_ downloadSession: DownloadSession)
+}
+
+@objc public final class DownloadSession: NSObject {
+
+	public let downloadProgress = DownloadProgress(numberOfTasks: 0)
+
+	private var urlSession: URLSession!
+	private var tasksInProgress = Set<URLSessionTask>()
+	private var tasksPending = Set<URLSessionTask>()
+	private var taskIdentifierToInfoDictionary = [Int: DownloadInfo]()
+	private var urlsInSession = Set<URL>()
+	private let delegate: DownloadSessionDelegate
+	private var redirectCache = [URL: URL]()
+	private var queue = [URL]()
+
+	// 429 Too Many Requests responses
+	private var retryAfterMessages = [String: HTTPResponse429]()
+
+	/// URLs with 400-499 responses (except for 429).
+	/// These URLs are skipped for the rest of the session.
+	private var urlsWith400s = Set<URL>()
+
+
+	public init(delegate: DownloadSessionDelegate) {
+
+		self.delegate = delegate
+
+		super.init()
+		
+		let sessionConfiguration = URLSessionConfiguration.ephemeral
+		sessionConfiguration.requestCachePolicy = .reloadIgnoringLocalCacheData
+		sessionConfiguration.timeoutIntervalForRequest = 15.0
+		sessionConfiguration.httpShouldSetCookies = false
+		sessionConfiguration.httpCookieAcceptPolicy = .never
+		sessionConfiguration.httpMaximumConnectionsPerHost = 1
+		sessionConfiguration.httpCookieStorage = nil
+		sessionConfiguration.urlCache = nil
+
+		if let userAgentHeaders = UserAgent.headers() {
+			sessionConfiguration.httpAdditionalHeaders = userAgentHeaders
+		}
+
+		urlSession = URLSession(configuration: sessionConfiguration, delegate: self, delegateQueue: OperationQueue.main)		
+	}
+	
+	deinit {
+		urlSession.invalidateAndCancel()
+	}
+
+	// MARK: - API
+
+	public func cancelAll() {
+		urlSession.getTasksWithCompletionHandler { dataTasks, uploadTasks, downloadTasks in
+			for task in dataTasks {
+				task.cancel()
+			}
+			for task in uploadTasks {
+				task.cancel()
+			}
+			for task in downloadTasks {
+				task.cancel()
+			}
+		}
+	}
+
+	public func download(_ urls: Set<URL>) {
+
+		let filteredURLs = Self.filteredURLs(urls)
+
+		for url in filteredURLs {
+			addDataTask(url)
+		}
+
+		urlsInSession = filteredURLs
+		updateDownloadProgress()
+	}
+}
+
+// MARK: - URLSessionTaskDelegate
+
+extension DownloadSession: URLSessionTaskDelegate {
+
+	public func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
+
+		defer {
+			removeTask(task)
+		}
+
+		guard let info = infoForTask(task) else {
+			return
+		}
+
+		delegate.downloadSession(self, downloadDidComplete: info.url, response: info.urlResponse, data: info.data as Data, error: error as NSError?)
+	}
+
+	private static let redirectStatusCodes = Set([HTTPResponseCode.redirectPermanent, HTTPResponseCode.redirectTemporary, HTTPResponseCode.redirectVeryTemporary, HTTPResponseCode.redirectPermanentPreservingMethod])
+
+	public func urlSession(_ session: URLSession, task: URLSessionTask, willPerformHTTPRedirection response: HTTPURLResponse, newRequest request: URLRequest, completionHandler: @escaping (URLRequest?) -> Void) {
+
+		if Self.redirectStatusCodes.contains(response.statusCode) {
+			if let oldURL = task.originalRequest?.url, let newURL = request.url {
+				cacheRedirect(oldURL, newURL)
+			}
+		}
+
+		var modifiedRequest = request
+
+		if let url = request.url, url.isOpenRSSOrgURL {
+			modifiedRequest.setValue(UserAgent.openRSSOrgUserAgent, forHTTPHeaderField: HTTPRequestHeader.userAgent)
+		}
+
+		completionHandler(modifiedRequest)
+	}
+}
+
+// MARK: - URLSessionDataDelegate
+
+extension DownloadSession: URLSessionDataDelegate {
+
+	public func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive response: URLResponse, completionHandler: @escaping (URLSession.ResponseDisposition) -> Void) {
+
+		defer {
+			updateDownloadProgress()
+		}
+
+		tasksInProgress.insert(dataTask)
+		tasksPending.remove(dataTask)
+
+		let taskInfo = infoForTask(dataTask)
+		if let taskInfo {
+			taskInfo.urlResponse = response
+		}
+
+		if !response.statusIsOK {
+
+			completionHandler(.cancel)
+			removeTask(dataTask)
+
+			let statusCode = response.forcedStatusCode
+
+			if statusCode == HTTPResponseCode.tooManyRequests {
+				handle429Response(dataTask, response)
+			} else if (400...499).contains(statusCode), let url = response.url {
+				urlsWith400s.insert(url)
+			}
+
+			return
+		}
+
+		addDataTaskFromQueueIfNecessary()
+		completionHandler(.allow)
+	}
+	
+	public func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
+
+		guard let info = infoForTask(dataTask) else {
+			return
+		}
+		info.addData(data)
+
+		if !delegate.downloadSession(self, shouldContinueAfterReceivingData: info.data as Data, url: info.url) {
+			dataTask.cancel()
+			removeTask(dataTask)
+		}
+	}
+}
+
+// MARK: - Private
+
+private extension DownloadSession {
+
+	func addDataTask(_ url: URL) {
+
+		guard tasksPending.count < 500 else {
+			queue.insert(url, at: 0)
+			return
+		}
+
+		// If received permanent redirect earlier, use that URL.
+		let urlToUse = cachedRedirect(for: url) ?? url
+
+		if requestShouldBeDroppedDueToActive429(urlToUse) {
+			os_log(.debug, "Dropping request for previous 429: \(urlToUse)")
+			return
+		}
+		if requestShouldBeDroppedDueToPrevious400(urlToUse) {
+			os_log(.debug, "Dropping request for previous 400-499: \(urlToUse)")
+			return
+		}
+
+		let urlRequest: URLRequest = {
+			var request = URLRequest(url: urlToUse)
+			if let conditionalGetInfo = delegate.downloadSession(self, conditionalGetInfoFor: url) {
+				conditionalGetInfo.addRequestHeadersToURLRequest(&request)
+			}
+			if url.isOpenRSSOrgURL {
+				request.setValue(UserAgent.openRSSOrgUserAgent, forHTTPHeaderField: HTTPRequestHeader.userAgent)
+			}
+			return request
+		}()
+
+		let task = urlSession.dataTask(with: urlRequest)
+
+		let info = DownloadInfo(url)
+		taskIdentifierToInfoDictionary[task.taskIdentifier] = info
+
+		tasksPending.insert(task)
+		task.resume()
+	}
+
+	func addDataTaskFromQueueIfNecessary() {
+		guard tasksPending.count < 500, let url = queue.popLast() else { return }
+		addDataTask(url)
+	}
+
+	func infoForTask(_ task: URLSessionTask) -> DownloadInfo? {
+		return taskIdentifierToInfoDictionary[task.taskIdentifier]
+	}
+
+	func removeTask(_ task: URLSessionTask) {
+		tasksInProgress.remove(task)
+		tasksPending.remove(task)
+		taskIdentifierToInfoDictionary[task.taskIdentifier] = nil
+
+		addDataTaskFromQueueIfNecessary()
+
+		updateDownloadProgress()
+	}
+
+	func urlStringIsBlackListedRedirect(_ urlString: String) -> Bool {
+
+		// Hotels and similar often do permanent redirects. We can catch some of those.
+
+		let s = urlString.lowercased()
+		let badStrings = ["solutionip", "lodgenet", "monzoon", "landingpage", "btopenzone", "register", "login", "authentic"]
+
+		for oneBadString in badStrings {
+			if s.contains(oneBadString) {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	func cacheRedirect(_ oldURL: URL, _ newURL: URL) {
+		if urlStringIsBlackListedRedirect(newURL.absoluteString) {
+			return
+		}
+		redirectCache[oldURL] = newURL
+	}
+
+	func cachedRedirect(for url: URL) -> URL? {
+
+		// Follow chains of redirects, but avoid loops.
+
+		var urls = Set<URL>()
+		urls.insert(url)
+
+		var currentURL = url
+
+		while(true) {
+
+			if let oneRedirectURL = redirectCache[currentURL] {
+
+				if urls.contains(oneRedirectURL) {
+					// Cycle. Bail.
+					return nil
+				}
+				urls.insert(oneRedirectURL)
+				currentURL = oneRedirectURL
+			}
+
+			else {
+				break
+			}
+		}
+
+		if currentURL == url {
+			return nil
+		}
+		return currentURL
+	}
+
+	// MARK: - Download Progress
+
+	func updateDownloadProgress() {
+
+		downloadProgress.numberOfTasks = urlsInSession.count
+
+		let numberRemaining = tasksPending.count + tasksInProgress.count + queue.count
+		downloadProgress.numberRemaining = min(numberRemaining, downloadProgress.numberOfTasks)
+
+		// Complete?
+		if downloadProgress.numberOfTasks > 0 && downloadProgress.numberRemaining < 1 {
+			delegate.downloadSessionDidComplete(self)
+			urlsInSession.removeAll()
+		}
+	}
+
+	// MARK: - 429 Too Many Requests
+
+	func handle429Response(_ dataTask: URLSessionDataTask, _ response: URLResponse) {
+
+		guard let message = createHTTPResponse429(dataTask, response) else {
+			return
+		}
+
+		retryAfterMessages[message.host] = message
+		cancelAndRemoveTasksWithHost(message.host)
+	}
+
+	func createHTTPResponse429(_ dataTask: URLSessionDataTask, _ response: URLResponse) -> HTTPResponse429? {
+
+		guard let url = dataTask.currentRequest?.url ?? dataTask.originalRequest?.url else {
+			return nil
+		}
+		guard let httpResponse = response as? HTTPURLResponse else {
+			return nil
+		}
+		guard let retryAfterValue = httpResponse.value(forHTTPHeaderField: HTTPResponseHeader.retryAfter) else {
+			return nil
+		}
+		guard let retryAfter = TimeInterval(retryAfterValue), retryAfter > 0 else {
+			return nil
+		}
+
+		return HTTPResponse429(url: url, retryAfter: retryAfter)
+	}
+
+	func cancelAndRemoveTasksWithHost(_ host: String) {
+
+		cancelAndRemoveTasksWithHost(host, in: tasksInProgress)
+		cancelAndRemoveTasksWithHost(host, in: tasksPending)
+	}
+
+	func cancelAndRemoveTasksWithHost(_ host: String, in tasks: Set<URLSessionTask>) {
+
+		let lowercaseHost = host.lowercased()
+
+		let tasksToRemove = tasks.filter { task in
+			if let taskHost = task.lowercaseHost, taskHost.contains(lowercaseHost) {
+				return false
+			}
+			return true
+		}
+
+		for task in tasksToRemove {
+			task.cancel()
+		}
+		for task in tasksToRemove {
+			removeTask(task)
+		}
+	}
+
+	func requestShouldBeDroppedDueToActive429(_ url: URL) -> Bool {
+
+		guard let host = url.host() else {
+			return false
+		}
+		guard let retryAfterMessage = retryAfterMessages[host] else {
+			return false
+		}
+
+		if retryAfterMessage.resumeDate < Date() {
+			retryAfterMessages[host] = nil
+			return false
+		}
+
+		return true
+	}
+
+	// MARK: - 400-499 responses
+
+	func requestShouldBeDroppedDueToPrevious400(_ url: URL) -> Bool {
+
+		if urlsWith400s.contains(url) {
+			return true
+		}
+		if let redirectedURL = cachedRedirect(for: url), urlsWith400s.contains(redirectedURL) {
+			return true
+		}
+
+		return false
+	}
+
+	// MARK: - Filtering URLs
+
+	static private let lastOpenRSSOrgFeedRefreshKey = "lastOpenRSSOrgFeedRefresh"
+	static private var lastOpenRSSOrgFeedRefresh: Date {
+		get {
+			UserDefaults.standard.value(forKey: lastOpenRSSOrgFeedRefreshKey) as? Date ?? Date.distantPast
+		}
+		set {
+			UserDefaults.standard.setValue(newValue, forKey: lastOpenRSSOrgFeedRefreshKey)
+		}
+	}
+
+	static private var canDownloadFromOpenRSSOrg: Bool {
+		let okayToDownloadDate = lastOpenRSSOrgFeedRefresh + TimeInterval(60 * 60 * 10) // 10 minutes (arbitrary)
+		return Date() > okayToDownloadDate
+	}
+
+	static func filteredURLs(_ urls: Set<URL>) -> Set<URL> {
+
+		// Possibly remove some openrss.org URLs.
+		// Can be extended later if necessary.
+
+		if canDownloadFromOpenRSSOrg {
+			// Allow only one feed from openrss.org per refresh session
+			lastOpenRSSOrgFeedRefresh = Date()
+			return urls.byRemovingAllButOneRandomOpenRSSOrgURL()
+		}
+
+		return urls.byRemovingOpenRSSOrgURLs()
+	}
+}
+
+extension URLSessionTask {
+
+	var lowercaseHost: String? {
+		guard let request = currentRequest ?? originalRequest else {
+			return nil
+		}
+		return request.url?.host()?.lowercased()
+	}
+}
+
+// MARK: - DownloadInfo
+
+private final class DownloadInfo {
+	
+	let url: URL
+	let data = NSMutableData()
+	var urlResponse: URLResponse?
+
+	init(_ url: URL) {
+
+		self.url = url
+	}
+	
+	func addData(_ d: Data) {
+		
+		data.append(d)
+	}
+}

Modules/RSWeb/Sources/RSWeb/Downloader.swift

@@ -0,0 +1,56 @@
+//
+//  Downloader.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 8/27/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public typealias DownloadCallback = (Data?, URLResponse?, Error?) -> Swift.Void
+
+/// Simple downloader, for a one-shot download like an image
+/// or a web page. For a download-feeds session, see DownloadSession.
+public final class Downloader {
+
+	public static let shared = Downloader()
+	private let urlSession: URLSession
+
+	private init() {
+
+		let sessionConfiguration = URLSessionConfiguration.ephemeral
+		sessionConfiguration.requestCachePolicy = .reloadIgnoringLocalCacheData
+		sessionConfiguration.httpShouldSetCookies = false
+		sessionConfiguration.httpCookieAcceptPolicy = .never
+		sessionConfiguration.httpMaximumConnectionsPerHost = 1
+		sessionConfiguration.httpCookieStorage = nil
+		
+		if let userAgentHeaders = UserAgent.headers() {
+			sessionConfiguration.httpAdditionalHeaders = userAgentHeaders
+		}
+
+		urlSession = URLSession(configuration: sessionConfiguration)
+	}
+
+	deinit {
+		urlSession.invalidateAndCancel()
+	}
+
+	public func download(_ url: URL, _ completion: DownloadCallback? = nil) {
+		download(URLRequest(url: url), completion)
+	}
+
+	public func download(_ urlRequest: URLRequest, _ completion: DownloadCallback? = nil) {
+
+		var urlRequestToUse = urlRequest
+		urlRequestToUse.addSpecialCaseUserAgentIfNeeded()
+
+		let task = urlSession.dataTask(with: urlRequestToUse) { (data, response, error) in
+			DispatchQueue.main.async() {
+				completion?(data, response, error)
+			}
+		}
+		task.resume()
+	}
+}

Modules/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift

@@ -0,0 +1,47 @@
+//
+//  HTMLMetadataCache.swift
+//
+//
+//  Created by Brent Simmons on 10/13/24.
+//
+
+import Foundation
+import Parser
+import RSCore
+
+extension Notification.Name {
+	// Sent when HTMLMetadata is cached. Posted on any thread.
+	static let htmlMetadataAvailable = Notification.Name("htmlMetadataAvailable")
+}
+
+final class HTMLMetadataCache: Sendable {
+
+	static let shared = HTMLMetadataCache()
+	
+	// Sent along with .htmlMetadataAvailable notification
+	struct UserInfoKey {
+		static let htmlMetadata = "htmlMetadata"
+		static let url = "url" // String value
+	}
+
+	private struct HTMLMetadataCacheRecord: CacheRecord {
+		let metadata: HTMLMetadata
+		let dateCreated = Date()
+	}
+
+	private let cache = Cache<HTMLMetadataCacheRecord>(timeToLive: TimeInterval(hours: 21), timeBetweenCleanups: TimeInterval(hours: 10))
+
+	subscript(_ url: String) -> HTMLMetadata? {
+		get {
+			return cache[url]?.metadata
+		}
+		set {
+			guard let htmlMetadata = newValue else {
+				return
+			}
+			let cacheRecord = HTMLMetadataCacheRecord(metadata: htmlMetadata)
+			cache[url] = cacheRecord
+			NotificationCenter.default.post(name: .htmlMetadataAvailable, object: self, userInfo: [UserInfoKey.htmlMetadata: htmlMetadata, UserInfoKey.url: url])
+		}
+	}
+}

Modules/RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift

@@ -0,0 +1,120 @@
+//
+//  HTMLMetadataDownloader.swift
+//  NetNewsWire
+//
+//  Created by Brent Simmons on 11/26/17.
+//  Copyright © 2017 Ranchero Software. All rights reserved.
+//
+
+import Foundation
+import os
+import Parser
+import RSCore
+
+public final class HTMLMetadataDownloader: Sendable {
+
+	public static let shared = HTMLMetadataDownloader()
+
+	private static let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "HTMLMetadataDownloader")
+	private static let debugLoggingEnabled = false
+	
+	private let cache = HTMLMetadataCache()
+	private let attemptDatesLock = OSAllocatedUnfairLock(initialState: [String: Date]())
+	private let urlsReturning4xxsLock = OSAllocatedUnfairLock(initialState: Set<String>())
+
+	public func cachedMetadata(for url: String) -> HTMLMetadata? {
+
+		if Self.debugLoggingEnabled {
+			Self.logger.debug("HTMLMetadataDownloader requested cached metadata for \(url)")
+		}
+
+		guard let htmlMetadata = cache[url] else {
+			downloadMetadataIfNeeded(url)
+			return nil
+		}
+
+		if Self.debugLoggingEnabled {
+			Self.logger.debug("HTMLMetadataDownloader returning cached metadata for \(url)")
+		}
+		return htmlMetadata
+	}
+}
+
+private extension HTMLMetadataDownloader {
+
+	func downloadMetadataIfNeeded(_ url: String) {
+
+		if urlShouldBeSkippedDueToPrevious4xxResponse(url) {
+			if Self.debugLoggingEnabled {
+				Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an earlier request returned a 4xx response.")
+			}
+			return
+		}
+
+		// Limit how often a download should be attempted.
+		let shouldDownload = attemptDatesLock.withLock { attemptDates in
+
+			let currentDate = Date()
+
+			let hoursBetweenAttempts = 3 // arbitrary
+			if let attemptDate = attemptDates[url], attemptDate > currentDate.bySubtracting(hours: hoursBetweenAttempts) {
+				if Self.debugLoggingEnabled {
+					Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an attempt was made less than an hour ago.")
+				}
+				return false
+			}
+
+			attemptDates[url] = currentDate
+			return true
+		}
+
+		if shouldDownload {
+			downloadMetadata(url)
+		}
+	}
+
+	func downloadMetadata(_ url: String) {
+
+		guard let actualURL = URL(string: url) else {
+			if Self.debugLoggingEnabled {
+				Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because it couldn’t construct a URL.")
+			}
+			return
+		}
+
+		if Self.debugLoggingEnabled {
+			Self.logger.debug("HTMLMetadataDownloader downloading for \(url)")
+		}
+
+		Downloader.shared.download(actualURL) { data, response, error in
+			if let data, !data.isEmpty, let response, response.statusIsOK {
+				let urlToUse = response.url ?? actualURL
+				let parserData = ParserData(url: urlToUse.absoluteString, data: data)
+				let htmlMetadata = HTMLMetadataParser.metadata(with: parserData)
+				if Self.debugLoggingEnabled {
+					Self.logger.debug("HTMLMetadataDownloader caching parsed metadata for \(url)")
+				}
+				self.cache[url] = htmlMetadata
+				return
+			}
+
+			if let statusCode = response?.forcedStatusCode, (400...499).contains(statusCode) {
+				self.noteURLDidReturn4xx(url)
+			}
+
+			if Self.debugLoggingEnabled {
+				Self.logger.debug("HTMLMetadataDownloader failed download for \(url)")
+			}
+		}
+	}
+
+	func urlShouldBeSkippedDueToPrevious4xxResponse(_ url: String) -> Bool {
+
+		urlsReturning4xxsLock.withLock { $0.contains(url) }
+	}
+
+	func noteURLDidReturn4xx(_ url: String) {
+
+		_ = urlsReturning4xxsLock.withLock { $0.insert(url) }
+	}
+}

Modules/RSWeb/Sources/RSWeb/HTTPConditionalGetInfo.swift

@@ -0,0 +1,46 @@
+//
+//  HTTPConditionalGetInfo.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 4/11/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public struct HTTPConditionalGetInfo: Codable, Equatable {
+	
+	public let lastModified: String?
+	public let etag: String?
+	
+	public init?(lastModified: String?, etag: String?) {
+		if lastModified == nil && etag == nil {
+			return nil
+		}
+		self.lastModified = lastModified
+		self.etag = etag
+	}
+	
+	public init?(urlResponse: HTTPURLResponse) {
+		let lastModified = urlResponse.valueForHTTPHeaderField(HTTPResponseHeader.lastModified)
+		let etag = urlResponse.valueForHTTPHeaderField(HTTPResponseHeader.etag)
+		self.init(lastModified: lastModified, etag: etag)
+	}
+
+	public init?(headers: [AnyHashable : Any]) {
+		let lastModified = headers[HTTPResponseHeader.lastModified] as? String
+		let etag = headers[HTTPResponseHeader.etag] as? String
+		self.init(lastModified: lastModified, etag: etag)
+	}
+	
+	public func addRequestHeadersToURLRequest(_ urlRequest: inout URLRequest) {
+		// Bug seen in the wild: lastModified with last possible 32-bit date, which is in 2038. Ignore those.
+		// TODO: drop this check in late 2037.
+		if let lastModified = lastModified, !lastModified.contains("2038") {
+			urlRequest.addValue(lastModified, forHTTPHeaderField: HTTPRequestHeader.ifModifiedSince)
+		}
+		if let etag = etag {
+			urlRequest.addValue(etag, forHTTPHeaderField: HTTPRequestHeader.ifNoneMatch)
+		}
+	}
+}

Modules/RSWeb/Sources/RSWeb/HTTPDateInfo.swift

@@ -0,0 +1,29 @@
+//
+//  HTTPDateInfo.swift
+//  RSWeb
+//
+//  Created by Maurice Parker on 5/12/19.
+//  Copyright © 2019 Ranchero Software. All rights reserved.
+//
+
+import Foundation
+
+public struct HTTPDateInfo: Codable, Equatable {
+	
+	private static let formatter: DateFormatter = {
+		let dateFormatter = DateFormatter()
+		dateFormatter.dateFormat = "EEEE, dd LLL yyyy HH:mm:ss zzz"
+		return dateFormatter
+	}()
+	
+	public let date: Date?
+	
+	public init?(urlResponse: HTTPURLResponse) {
+		if let headerDate = urlResponse.valueForHTTPHeaderField(HTTPResponseHeader.date) {
+			date = HTTPDateInfo.formatter.date(from: headerDate)
+		} else {
+			date = nil
+		}
+	}
+	
+}

Modules/RSWeb/Sources/RSWeb/HTTPLinkPagingInfo.swift

@@ -0,0 +1,41 @@
+//
+//  HTTPLinkPagingInfo.swift
+//  RSWeb
+//
+//  Created by Maurice Parker on 5/12/19.
+//  Copyright © 2019 Ranchero Software. All rights reserved.
+//
+
+import Foundation
+
+public struct HTTPLinkPagingInfo {
+	
+	public let nextPage: String?
+	public let lastPage: String?
+	
+	public init(nextPage: String?, lastPage: String?) {
+		self.nextPage = nextPage
+		self.lastPage = lastPage
+	}
+
+	public init(urlResponse: HTTPURLResponse) {
+		
+		guard let linkHeader = urlResponse.valueForHTTPHeaderField(HTTPResponseHeader.link) else {
+			self.init(nextPage: nil, lastPage: nil)
+			return
+		}
+
+		let links = linkHeader.components(separatedBy: ",")
+		
+		var dict: [String: String] = [:]
+		for link in links {
+			let components = link.components(separatedBy:"; ")
+			let page = components[0].trimmingCharacters(in: CharacterSet(charactersIn: " <>"))
+			dict[components[1]] = page
+		}
+		
+		self.init(nextPage: dict["rel=\"next\""], lastPage: dict["rel=\"last\""])
+		
+	}
+
+}

Modules/RSWeb/Sources/RSWeb/HTTPMethod.swift

@@ -0,0 +1,18 @@
+//
+//  HTTPMethod.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/26/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public struct HTTPMethod {
+
+	public static let get = "GET"
+	public static let post = "POST"
+	public static let put = "PUT"
+	public static let patch = "PATCH"
+	public static let delete = "DELETE"
+}

Modules/RSWeb/Sources/RSWeb/HTTPRequestHeader.swift

@@ -0,0 +1,21 @@
+//
+//  HTTPRequestHeader.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/26/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public struct HTTPRequestHeader {
+
+	public static let userAgent = "User-Agent"
+	public static let authorization = "Authorization"
+	public static let contentType = "Content-Type"
+	
+	// Conditional GET
+	
+	public static let ifModifiedSince = "If-Modified-Since"
+	public static let ifNoneMatch = "If-None-Match" //Etag
+}

Modules/RSWeb/Sources/RSWeb/HTTPResponse429.swift

@@ -0,0 +1,37 @@
+//
+//  File.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 11/24/24.
+//
+
+import Foundation
+
+// 429 Too Many Requests
+
+struct HTTPResponse429 {
+
+	let url: URL
+	let host: String // lowercased
+	let dateCreated: Date
+	let retryAfter: TimeInterval
+
+	var resumeDate: Date {
+		dateCreated + TimeInterval(retryAfter)
+	}
+	var canResume: Bool {
+		Date() >= resumeDate
+	}
+
+	init?(url: URL, retryAfter: TimeInterval) {
+
+		guard let host = url.host() else {
+			return nil
+		}
+
+		self.url = url
+		self.host = host.lowercased()
+		self.retryAfter = retryAfter
+		self.dateCreated = Date()
+	}
+}

Modules/RSWeb/Sources/RSWeb/HTTPResponseCode.swift

@@ -0,0 +1,73 @@
+//
+//  HTTPResponseCode.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/26/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public struct HTTPResponseCode {
+
+	// http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+	// Not an enum because the main interest is the actual values.
+	
+	public static let responseContinue = 100 //"continue" is a language keyword, hence the weird name
+	public static let switchingProtocols = 101
+	
+	public static let OK = 200
+	public static let created = 201
+	public static let accepted = 202
+	public static let nonAuthoritativeInformation = 203
+	public static let noContent = 204
+	public static let resetContent = 205
+	public static let partialContent = 206
+	
+	public static let redirectMultipleChoices = 300
+	public static let redirectPermanent = 301
+	public static let redirectTemporary = 302
+	public static let redirectSeeOther = 303
+	public static let notModified = 304
+	public static let useProxy = 305
+	public static let unused = 306
+	public static let redirectVeryTemporary = 307
+	public static let redirectPermanentPreservingMethod = 308
+
+	public static let badRequest = 400
+	public static let unauthorized = 401
+	public static let paymentRequired = 402
+	public static let forbidden = 403
+	public static let notFound = 404
+	public static let methodNotAllowed = 405
+	public static let notAcceptable = 406
+	public static let proxyAuthenticationRequired = 407
+	public static let requestTimeout = 408
+	public static let conflict = 409
+	public static let gone = 410
+	public static let lengthRequired = 411
+	public static let preconditionFailed = 412
+	public static let entityTooLarge = 413
+	public static let URITooLong = 414
+	public static let unsupportedMediaType = 415
+	public static let requestedRangeNotSatisfiable = 416
+	public static let expectationFailed = 417
+	public static let imATeapot = 418
+	public static let misdirectedRequest = 421
+	public static let unprocessableContentWebDAV = 422
+	public static let lockedWebDAV = 423
+	public static let failedDependencyWebDAV = 424
+	public static let tooEarly = 425
+	public static let upgradeRequired = 426
+	public static let preconditionRequired = 428
+	public static let tooManyRequests = 429
+	public static let requestHeaderFieldsTooLarge = 431
+	public static let unavailableForLegalReasons = 451
+
+	public static let internalServerError = 500
+	public static let notImplemented = 501
+	public static let badGateway = 502
+	public static let serviceUnavailable = 503
+	public static let gatewayTimeout = 504
+	public static let HTTPVersionNotSupported = 505
+}

Modules/RSWeb/Sources/RSWeb/HTTPResponseHeader.swift

@@ -0,0 +1,28 @@
+//
+//  HTTPResponseHeader.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/26/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public struct HTTPResponseHeader {
+
+	public static let contentType = "Content-Type"
+	public static let location = "Location"
+	public static let link = "Links"
+	public static let date = "Date"
+
+	// Conditional GET. See:
+	// http://fishbowl.pastiche.org/2002/10/21/http_conditional_get_for_rss_hackers/
+	
+	public static let lastModified = "Last-Modified"
+	// Changed to the canonical case for lookups against a case sensitive dictionary
+	// https://developer.apple.com/documentation/foundation/httpurlresponse/1417930-allheaderfields
+	public static let etag = "Etag"
+
+	public static let cacheControl = "Cache-Control"
+	public static let retryAfter = "Retry-After"
+}

Modules/RSWeb/Sources/RSWeb/MacWebBrowser.swift

@@ -0,0 +1,159 @@
+//
+//  MacWebBrowser.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/27/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+#if os(macOS)
+import AppKit
+import UniformTypeIdentifiers
+
+public class MacWebBrowser {
+
+	/// Opens a URL in the default browser.
+	@discardableResult public class func openURL(_ url: URL, inBackground: Bool = false) -> Bool {
+
+		// TODO: make this function async
+
+		guard let preparedURL = url.preparedForOpeningInBrowser() else {
+			return false
+		}
+
+		if (inBackground) {
+
+			let configuration = NSWorkspace.OpenConfiguration()
+			configuration.activates = false
+			NSWorkspace.shared.open(url, configuration: configuration, completionHandler: nil)
+
+			return true
+		}
+
+		return NSWorkspace.shared.open(preparedURL)
+	}
+
+	/// Returns an array of the browsers installed on the system, sorted by name.
+	///
+	/// "Browsers" are applications that can both handle `https` URLs and display HTML documents.
+	public class func sortedBrowsers() -> [MacWebBrowser] {
+
+		let httpsAppURLs = NSWorkspace.shared.urlsForApplications(toOpen: URL(string: "https://apple.com/")!)
+		let htmlAppURLs = NSWorkspace.shared.urlsForApplications(toOpen: UTType.html)
+		let browserAppURLs = Set(httpsAppURLs).intersection(Set(htmlAppURLs))
+
+		return browserAppURLs.compactMap { MacWebBrowser(url: $0) }.sorted {
+			if let leftName = $0.name, let rightName = $1.name {
+				return leftName < rightName
+			}
+
+			return false
+		}
+	}
+
+	/// The filesystem URL of the default web browser.
+	private class var defaultBrowserURL: URL? {
+		return NSWorkspace.shared.urlForApplication(toOpen: URL(string: "https:///")!)
+	}
+
+	/// The user's default web browser.
+	public class var `default`: MacWebBrowser {
+		return MacWebBrowser(url: defaultBrowserURL!)
+	}
+
+	/// The filesystem URL of the web browser.
+	public let url: URL
+
+	private lazy var _icon: NSImage? = {
+		if let values = try? url.resourceValues(forKeys: [.effectiveIconKey]) {
+			return values.effectiveIcon as? NSImage
+		}
+
+		return nil
+	}()
+
+	/// The application icon of the web browser.
+	public var icon: NSImage? {
+		return _icon
+	}
+
+	private lazy var _name: String? = {
+		if let values = try? url.resourceValues(forKeys: [.localizedNameKey]), var name = values.localizedName {
+			if let extensionRange = name.range(of: ".app", options: [.anchored, .backwards]) {
+				name = name.replacingCharacters(in: extensionRange, with: "")
+			}
+
+			return name
+		}
+
+		return nil
+	}()
+
+	/// The localized name of the web browser, with any `.app` extension removed.
+	public var name: String? {
+		return _name
+	}
+
+	private lazy var _bundleIdentifier: String? = {
+		return Bundle(url: url)?.bundleIdentifier
+	}()
+
+	/// The bundle identifier of the web browser.
+	public var bundleIdentifier: String? {
+		return _bundleIdentifier
+	}
+
+	/// Initializes a `MacWebBrowser` with a URL on disk.
+	/// - Parameter url: The filesystem URL of the browser.
+	public init(url: URL) {
+		self.url = url
+	}
+
+	/// Initializes a `MacWebBrowser` from a bundle identifier.
+	/// - Parameter bundleIdentifier: The bundle identifier of the browser.
+	public convenience init?(bundleIdentifier: String) {
+		guard let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundleIdentifier) else {
+			return nil
+		}
+
+		self.init(url: url)
+	}
+
+	/// Opens a URL in this browser.
+	/// - Parameters:
+	///   - url: The URL to open.
+	///   - inBackground: If `true`, attempt to load the URL without bringing the browser to the foreground.
+	@discardableResult public func openURL(_ url: URL, inBackground: Bool = false) -> Bool {
+		
+		// TODO: make this function async.
+		
+		guard let preparedURL = url.preparedForOpeningInBrowser() else {
+			return false
+		}
+		
+		Task { @MainActor in
+			
+			let configuration = NSWorkspace.OpenConfiguration()
+			if inBackground {
+				configuration.activates = false
+			}
+			
+			NSWorkspace.shared.open([preparedURL], withApplicationAt: self.url, configuration: configuration, completionHandler: nil)
+		}
+		
+		return true
+	}
+}
+
+extension MacWebBrowser: CustomDebugStringConvertible {
+
+	public var debugDescription: String {
+		if let name = name, let bundleIdentifier = bundleIdentifier{
+			return "MacWebBrowser: \(name) (\(bundleIdentifier))"
+		} else {
+			return "MacWebBrowser"
+		}
+	}
+}
+
+#endif

Modules/RSWeb/Sources/RSWeb/MimeType.swift

@@ -0,0 +1,55 @@
+//
+//  MimeType.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/26/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public struct MimeType {
+	
+	// This could certainly use expansion.
+	
+	public static let png = "image/png"
+	public static let jpeg = "image/jpeg"
+	public static let jpg = "image/jpg"
+	public static let gif = "image/gif"
+	public static let tiff = "image/tiff"
+}
+
+public extension String {
+	
+	func isMimeTypeImage() -> Bool {
+		
+		return self.isOfGeneralMimeType("image")
+	}
+	
+	func isMimeTypeAudio() -> Bool {
+		
+		return self.isOfGeneralMimeType("audio")
+	}
+	
+	func isMimeTypeVideo() -> Bool {
+		
+		return self.isOfGeneralMimeType("video")
+	}
+	
+	func isMimeTypeTimeBasedMedia() -> Bool {
+		
+		return self.isMimeTypeAudio() || self.isMimeTypeVideo()
+	}
+	
+	private func isOfGeneralMimeType(_ type: String) -> Bool {
+		
+		let lower = self.lowercased()
+		if lower.hasPrefix(type) {
+			return true
+		}
+		if lower.hasPrefix("x-\(type)") {
+			return true
+		}
+		return false
+	}
+}

Modules/RSWeb/Sources/RSWeb/Reachability.swift

@@ -0,0 +1,207 @@
+/*
+Copyright (c) 2014, Ashley Mills
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+*/
+
+import SystemConfiguration
+import Foundation
+
+public enum ReachabilityError: Error {
+	case failedToCreateWithHostname(String, Int32)
+	case unableToGetFlags(Int32)
+}
+
+public class Reachability {
+
+	/// Returns true if the internet is reachable.
+	///
+	/// Uses apple.com as an indicator for the internet at large,
+	/// since it’s surely one of the easiest-to-reach domain names.
+	/// Added 6 April 2024. Not part of Ashley Mills’s original code.
+	public static var internetIsReachable: Bool {
+
+		guard let reachability = try? Reachability(hostname: "apple.com") else {
+			return false
+		}
+		return reachability.connection != .unavailable
+	}
+
+    public typealias NetworkReachable = (Reachability) -> ()
+    public typealias NetworkUnreachable = (Reachability) -> ()
+
+    public enum Connection: CustomStringConvertible {
+        @available(*, deprecated, renamed: "unavailable")
+        case none
+        case unavailable, wifi, cellular
+        public var description: String {
+            switch self {
+            case .cellular: return "Cellular"
+            case .wifi: return "WiFi"
+            case .unavailable: return "No Connection"
+            case .none: return "unavailable"
+            }
+        }
+    }
+
+    /// Set to `false` to force Reachability.connection to .none when on cellular connection (default value `true`)
+    public var allowsCellularConnection: Bool
+
+	public var connection: Connection {
+        if flags == nil {
+            try? setReachabilityFlags()
+        }
+        
+        switch flags?.connection {
+        case .unavailable?, nil: return .unavailable
+        case .none?: return .unavailable
+        case .cellular?: return allowsCellularConnection ? .cellular : .unavailable
+        case .wifi?: return .wifi
+        }
+    }
+
+    fileprivate(set) var notifierRunning = false
+    fileprivate let reachabilityRef: SCNetworkReachability
+    fileprivate let reachabilitySerialQueue: DispatchQueue
+	fileprivate(set) var flags: SCNetworkReachabilityFlags?
+
+	required public init(reachabilityRef: SCNetworkReachability,
+						 queueQoS: DispatchQoS = .default,
+						 targetQueue: DispatchQueue? = nil) {
+		self.allowsCellularConnection = true
+		self.reachabilityRef = reachabilityRef
+		self.reachabilitySerialQueue = DispatchQueue(label: "uk.co.ashleymills.reachability", qos: queueQoS, target: targetQueue)
+	}
+
+	public convenience init(hostname: String,
+							queueQoS: DispatchQoS = .default,
+							targetQueue: DispatchQueue? = nil) throws {
+		guard let ref = SCNetworkReachabilityCreateWithName(nil, hostname) else {
+			throw ReachabilityError.failedToCreateWithHostname(hostname, SCError())
+		}
+		self.init(reachabilityRef: ref, queueQoS: queueQoS, targetQueue: targetQueue)
+	}
+}
+
+public extension Reachability {
+
+	var description: String {
+		return flags?.description ?? "unavailable flags"
+	}
+}
+
+fileprivate extension Reachability {
+
+	func setReachabilityFlags() throws {
+		try reachabilitySerialQueue.sync { [unowned self] in
+			var flags = SCNetworkReachabilityFlags()
+			if !SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags) {
+				throw ReachabilityError.unableToGetFlags(SCError())
+			}
+
+			self.flags = flags
+		}
+	}
+}
+
+extension SCNetworkReachabilityFlags {
+
+	typealias Connection = Reachability.Connection
+
+	var connection: Connection {
+		guard isReachableFlagSet else { return .unavailable }
+
+		// If we're reachable, but not on an iOS device (i.e. simulator), we must be on WiFi
+		#if targetEnvironment(simulator)
+		return .wifi
+		#else
+		var connection = Connection.unavailable
+
+		if !isConnectionRequiredFlagSet {
+			connection = .wifi
+		}
+
+		if isConnectionOnTrafficOrDemandFlagSet {
+			if !isInterventionRequiredFlagSet {
+				connection = .wifi
+			}
+		}
+
+		if isOnWWANFlagSet {
+			connection = .cellular
+		}
+
+		return connection
+		#endif
+	}
+
+	var isOnWWANFlagSet: Bool {
+		#if os(iOS)
+		return contains(.isWWAN)
+		#else
+		return false
+		#endif
+	}
+	var isReachableFlagSet: Bool {
+		return contains(.reachable)
+	}
+	var isConnectionRequiredFlagSet: Bool {
+		return contains(.connectionRequired)
+	}
+	var isInterventionRequiredFlagSet: Bool {
+		return contains(.interventionRequired)
+	}
+	var isConnectionOnTrafficFlagSet: Bool {
+		return contains(.connectionOnTraffic)
+	}
+	var isConnectionOnDemandFlagSet: Bool {
+		return contains(.connectionOnDemand)
+	}
+	var isConnectionOnTrafficOrDemandFlagSet: Bool {
+		return !intersection([.connectionOnTraffic, .connectionOnDemand]).isEmpty
+	}
+	var isTransientConnectionFlagSet: Bool {
+		return contains(.transientConnection)
+	}
+	var isLocalAddressFlagSet: Bool {
+		return contains(.isLocalAddress)
+	}
+	var isDirectFlagSet: Bool {
+		return contains(.isDirect)
+	}
+
+	var description: String {
+		let W = isOnWWANFlagSet ? "W" : "-"
+		let R = isReachableFlagSet ? "R" : "-"
+		let c = isConnectionRequiredFlagSet ? "c" : "-"
+		let t = isTransientConnectionFlagSet ? "t" : "-"
+		let i = isInterventionRequiredFlagSet ? "i" : "-"
+		let C = isConnectionOnTrafficFlagSet ? "C" : "-"
+		let D = isConnectionOnDemandFlagSet ? "D" : "-"
+		let l = isLocalAddressFlagSet ? "l" : "-"
+		let d = isDirectFlagSet ? "d" : "-"
+
+		return "\(W)\(R) \(c)\(t)\(i)\(C)\(D)\(l)\(d)"
+	}
+}

Modules/RSWeb/Sources/RSWeb/SpecialCases.swift

@@ -0,0 +1,108 @@
+//
+//  SpecialCases.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/12/24.
+//
+
+import Foundation
+import os
+
+extension URL {
+
+	private static let openRSSOrgURLCache = OSAllocatedUnfairLock(initialState: [URL: Bool]())
+
+	public var isOpenRSSOrgURL: Bool {
+
+		Self.openRSSOrgURLCache.withLock { cache in
+			if let cachedResult = cache[self] {
+				return cachedResult
+			}
+
+			let result: Bool
+			if let host = host(), host.contains("openrss.org") {
+				result = true
+			}
+			else {
+				result = false
+			}
+
+			cache[self] = result
+			return result
+		}
+	}
+}
+
+extension Set where Element == URL {
+
+	func byRemovingOpenRSSOrgURLs() -> Set<URL> {
+
+		filter { !$0.isOpenRSSOrgURL }
+	}
+
+	func openRSSOrgURLs() -> Set<URL> {
+
+		filter { $0.isOpenRSSOrgURL }
+	}
+
+	func byRemovingAllButOneRandomOpenRSSOrgURL() -> Set<URL> {
+
+		if self.isEmpty || self.count == 1 {
+			return self
+		}
+
+		let openRSSOrgURLs = openRSSOrgURLs()
+		if openRSSOrgURLs.isEmpty || openRSSOrgURLs.count == 1 {
+			return self
+		}
+
+		let randomIndex = Int.random(in: 0..<openRSSOrgURLs.count)
+		let singleOpenRSSOrgURLToRead = (Array(openRSSOrgURLs))[randomIndex]
+
+		var urls = byRemovingOpenRSSOrgURLs()
+		urls.insert(singleOpenRSSOrgURLToRead)
+
+		return urls
+	}
+}
+
+extension URLRequest {
+
+	mutating func addSpecialCaseUserAgentIfNeeded() {
+		
+		if let url, url.isOpenRSSOrgURL {
+			setValue(UserAgent.openRSSOrgUserAgent, forHTTPHeaderField: HTTPRequestHeader.userAgent)
+		}
+	}
+}
+
+extension UserAgent {
+
+	static let openRSSOrgUserAgent = {
+
+#if os(iOS)
+		let platform = "iOS"
+#else
+		let platform = "Mac"
+#endif
+		let version = stringFromInfoPlist("CFBundleShortVersionString") ?? "Unknown"
+		let build = stringFromInfoPlist("CFBundleVersion") ?? "Unknown"
+
+		let template = Bundle.main.object(forInfoDictionaryKey: "UserAgentExtended") as? String
+
+		var userAgent = template!.replacingOccurrences(of: "[platform]", with: platform)
+		userAgent = userAgent.replacingOccurrences(of: "[version]", with: version)
+		userAgent = userAgent.replacingOccurrences(of: "[build]", with: build)
+
+		return userAgent
+	}()
+
+	private static func stringFromInfoPlist(_ key: String) -> String? {
+
+		guard let s = Bundle.main.object(forInfoDictionaryKey: key) as? String else {
+			assertionFailure("Expected to get \(key) from infoDictionary.")
+			return nil
+		}
+		return s
+	}
+}

Modules/RSWeb/Sources/RSWeb/String+RSWeb.swift

@@ -0,0 +1,39 @@
+//
+//  String+RSWeb.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 1/13/18.
+//  Copyright © 2018 Ranchero Software. All rights reserved.
+//
+
+import Foundation
+
+public extension String {
+
+	/// Escapes special HTML characters.
+	///
+	/// Escaped characters are `&`, `<`, `>`, `"`, and `'`.
+	var escapedHTML: String {
+		var escaped = String()
+
+		for char in self {
+			switch char {
+				case "&":
+					escaped.append("&amp;")
+				case "<":
+					escaped.append("&lt;")
+				case ">":
+					escaped.append("&gt;")
+				case "\"":
+					escaped.append("&quot;")
+				case "'":
+					escaped.append("&apos;")
+				default:
+					escaped.append(char)
+			}
+		}
+
+		return escaped
+	}
+	
+}

Modules/RSWeb/Sources/RSWeb/URL+RSWeb.swift

@@ -0,0 +1,89 @@
+//
+//  NSURL+RSWeb.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/26/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+private struct URLConstants {
+	static let schemeHTTP = "http"
+	static let schemeHTTPS = "https"
+	static let prefixHTTP = "http://"
+	static let prefixHTTPS = "https://"
+}
+
+public extension URL {
+
+	func isHTTPSURL() -> Bool {
+		return self.scheme?.lowercased() == URLConstants.schemeHTTPS
+	}
+
+	func isHTTPURL() -> Bool {
+		return self.scheme?.lowercased() == URLConstants.schemeHTTP
+	}
+
+	func isHTTPOrHTTPSURL() -> Bool {
+		return self.isHTTPSURL() || self.isHTTPURL()
+	}
+
+	func absoluteStringWithHTTPOrHTTPSPrefixRemoved() -> String? {
+		// Case-inensitive. Turns http://example.com/foo into example.com/foo
+
+		if isHTTPSURL() {
+			return absoluteString.stringByRemovingCaseInsensitivePrefix(URLConstants.prefixHTTPS)
+		}
+		else if isHTTPURL() {
+			return absoluteString.stringByRemovingCaseInsensitivePrefix(URLConstants.prefixHTTP)
+		}
+
+		return nil
+	}
+
+	func appendingQueryItem(_ queryItem: URLQueryItem) -> URL? {
+		appendingQueryItems([queryItem])
+	}
+
+	func appendingQueryItems(_ queryItems: [URLQueryItem]) -> URL? {
+		guard var components = URLComponents(url: self, resolvingAgainstBaseURL: false) else {
+			return nil
+		}
+
+		var newQueryItems = components.queryItems ?? []
+		newQueryItems.append(contentsOf: queryItems)
+		components.queryItems = newQueryItems
+
+		return components.url
+	}
+
+	func preparedForOpeningInBrowser() -> URL? {
+		var urlString = absoluteString.replacingOccurrences(of: " ", with: "%20")
+		urlString = urlString.replacingOccurrences(of: "^", with: "%5E")
+		urlString = urlString.replacingOccurrences(of: "&", with: "&")
+		urlString = urlString.replacingOccurrences(of: "&", with: "&")
+
+		return URL(string: urlString)
+	}
+}
+
+private extension String {
+
+	func stringByRemovingCaseInsensitivePrefix(_ prefix: String) -> String {
+		// Returns self if it doesn’t have the given prefix.
+
+		let lowerPrefix = prefix.lowercased()
+		let lowerSelf = self.lowercased()
+
+		if (lowerSelf == lowerPrefix) {
+			return ""
+		}
+		if !lowerSelf.hasPrefix(lowerPrefix) {
+			return self
+		}
+
+		let index = self.index(self.startIndex, offsetBy: prefix.count)
+		return String(self[..<index])
+	}
+}

Modules/RSWeb/Sources/RSWeb/URLComponents+RSWeb.swift

@@ -0,0 +1,34 @@
+//
+//  URLComponents.swift
+//  
+//
+//  Created by Maurice Parker on 11/8/20.
+//
+
+import Foundation
+
+public extension URLComponents {
+	
+    	// `+` is a valid character in query component as per RFC 3986 (https://developer.apple.com/documentation/foundation/nsurlcomponents/1407752-queryitems)
+ 	// workaround:
+ 	// - http://www.openradar.me/24076063
+ 	// - https://stackoverflow.com/a/37314144
+	var enhancedPercentEncodedQuery: String? {
+		guard !(queryItems?.isEmpty ?? true) else {
+			return nil
+		}
+		
+		var allowedCharacters = CharacterSet.urlQueryAllowed
+		allowedCharacters.remove(charactersIn: "!*'();:@&=+$,/?%#[]")
+		
+		var queries = [String]()
+		for queryItem in queryItems! {
+			if let value = queryItem.value?.addingPercentEncoding(withAllowedCharacters: allowedCharacters)?.replacingOccurrences(of: "%20", with: "+") {
+				queries.append("\(queryItem.name)=\(value)")
+			}
+		}
+		
+		return queries.joined(separator: "&")
+	}
+	
+}

Modules/RSWeb/Sources/RSWeb/URLRequest+RSWeb.swift

@@ -0,0 +1,28 @@
+//
+//  NSMutableURLRequest+RSWeb.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 12/27/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public extension URLRequest {
+
+	@discardableResult mutating func addBasicAuthorization(username: String, password: String) -> Bool {
+		
+		// Do this *only* with https. And not even then if you can help it.
+		
+		let s = "\(username):\(password)"
+		guard let d = s.data(using: .utf8, allowLossyConversion: false) else {
+			return false
+		}
+		
+		let base64EncodedString = d.base64EncodedString()
+		let authorization = "Basic \(base64EncodedString)"
+		setValue(authorization, forHTTPHeaderField: HTTPRequestHeader.authorization)
+		
+		return true
+	}
+}

Modules/RSWeb/Sources/RSWeb/URLResponse+RSWeb.swift

@@ -0,0 +1,45 @@
+//
+//  URLResponse+RSWeb.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 8/14/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public extension URLResponse {
+	
+	var statusIsOK: Bool {
+		return forcedStatusCode >= 200 && forcedStatusCode <= 299
+	}
+	
+	var forcedStatusCode: Int {
+		
+		// Return actual statusCode or 0 if there isn’t one.
+		
+		if let response = self as? HTTPURLResponse {
+			return response.statusCode
+		}
+		return 0
+	}
+}
+
+public extension HTTPURLResponse {
+	
+	func valueForHTTPHeaderField(_ headerField: String) -> String? {
+		
+		// Case-insensitive. HTTP headers may not be in the case you expect.
+		
+		let lowerHeaderField = headerField.lowercased()
+		
+		for (key, value) in allHeaderFields {
+			
+			if lowerHeaderField == (key as? String)?.lowercased() {
+				return value as? String
+			}
+		}
+		
+		return nil
+	}
+}

Modules/RSWeb/Sources/RSWeb/UserAgent.swift

@@ -0,0 +1,26 @@
+//
+//  UserAgent.swift
+//  RSWeb
+//
+//  Created by Brent Simmons on 8/27/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import Foundation
+
+public struct UserAgent {
+	
+	public static func fromInfoPlist() -> String? {
+
+		return Bundle.main.object(forInfoDictionaryKey: "UserAgent") as? String
+	}
+
+	public static func headers() -> [AnyHashable: String]? {
+
+		guard let userAgent = fromInfoPlist() else {
+			return nil
+		}
+
+		return [HTTPRequestHeader.userAgent: userAgent]
+	}
+}

Modules/RSWeb/Sources/RSWeb/WebServices/Transport.swift

@@ -0,0 +1,241 @@
+//
+//  Transport.swift
+//  RSWeb
+//
+//  Created by Maurice Parker on 5/4/19.
+//  Copyright © 2019 Ranchero Software. All rights reserved.
+//
+// Inspired by: http://robnapier.net/a-mockery-of-protocols
+
+import Foundation
+
+public enum TransportError: LocalizedError {
+	case noData
+    case noURL
+	case suspended
+	case httpError(status: Int)
+	
+	public var errorDescription: String? {
+		switch self {
+		case .httpError(let status):
+			switch status {
+			case 400:
+				return NSLocalizedString("Bad Request", comment: "Bad Request")
+			case 401:
+				return NSLocalizedString("Unauthorized", comment: "Unauthorized")
+			case 402:
+				return NSLocalizedString("Payment Required", comment: "Payment Required")
+			case 403:
+				return NSLocalizedString("Forbidden", comment: "Forbidden")
+			case 404:
+				return NSLocalizedString("Not Found", comment: "Not Found")
+			case 405:
+				return NSLocalizedString("Method Not Allowed", comment: "Method Not Allowed")
+			case 406:
+				return NSLocalizedString("Not Acceptable", comment: "Not Acceptable")
+			case 407:
+				return NSLocalizedString("Proxy Authentication Required", comment: "Proxy Authentication Required")
+			case 408:
+				return NSLocalizedString("Request Timeout", comment: "Request Timeout")
+			case 409:
+				return NSLocalizedString("Conflict", comment: "Conflict")
+			case 410:
+				return NSLocalizedString("Gone", comment: "Gone")
+			case 411:
+				return NSLocalizedString("Length Required", comment: "Length Required")
+			case 412:
+				return NSLocalizedString("Precondition Failed", comment: "Precondition Failed")
+			case 413:
+				return NSLocalizedString("Payload Too Large", comment: "Payload Too Large")
+			case 414:
+				return NSLocalizedString("Request-URI Too Long", comment: "Request-URI Too Long")
+			case 415:
+				return NSLocalizedString("Unsupported Media Type", comment: "Unsupported Media Type")
+			case 416:
+				return NSLocalizedString("Requested Range Not Satisfiable", comment: "Requested Range Not Satisfiable")
+			case 417:
+				return NSLocalizedString("Expectation Failed", comment: "Expectation Failed")
+			case 418:
+				return NSLocalizedString("I'm a teapot", comment: "I'm a teapot")
+			case 421:
+				return NSLocalizedString("Misdirected Request", comment: "Misdirected Request")
+			case 422:
+				return NSLocalizedString("Unprocessable Entity", comment: "Unprocessable Entity")
+			case 423:
+				return NSLocalizedString("Locked", comment: "Locked")
+			case 424:
+				return NSLocalizedString("Failed Dependency", comment: "Failed Dependency")
+			case 426:
+				return NSLocalizedString("Upgrade Required", comment: "Upgrade Required")
+			case 428:
+				return NSLocalizedString("Precondition Required", comment: "Precondition Required")
+			case 429:
+				return NSLocalizedString("Too Many Requests", comment: "Too Many Requests")
+			case 431:
+				return NSLocalizedString("Request Header Fields Too Large", comment: "Request Header Fields Too Large")
+			case 444:
+				return NSLocalizedString("Connection Closed Without Response", comment: "Connection Closed Without Response")
+			case 451:
+				return NSLocalizedString("Unavailable For Legal Reasons", comment: "Unavailable For Legal Reasons")
+			case 499:
+				return NSLocalizedString("Client Closed Request", comment: "Client Closed Request")
+			case 500:
+				return NSLocalizedString("Internal Server Error", comment: "Internal Server Error")
+			case 501:
+				return NSLocalizedString("Not Implemented", comment: "Not Implemented")
+			case 502:
+				return NSLocalizedString("Bad Gateway", comment: "Bad Gateway")
+			case 503:
+				return NSLocalizedString("Service Unavailable", comment: "Service Unavailable")
+			case 504:
+				return NSLocalizedString("Gateway Timeout", comment: "Gateway Timeout")
+			case 505:
+				return NSLocalizedString("HTTP Version Not Supported", comment: "HTTP Version Not Supported")
+			case 506:
+				return NSLocalizedString("Variant Also Negotiates", comment: "Variant Also Negotiates")
+			case 507:
+				return NSLocalizedString("Insufficient Storage", comment: "Insufficient Storage")
+			case 508:
+				return NSLocalizedString("Loop Detected", comment: "Loop Detected")
+			case 510:
+				return NSLocalizedString("Not Extended", comment: "Not Extended")
+			case 511:
+				return NSLocalizedString("Network Authentication Required", comment: "Network Authentication Required")
+			case 599:
+				return NSLocalizedString("Network Connect Timeout Error", comment: "Network Connect Timeout Error")
+			default:
+				let msg = NSLocalizedString("HTTP Status: ", comment: "Unexpected error")
+				return "\(msg) \(status)"
+			}
+		default:
+			return NSLocalizedString("An unknown network error occurred.", comment: "Unknown error")
+		}
+	}
+	
+}
+
+public protocol Transport {
+	
+	/// Cancels all pending requests
+	func cancelAll()
+	
+	/// Sends URLRequest and returns the HTTP headers and the data payload.
+	func send(request: URLRequest, completion: @escaping (Result<(HTTPURLResponse, Data?), Error>) -> Void)
+	
+	/// Sends URLRequest that doesn't require any result information.
+	func send(request: URLRequest, method: String, completion: @escaping (Result<Void, Error>) -> Void)
+	
+	/// Sends URLRequest with a data payload and returns the HTTP headers and the data payload.
+	func send(request: URLRequest, method: String, payload: Data, completion: @escaping (Result<(HTTPURLResponse, Data?), Error>) -> Void)
+	
+}
+
+extension URLSession: Transport {
+	
+	public func cancelAll() {
+		getTasksWithCompletionHandler { dataTasks, uploadTasks, downloadTasks in
+			for dataTask in dataTasks {
+				dataTask.cancel()
+			}
+			for uploadTask in uploadTasks {
+				uploadTask.cancel()
+			}
+			for downloadTask in downloadTasks {
+				downloadTask.cancel()
+			}
+		}
+	}
+	
+	public func send(request: URLRequest, completion: @escaping (Result<(HTTPURLResponse, Data?), Error>) -> Void) {
+		let task = self.dataTask(with: request) { (data, response, error) in
+			DispatchQueue.main.async {
+				if let error = error {
+					return completion(.failure(error))
+				}
+
+				guard let response = response as? HTTPURLResponse, let data = data else {
+					return completion(.failure(TransportError.noData))
+				}
+
+				switch response.forcedStatusCode {
+				case 200...399:
+					completion(.success((response, data)))
+				default:
+					completion(.failure(TransportError.httpError(status: response.forcedStatusCode)))
+				}
+			}
+		}
+		task.resume()
+	}
+
+	public func send(request: URLRequest, method: String, completion: @escaping (Result<Void, Error>) -> Void) {
+		
+		var sendRequest = request
+		sendRequest.httpMethod = method
+		
+		let task = self.dataTask(with: sendRequest) { (data, response, error) in
+			DispatchQueue.main.async {
+				if let error = error {
+					return completion(.failure(error))
+				}
+
+				guard let response = response as? HTTPURLResponse else {
+					return completion(.failure(TransportError.noData))
+				}
+
+				switch response.forcedStatusCode {
+				case 200...399:
+					completion(.success(()))
+				default:
+					completion(.failure(TransportError.httpError(status: response.forcedStatusCode)))
+				}
+			}
+		}
+		task.resume()
+	}
+	
+	public func send(request: URLRequest, method: String, payload: Data, completion: @escaping (Result<(HTTPURLResponse, Data?), Error>) -> Void) {
+		
+		var sendRequest = request
+		sendRequest.httpMethod = method
+		
+		let task = self.uploadTask(with: sendRequest, from: payload) { (data, response, error) in
+			DispatchQueue.main.async {
+				if let error = error {
+					return completion(.failure(error))
+				}
+
+				guard let response = response as? HTTPURLResponse, let data = data else {
+					return completion(.failure(TransportError.noData))
+				}
+
+				switch response.forcedStatusCode {
+				case 200...399:
+					completion(.success((response, data)))
+				default:
+					completion(.failure(TransportError.httpError(status: response.forcedStatusCode)))
+				}
+				
+			}
+		}
+		task.resume()
+	}
+	
+	public static func webserviceTransport() -> Transport {
+	
+		let sessionConfiguration = URLSessionConfiguration.default
+		sessionConfiguration.requestCachePolicy = .reloadIgnoringLocalCacheData
+		sessionConfiguration.timeoutIntervalForRequest = 60.0
+		sessionConfiguration.httpShouldSetCookies = false
+		sessionConfiguration.httpCookieAcceptPolicy = .never
+		sessionConfiguration.httpMaximumConnectionsPerHost = 2
+		sessionConfiguration.httpCookieStorage = nil
+		sessionConfiguration.urlCache = nil
+		
+		if let userAgentHeaders = UserAgent.headers() {
+			sessionConfiguration.httpAdditionalHeaders = userAgentHeaders
+		}
+		
+		return URLSession(configuration: sessionConfiguration)
+	}
+}

Modules/RSWeb/Sources/RSWeb/WebServices/TransportJSON.swift

@@ -0,0 +1,153 @@
+//
+//  JSONTransport.swift
+//  RSWeb
+//
+//  Created by Maurice Parker on 5/6/19.
+//  Copyright © 2019 Ranchero Software. All rights reserved.
+//
+
+import Foundation
+
+extension Transport {
+	
+	/**
+	 Sends an HTTP get and returns JSON object(s)
+	 */
+    public func send<R: Decodable>(request: URLRequest, resultType: R.Type, dateDecoding: JSONDecoder.DateDecodingStrategy = .iso8601, keyDecoding: JSONDecoder.KeyDecodingStrategy = .useDefaultKeys, completion: @escaping (Result<(HTTPURLResponse, R?), Error>) -> Void) {
+		
+		send(request: request) { result in
+			DispatchQueue.main.async {
+
+				switch result {
+				case .success(let (response, data)):
+					if let data = data, !data.isEmpty {
+						// PBS 27 Sep. 2019: decode the JSON on a background thread.
+						// The profiler says that this is 45% of what’s happening on the main thread
+						// during an initial sync with Feedbin.
+						DispatchQueue.global(qos: .background).async {
+							let decoder = JSONDecoder()
+							decoder.dateDecodingStrategy = dateDecoding
+							decoder.keyDecodingStrategy = keyDecoding
+							do {
+								let decoded = try decoder.decode(R.self, from: data)
+								DispatchQueue.main.async {
+									completion(.success((response, decoded)))
+								}
+							}
+							catch {
+								DispatchQueue.main.async {
+									completion(.failure(error))
+								}
+							}
+						}
+					}
+					else {
+						completion(.success((response, nil)))
+					}
+
+				case .failure(let error):
+					completion(.failure(error))
+				}
+			}
+		}
+	}
+	
+	/**
+	Sends the specified HTTP method with a JSON payload.
+	*/
+	public func send<P: Encodable>(request: URLRequest, method: String, payload: P, completion: @escaping (Result<Void, Error>) -> Void) {
+		
+		var postRequest = request
+		postRequest.addValue("application/json; charset=utf-8", forHTTPHeaderField: HTTPRequestHeader.contentType)
+
+		let data: Data
+		do {
+			data = try JSONEncoder().encode(payload)
+		} catch {
+			completion(.failure(error))
+			return
+		}
+
+		send(request: postRequest, method: method, payload: data) { result in
+			DispatchQueue.main.async {
+				switch result {
+				case .success((_, _)):
+					completion(.success(()))
+				case .failure(let error):
+					completion(.failure(error))
+				}
+			}
+		}
+	}
+	
+	/**
+	Sends the specified HTTP method with a JSON payload and returns JSON object(s).
+	*/
+	public func send<P: Encodable, R: Decodable>(request: URLRequest, method: String, payload: P, resultType: R.Type, dateDecoding: JSONDecoder.DateDecodingStrategy = .iso8601, keyDecoding: JSONDecoder.KeyDecodingStrategy = .useDefaultKeys, completion: @escaping (Result<(HTTPURLResponse, R?), Error>) -> Void) {
+		
+		var postRequest = request
+		postRequest.addValue("application/json; charset=utf-8", forHTTPHeaderField: HTTPRequestHeader.contentType)
+
+		let data: Data
+		do {
+			data = try JSONEncoder().encode(payload)
+		} catch {
+			completion(.failure(error))
+			return
+		}
+		
+		send(request: postRequest, method: method, payload: data) { result in
+			DispatchQueue.main.async {
+
+				switch result {
+				case .success(let (response, data)):
+					do {
+						if let data = data, !data.isEmpty {
+							let decoder = JSONDecoder()
+							decoder.dateDecodingStrategy = dateDecoding
+                            decoder.keyDecodingStrategy = keyDecoding
+							let decoded = try decoder.decode(R.self, from: data)
+							completion(.success((response, decoded)))
+						} else {
+							completion(.success((response, nil)))
+						}
+					} catch {
+						completion(.failure(error))
+					}
+				case .failure(let error):
+					completion(.failure(error))
+				}
+			}
+		}
+	}
+
+    /**
+     Sends the specified HTTP method with a Raw payload and returns JSON object(s).
+     */
+	public func send<R: Decodable>(request: URLRequest, method: String, data: Data, resultType: R.Type, dateDecoding: JSONDecoder.DateDecodingStrategy = .iso8601, keyDecoding: JSONDecoder.KeyDecodingStrategy = .useDefaultKeys, completion: @escaping (Result<(HTTPURLResponse, R?), Error>) -> Void) {
+
+		send(request: request, method: method, payload: data) { result in
+			DispatchQueue.main.async {
+
+				switch result {
+				case .success(let (response, data)):
+					do {
+						if let data = data, !data.isEmpty {
+							let decoder = JSONDecoder()
+							decoder.dateDecodingStrategy = dateDecoding
+                            decoder.keyDecodingStrategy = keyDecoding
+							let decoded = try decoder.decode(R.self, from: data)
+							completion(.success((response, decoded)))
+						} else {
+							completion(.success((response, nil)))
+						}
+					} catch {
+						completion(.failure(error))
+					}
+				case .failure(let error):
+					completion(.failure(error))
+				}
+			}
+		}
+	}
+}

Modules/RSWeb/Tests/RSWebTests/DictionaryTests.swift

@@ -0,0 +1,45 @@
+//
+//  DictionaryTests.swift
+//  RSWebTests
+//
+//  Created by Brent Simmons on 1/13/18.
+//  Copyright © 2018 Ranchero Software. All rights reserved.
+//
+
+import XCTest
+
+class DictionaryTests: XCTestCase {
+
+	func testSimpleQueryString() {
+
+		let d = ["foo": "bar", "param1": "This is a value."]
+		let s = d.urlQueryString
+
+		XCTAssertTrue(s == "foo=bar&param1=This%20is%20a%20value." || s == "param1=This%20is%20a%20value.&foo=bar")
+	}
+
+	func testQueryStringWithAmpersand() {
+
+		let d = ["fo&o": "bar", "param1": "This is a&value."]
+		let s = d.urlQueryString
+
+		XCTAssertTrue(s == "fo%26o=bar&param1=This%20is%20a%26value." || s == "param1=This%20is%20a%26value.&fo%26o=bar")
+	}
+
+	func testQueryStringWithAccentedCharacters() {
+
+		let d = ["fée": "bør"]
+		let s = d.urlQueryString
+
+		XCTAssertTrue(s == "f%C3%A9e=b%C3%B8r")
+	}
+
+	func testQueryStringWithEmoji() {
+
+		let d = ["🌴e": "bar🎩🌴"]
+		let s = d.urlQueryString
+
+		XCTAssertTrue(s == "%F0%9F%8C%B4e=bar%F0%9F%8E%A9%F0%9F%8C%B4")
+	}
+
+}

Modules/RSWeb/Tests/RSWebTests/RSWebTests.swift

@@ -0,0 +1,42 @@
+//
+//  RSWebTests.swift
+//  RSWebTests
+//
+//  Created by Brent Simmons on 12/22/16.
+//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
+//
+
+import XCTest
+@testable import RSWeb
+
+class RSWebTests: XCTestCase {
+    
+    override func setUp() {
+        super.setUp()
+        // Put setup code here. This method is called before the invocation of each test method in the class.
+    }
+    
+    override func tearDown() {
+        // Put teardown code here. This method is called after the invocation of each test method in the class.
+        super.tearDown()
+    }
+    
+    func testExample() {
+        // This is an example of a functional test case.
+        // Use XCTAssert and related functions to verify your tests produce the correct results.
+    }
+    
+    func testPerformanceExample() {
+        // This is an example of a performance test case.
+        self.measure {
+            // Put the code you want to measure the time of here.
+        }
+    }
+
+	func testAllBrowsers() {
+		let browsers = MacWebBrowser.sortedBrowsers()
+
+		XCTAssertNotNil(browsers);
+	}
+    
+}

Modules/RSWeb/Tests/RSWebTests/StringTests.swift

@@ -0,0 +1,19 @@
+//
+//  StringTests.swift
+//  RSWebTests
+//
+//  Created by Brent Simmons on 1/13/18.
+//  Copyright © 2018 Ranchero Software. All rights reserved.
+//
+
+import XCTest
+
+class StringTests: XCTestCase {
+
+    func testHTMLEscaping() {
+
+		let s = #"<foo>"bar"&'baz'"#.escapedHTML
+		XCTAssertEqual(s, "&lt;foo&gt;&quot;bar&quot;&amp;&apos;baz&apos;")
+
+    }
+}