renun/NetNewsWire
1
0
Fork 0
mirror of https://github.com/Ranchero-Software/NetNewsWire synced 2025-08-12 06:26:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add and use new HTMLMetadataDownloader. Fix #4414.

Browse Source
This commit is contained in:
Brent Simmons
2024-12-14 10:15:51 -08:00
parent c38e5d6abe
commit b651131fbd
8 changed files with 221 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
Core/Sources/Core/FoundationExtras/Date+Extensions.swift Executable file
View File

@@ -0,0 +1,37 @@
//
// Date+Extensions.swift
// RSCore
//
// Created by Brent Simmons on 6/21/16.
// Copyright © 2016 Ranchero Software, LLC. All rights reserved.
//
import Foundation
public extension Date {
// Below are for rough use only — they don't use the calendar.
func bySubtracting(days: Int) -> Date {
return addingTimeInterval(0.0 - TimeInterval(days: days))
}
func bySubtracting(hours: Int) -> Date {
return addingTimeInterval(0.0 - TimeInterval(hours: hours))
}
func byAdding(days: Int) -> Date {
return addingTimeInterval(TimeInterval(days: days))
}
}
public extension TimeInterval {
init(days: Int) {
self.init(days * 24 * 60 * 60)
}
init(hours: Int) {
self.init(hours * 60 * 60)
}
}

20
NetNewsWire.xcodeproj/project.pbxproj
View File

@@ -296,12 +296,10 @@
51C4529A22650A0400C03939 /* ArticleTheme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849A97871ED9ECEF007D329B /* ArticleTheme.swift */; };
51C4529B22650A1000C03939 /* FaviconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 848F6AE41FC29CFA002D422E /* FaviconDownloader.swift */; };
51C4529C22650A1000C03939 /* SingleFaviconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845A29081FC74B8E007B49E3 /* SingleFaviconDownloader.swift */; };
51C4529D22650A1000C03939 /* FaviconURLFinder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */; };
51C4529E22650A1900C03939 /* ImageDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845213221FCA5B10003B6E93 /* ImageDownloader.swift */; };
51C4529F22650A1900C03939 /* AuthorAvatarDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84E850851FCB60CE0072EA88 /* AuthorAvatarDownloader.swift */; };
51C452A022650A1900C03939 /* WebFeedIconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611891FCB67AA0086A189 /* WebFeedIconDownloader.swift */; };
51C452A222650A1900C03939 /* RSHTMLMetadata+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611A11FCB769D0086A189 /* RSHTMLMetadata+Extension.swift */; };
51C452A322650A1E00C03939 /* HTMLMetadataDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */; };
51C452A422650A2D00C03939 /* ArticleUtilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849A97581ED9EB0D007D329B /* ArticleUtilities.swift */; };
51C452A522650A2D00C03939 /* SmallIconProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84411E701FE5FBFA004B527F /* SmallIconProvider.swift */; };
51C452A622650A3500C03939 /* Node-Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849A97971ED9EFAA007D329B /* Node-Extensions.swift */; };
@@ -400,7 +398,6 @@
841ABA6020145EC100980E11 /* BuiltinSmartFeedInspectorViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 841ABA5F20145EC100980E11 /* BuiltinSmartFeedInspectorViewController.swift */; };
84216D0322128B9D0049B9B9 /* DetailWebViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84216D0222128B9D0049B9B9 /* DetailWebViewController.swift */; };
8426118A1FCB67AA0086A189 /* WebFeedIconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611891FCB67AA0086A189 /* WebFeedIconDownloader.swift */; };
8426119E1FCB6ED40086A189 /* HTMLMetadataDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */; };
842611A21FCB769D0086A189 /* RSHTMLMetadata+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611A11FCB769D0086A189 /* RSHTMLMetadata+Extension.swift */; };
842E45CE1ED8C308000A8B52 /* AppNotifications.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842E45CD1ED8C308000A8B52 /* AppNotifications.swift */; };
842E45DD1ED8C54B000A8B52 /* Browser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842E45DC1ED8C54B000A8B52 /* Browser.swift */; };
@@ -537,7 +534,6 @@
84F9EAF3213660A100CF2DE4 /* testCurrentArticleIsNil.applescript in Sources */ = {isa = PBXBuildFile; fileRef = 84F9EAE0213660A100CF2DE4 /* testCurrentArticleIsNil.applescript */; };
84F9EAF4213660A100CF2DE4 /* testGenericScript.applescript in Sources */ = {isa = PBXBuildFile; fileRef = 84F9EAE1213660A100CF2DE4 /* testGenericScript.applescript */; };
84F9EAF5213660A100CF2DE4 /* establishMainWindowStartingState.applescript in Sources */ = {isa = PBXBuildFile; fileRef = 84F9EAE2213660A100CF2DE4 /* establishMainWindowStartingState.applescript */; };
84FF69B11FC3793300DC198E /* FaviconURLFinder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */; };
B24E9ADC245AB88400DA5718 /* NSAttributedString+NetNewsWire.swift in Sources */ = {isa = PBXBuildFile; fileRef = B24E9ABA245AB88300DA5718 /* NSAttributedString+NetNewsWire.swift */; };
B24E9ADE245AB88400DA5718 /* NSAttributedString+NetNewsWire.swift in Sources */ = {isa = PBXBuildFile; fileRef = B24E9ABA245AB88300DA5718 /* NSAttributedString+NetNewsWire.swift */; };
B27EEBF9244D15F3000932E6 /* stylesheet.css in Resources */ = {isa = PBXBuildFile; fileRef = B27EEBDF244D15F2000932E6 /* stylesheet.css */; };
@@ -1042,7 +1038,6 @@
841ABA5F20145EC100980E11 /* BuiltinSmartFeedInspectorViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BuiltinSmartFeedInspectorViewController.swift; sourceTree = "<group>"; };
84216D0222128B9D0049B9B9 /* DetailWebViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DetailWebViewController.swift; sourceTree = "<group>"; };
842611891FCB67AA0086A189 /* WebFeedIconDownloader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebFeedIconDownloader.swift; sourceTree = "<group>"; };
8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = HTMLMetadataDownloader.swift; sourceTree = "<group>"; };
8426119F1FCB72600086A189 /* FeaturedImageDownloader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeaturedImageDownloader.swift; sourceTree = "<group>"; };
842611A11FCB769D0086A189 /* RSHTMLMetadata+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "RSHTMLMetadata+Extension.swift"; sourceTree = "<group>"; };
842E45CD1ED8C308000A8B52 /* AppNotifications.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppNotifications.swift; sourceTree = "<group>"; };
@@ -1182,7 +1177,6 @@
84F9EAE1213660A100CF2DE4 /* testGenericScript.applescript */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.applescript; path = testGenericScript.applescript; sourceTree = "<group>"; };
84F9EAE2213660A100CF2DE4 /* establishMainWindowStartingState.applescript */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.applescript; path = establishMainWindowStartingState.applescript; sourceTree = "<group>"; };
84F9EAE4213660A100CF2DE4 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FaviconURLFinder.swift; sourceTree = "<group>"; };
B24E9ABA245AB88300DA5718 /* NSAttributedString+NetNewsWire.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "NSAttributedString+NetNewsWire.swift"; sourceTree = "<group>"; };
B24EFD482330FF99006C6242 /* NetNewsWire-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "NetNewsWire-Bridging-Header.h"; sourceTree = "<group>"; };
B24EFD5923310109006C6242 /* WKPreferencesPrivate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WKPreferencesPrivate.h; sourceTree = "<group>"; };
@@ -1772,14 +1766,6 @@
path = "NetNewsWire-iOSTests";
sourceTree = "<group>";
};
8426119C1FCB6ED40086A189 /* HTMLMetadata */ = {
isa = PBXGroup;
children = (
8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */,
);
path = HTMLMetadata;
sourceTree = "<group>";
};
842E45E11ED8C681000A8B52 /* MainWindow */ = {
isa = PBXGroup;
children = (
@@ -1882,7 +1868,6 @@
51EF0F78227716380050506E /* ColorHash.swift */,
848F6AE41FC29CFA002D422E /* FaviconDownloader.swift */,
51EF0F76227716200050506E /* FaviconGenerator.swift */,
84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */,
845A29081FC74B8E007B49E3 /* SingleFaviconDownloader.swift */,
);
path = Favicons;
@@ -2117,7 +2102,6 @@
51B5C85A23F22A7A00032075 /* ShareExtension */,
848F6AE31FC29CFA002D422E /* Favicons */,
845213211FCA5B10003B6E93 /* Images */,
8426119C1FCB6ED40086A189 /* HTMLMetadata */,
5183CCEA226F70350010922C /* Timer */,
512E08DD22687FA000BDCFDD /* Tree */,
849A97561ED9EB0D007D329B /* Extensions */,
@@ -3216,7 +3200,6 @@
512AF9DD236F05230066F8BE /* InteractiveLabel.swift in Sources */,
51E3EB3D229AB08300645299 /* ErrorHandler.swift in Sources */,
5183CCE5226F4DFA0010922C /* RefreshInterval.swift in Sources */,
51C4529D22650A1000C03939 /* FaviconURLFinder.swift in Sources */,
5142192A23522B5500E07E2C /* ImageViewController.swift in Sources */,
516244E3241E19F000B61C47 /* ColorPaletteTableViewController.swift in Sources */,
51C45258226508CF00C03939 /* AppAssets.swift in Sources */,
@@ -3290,7 +3273,6 @@
5108F6D22375EED2001ABC45 /* TimelineCustomizerViewController.swift in Sources */,
519E743D22C663F900A78E47 /* SceneDelegate.swift in Sources */,
FFD43E412340F488009E5CA3 /* MarkAsReadAlertController.swift in Sources */,
51C452A322650A1E00C03939 /* HTMLMetadataDownloader.swift in Sources */,
51C4528D2265095F00C03939 /* AddFolderViewController.swift in Sources */,
8413C1382D050A1E002E3D0F /* UniformTypeIdentifiers+Extras.swift in Sources */,
51C452782265091600C03939 /* MasterTimelineCellData.swift in Sources */,
@@ -3369,7 +3351,6 @@
84A14FF320048CA70046AD9A /* SendToMicroBlogCommand.swift in Sources */,
B2B8075E239C49D300F191E0 /* RSImage-AppIcons.swift in Sources */,
849A97891ED9ECEF007D329B /* ArticleTheme.swift in Sources */,
84FF69B11FC3793300DC198E /* FaviconURLFinder.swift in Sources */,
84B7178C201E66580091657D /* SidebarViewController+ContextualMenus.swift in Sources */,
842611A21FCB769D0086A189 /* RSHTMLMetadata+Extension.swift in Sources */,
84A1500520048DDF0046AD9A /* SendToMarsEditCommand.swift in Sources */,
@@ -3427,7 +3408,6 @@
510C417F24E5D1AE008226FD /* ExtensionContainersFile.swift in Sources */,
84C9FC7A22629E1200D921D6 /* PreferencesTableViewBackgroundView.swift in Sources */,
84CAFCAF22BC8C35007694F0 /* FetchRequestOperation.swift in Sources */,
8426119E1FCB6ED40086A189 /* HTMLMetadataDownloader.swift in Sources */,
849A976E1ED9EBC8007D329B /* TimelineViewController.swift in Sources */,
5154368B229404D1005E1CDF /* FaviconGenerator.swift in Sources */,
5183CCE6226F4E110010922C /* RefreshInterval.swift in Sources */,

2
RSWeb/Sources/RSWeb/HTMLMetadataCache.swift
View File

@@ -29,7 +29,7 @@ final class HTMLMetadataCache: Sendable {
let dateCreated = Date()
}
private let cache = Cache<HTMLMetadataCacheRecord>(timeToLive: TimeInterval(21 * 60 * 60), timeBetweenCleanups: TimeInterval(10 * 60 * 60))
private let cache = Cache<HTMLMetadataCacheRecord>(timeToLive: TimeInterval(hours: 21), timeBetweenCleanups: TimeInterval(hours: 10))
subscript(_ url: String) -> RSHTMLMetadata? {
get {

118
RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift Normal file
View File

@@ -0,0 +1,118 @@
//
// HTMLMetadataDownloader.swift
// NetNewsWire
//
// Created by Brent Simmons on 11/26/17.
// Copyright © 2017 Ranchero Software. All rights reserved.
//
import Foundation
import os
import RSParser
public final class HTMLMetadataDownloader: Sendable {
public static let shared = HTMLMetadataDownloader()
private static let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "HTMLMetadataDownloader")
private static let debugLoggingEnabled = true
private let cache = HTMLMetadataCache()
private let attemptDatesLock = OSAllocatedUnfairLock(initialState: [String: Date]())
private let urlsReturning4xxsLock = OSAllocatedUnfairLock(initialState: Set<String>())
public func cachedMetadata(for url: String) -> RSHTMLMetadata? {
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader requested cached metadata for \(url)")
}
guard let htmlMetadata = cache[url] else {
downloadMetadataIfNeeded(url)
return nil
}
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader returning cached metadata for \(url)")
}
return htmlMetadata
}
}
private extension HTMLMetadataDownloader {
func downloadMetadataIfNeeded(_ url: String) {
if urlShouldBeSkippedDueToPrevious4xxResponse(url) {
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an earlier request returned a 4xx response.")
}
return
}
// We try a download once an hour at most.
let shouldDownload = attemptDatesLock.withLock { attemptDates in
let currentDate = Date()
if let attemptDate = attemptDates[url], attemptDate > currentDate.bySubtracting(hours: 1) {
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an attempt was made less than an hour ago.")
}
return false
}
attemptDates[url] = currentDate
return true
}
if shouldDownload {
downloadMetadata(url)
}
}
func downloadMetadata(_ url: String) {
guard let actualURL = URL(unicodeString: url) else {
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because it couldn’t construct a URL.")
}
return
}
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader downloading for \(url)")
}
Downloader.shared.download(actualURL) { data, response, error in
if let data, !data.isEmpty, let response, response.statusIsOK {
let urlToUse = response.url ?? actualURL
let parserData = ParserData(url: urlToUse.absoluteString, data: data)
let htmlMetadata = RSHTMLMetadataParser.htmlMetadata(with: parserData)
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader caching parsed metadata for \(url)")
}
self.cache[url] = htmlMetadata
return
}
if let statusCode = response?.forcedStatusCode, (400...499).contains(statusCode) {
self.noteURLDidReturn4xx(url)
}
if Self.debugLoggingEnabled {
Self.logger.debug("HTMLMetadataDownloader failed download for \(url)")
}
}
}
func urlShouldBeSkippedDueToPrevious4xxResponse(_ url: String) -> Bool {
urlsReturning4xxsLock.withLock { $0.contains(url) }
}
func noteURLDidReturn4xx(_ url: String) {
_ = urlsReturning4xxsLock.withLock { $0.insert(url) }
}
}

84
Shared/Favicons/FaviconDownloader.swift
View File

@@ -11,6 +11,9 @@ import CoreServices
import Articles
import Account
import RSCore
import RSWeb
import RSParser
import UniformTypeIdentifiers
extension Notification.Name {
@@ -132,15 +135,13 @@ final class FaviconDownloader {
return favicon(with: faviconURL, homePageURL: url)
}
findFaviconURLs(with: url) { (faviconURLs) in
if let faviconURLs = faviconURLs {
// If the site explicitly specifies favicon.ico, it will appear twice.
self.currentHomePageHasOnlyFaviconICO = faviconURLs.count == 1
if let faviconURLs = findFaviconURLs(with: url) {
// If the site explicitly specifies favicon.ico, it will appear twice.
self.currentHomePageHasOnlyFaviconICO = faviconURLs.count == 1
if let firstIconURL = faviconURLs.first {
let _ = self.favicon(with: firstIconURL, homePageURL: url)
self.remainingFaviconURLs[url] = faviconURLs.dropFirst()
}
if let firstIconURL = faviconURLs.first {
let _ = self.favicon(with: firstIconURL, homePageURL: url)
self.remainingFaviconURLs[url] = faviconURLs.dropFirst()
}
}
@@ -196,31 +197,22 @@ private extension FaviconDownloader {
static let localeForLowercasing = Locale(identifier: "en_US")
func findFaviconURLs(with homePageURL: String, _ completion: @escaping ([String]?) -> Void) {
func findFaviconURLs(with homePageURL: String) -> [String]? {
guard let url = URL(unicodeString: homePageURL) else {
completion(nil)
return
guard let url = URL(string: homePageURL) else {
return nil
}
guard let htmlMetadata = HTMLMetadataDownloader.shared.cachedMetadata(for: homePageURL) else {
return nil
}
let faviconURLs = htmlMetadata.usableFaviconURLs() ?? [String]()
guard let scheme = url.scheme, let host = url.host else {
return faviconURLs.isEmpty ? nil : faviconURLs
}
FaviconURLFinder.findFaviconURLs(with: homePageURL) { (faviconURLs) in
guard var faviconURLs = faviconURLs else {
completion(nil)
return
}
var defaultFaviconURL: String? = nil
if let scheme = url.scheme, let host = url.host {
defaultFaviconURL = "\(scheme)://\(host)/favicon.ico".lowercased(with: FaviconDownloader.localeForLowercasing)
}
if let defaultFaviconURL = defaultFaviconURL {
faviconURLs.append(defaultFaviconURL)
}
completion(faviconURLs)
}
let defaultFaviconURL = "\(scheme)://\(host)/favicon.ico".lowercased(with: FaviconDownloader.localeForLowercasing)
return faviconURLs + [defaultFaviconURL]
}
func faviconDownloader(withURL faviconURL: String, homePageURL: String?) -> SingleFaviconDownloader {
@@ -311,5 +303,35 @@ private extension FaviconDownloader {
assertionFailure(error.localizedDescription)
}
}
}
private extension RSHTMLMetadata {
func usableFaviconURLs() -> [String]? {
favicons.compactMap { favicon in
shouldAllowFavicon(favicon) ? favicon.urlString : nil
}
}
static let ignoredTypes = [UTType.svg]
private func shouldAllowFavicon(_ favicon: RSHTMLMetadataFavicon) -> Bool {
// Check mime type.
if let mimeType = favicon.type, let utType = UTType(mimeType: mimeType) {
if Self.ignoredTypes.contains(utType) {
return false
}
}
// Check file extension.
if let urlString = favicon.urlString, let url = URL(string: urlString), let utType = UTType(filenameExtension: url.pathExtension) {
if Self.ignoredTypes.contains(utType) {
return false
}
}
return true
}
}

66
Shared/Favicons/FaviconURLFinder.swift
View File

@@ -1,66 +0,0 @@
//
// FaviconURLFinder.swift
// NetNewsWire
//
// Created by Brent Simmons on 11/20/17.
// Copyright © 2017 Ranchero Software. All rights reserved.
//
import Foundation
import CoreServices
import RSParser
import UniformTypeIdentifiers
// The favicon URLs may be specified in the head section of the home page.
struct FaviconURLFinder {
/// Finds favicon URLs in a web page.
/// - Parameters:
/// - homePageURL: The page to search.
/// - completion: A closure called when the links have been found.
/// - urls: An array of favicon URLs as strings.
static func findFaviconURLs(with homePageURL: String, _ completion: @escaping (_ urls: [String]?) -> Void) {
guard let _ = URL(unicodeString: homePageURL) else {
completion(nil)
return
}
// If the favicon has an explicit type, check that for an ignored type; otherwise, check the file extension.
HTMLMetadataDownloader.downloadMetadata(for: homePageURL) { (htmlMetadata) in
guard let favicons = htmlMetadata?.favicons else {
completion(nil)
return
}
let faviconURLs = favicons.compactMap {
shouldAllowFavicon($0) ? $0.urlString : nil
}
completion(faviconURLs)
}
}
private static let ignoredTypes = [UTType.svg]
private static func shouldAllowFavicon(_ favicon: RSHTMLMetadataFavicon) -> Bool {
// Check mime type.
if let mimeType = favicon.type, let utType = UTType(mimeType: mimeType) {
if Self.ignoredTypes.contains(utType) {
return false
}
}
// Check file extension.
if let urlString = favicon.urlString, let url = URL(string: urlString), let utType = UTType(filenameExtension: url.pathExtension) {
if Self.ignoredTypes.contains(utType) {
return false
}
}
return true
}
}

64
Shared/HTMLMetadata/HTMLMetadataDownloader.swift
View File

@@ -1,64 +0,0 @@
//
// HTMLMetadataDownloader.swift
// NetNewsWire
//
// Created by Brent Simmons on 11/26/17.
// Copyright © 2017 Ranchero Software. All rights reserved.
//
import Foundation
import RSWeb
import RSParser
import os
struct HTMLMetadataDownloader {
static let serialDispatchQueue = DispatchQueue(label: "HTMLMetadataDownloader")
static let currentURLsLock = OSAllocatedUnfairLock(initialState: Set<URL>())
static func downloadMetadata(for url: String, _ completion: @escaping (RSHTMLMetadata?) -> Void) {
guard let actualURL = URL(unicodeString: url) else {
completion(nil)
return
}
let urlDownloadIsInProgress = currentURLsLock.withLock { currentURLs in
if currentURLs.contains(actualURL) {
return true
}
currentURLs.insert(actualURL)
return false
}
if urlDownloadIsInProgress {
completion(nil)
return
}
Downloader.shared.download(actualURL) { (data, response, error) in
_ = currentURLsLock.withLock { currentURLs in
currentURLs.remove(actualURL)
}
if let data = data, !data.isEmpty, let response = response, response.statusIsOK, error == nil {
let urlToUse = response.url ?? actualURL
let parserData = ParserData(url: urlToUse.absoluteString, data: data)
parseMetadata(with: parserData, completion)
return
}
completion(nil)
}
}
private static func parseMetadata(with parserData: ParserData, _ completion: @escaping (RSHTMLMetadata?) -> Void) {
serialDispatchQueue.async {
let htmlMetadata = RSHTMLMetadataParser.htmlMetadata(with: parserData)
DispatchQueue.main.async {
completion(htmlMetadata)
}
}
}
}

30
Shared/Images/WebFeedIconDownloader.swift
View File

@@ -165,7 +165,18 @@ private extension WebFeedIconDownloader {
return
}
findIconURLForHomePageURL(homePageURL, feed: feed)
guard let metadata = HTMLMetadataDownloader.shared.cachedMetadata(for: homePageURL) else {
imageResultBlock(nil)
return
}
if let url = metadata.bestWebsiteIconURL() {
cacheIconURL(for: homePageURL, url)
icon(forURL: url, feed: feed, imageResultBlock)
return
}
homePagesWithNoIconURLCache.insert(homePageURL)
}
func icon(forURL url: String, feed: WebFeed, _ imageResultBlock: @escaping (RSImage?) -> Void) {
@@ -197,23 +208,6 @@ private extension WebFeedIconDownloader {
homePageToIconURLCacheDirty = true
}
func findIconURLForHomePageURL(_ homePageURL: String, feed: WebFeed) {
guard !urlsInProgress.contains(homePageURL) else {
return
}
urlsInProgress.insert(homePageURL)
HTMLMetadataDownloader.downloadMetadata(for: homePageURL) { (metadata) in
self.urlsInProgress.remove(homePageURL)
guard let metadata = metadata else {
return
}
self.pullIconURL(from: metadata, homePageURL: homePageURL, feed: feed)
}
}
func pullIconURL(from metadata: RSHTMLMetadata, homePageURL: String, feed: WebFeed) {
if let url = metadata.bestWebsiteIconURL() {