diff --git a/Core/Sources/Core/FoundationExtras/Date+Extensions.swift b/Core/Sources/Core/FoundationExtras/Date+Extensions.swift new file mode 100755 index 000000000..f1ef88f01 --- /dev/null +++ b/Core/Sources/Core/FoundationExtras/Date+Extensions.swift @@ -0,0 +1,37 @@ +// +// Date+Extensions.swift +// RSCore +// +// Created by Brent Simmons on 6/21/16. +// Copyright © 2016 Ranchero Software, LLC. All rights reserved. +// + +import Foundation + +public extension Date { + + // Below are for rough use only — they don't use the calendar. + + func bySubtracting(days: Int) -> Date { + return addingTimeInterval(0.0 - TimeInterval(days: days)) + } + + func bySubtracting(hours: Int) -> Date { + return addingTimeInterval(0.0 - TimeInterval(hours: hours)) + } + + func byAdding(days: Int) -> Date { + return addingTimeInterval(TimeInterval(days: days)) + } +} + +public extension TimeInterval { + + init(days: Int) { + self.init(days * 24 * 60 * 60) + } + + init(hours: Int) { + self.init(hours * 60 * 60) + } +} diff --git a/NetNewsWire.xcodeproj/project.pbxproj b/NetNewsWire.xcodeproj/project.pbxproj index 9f7020527..7e458c6a5 100644 --- a/NetNewsWire.xcodeproj/project.pbxproj +++ b/NetNewsWire.xcodeproj/project.pbxproj @@ -296,12 +296,10 @@ 51C4529A22650A0400C03939 /* ArticleTheme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849A97871ED9ECEF007D329B /* ArticleTheme.swift */; }; 51C4529B22650A1000C03939 /* FaviconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 848F6AE41FC29CFA002D422E /* FaviconDownloader.swift */; }; 51C4529C22650A1000C03939 /* SingleFaviconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845A29081FC74B8E007B49E3 /* SingleFaviconDownloader.swift */; }; - 51C4529D22650A1000C03939 /* FaviconURLFinder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */; }; 51C4529E22650A1900C03939 /* ImageDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845213221FCA5B10003B6E93 /* ImageDownloader.swift */; }; 51C4529F22650A1900C03939 /* AuthorAvatarDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84E850851FCB60CE0072EA88 /* AuthorAvatarDownloader.swift */; }; 51C452A022650A1900C03939 /* WebFeedIconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611891FCB67AA0086A189 /* WebFeedIconDownloader.swift */; }; 51C452A222650A1900C03939 /* RSHTMLMetadata+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611A11FCB769D0086A189 /* RSHTMLMetadata+Extension.swift */; }; - 51C452A322650A1E00C03939 /* HTMLMetadataDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */; }; 51C452A422650A2D00C03939 /* ArticleUtilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849A97581ED9EB0D007D329B /* ArticleUtilities.swift */; }; 51C452A522650A2D00C03939 /* SmallIconProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84411E701FE5FBFA004B527F /* SmallIconProvider.swift */; }; 51C452A622650A3500C03939 /* Node-Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849A97971ED9EFAA007D329B /* Node-Extensions.swift */; }; @@ -400,7 +398,6 @@ 841ABA6020145EC100980E11 /* BuiltinSmartFeedInspectorViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 841ABA5F20145EC100980E11 /* BuiltinSmartFeedInspectorViewController.swift */; }; 84216D0322128B9D0049B9B9 /* DetailWebViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84216D0222128B9D0049B9B9 /* DetailWebViewController.swift */; }; 8426118A1FCB67AA0086A189 /* WebFeedIconDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611891FCB67AA0086A189 /* WebFeedIconDownloader.swift */; }; - 8426119E1FCB6ED40086A189 /* HTMLMetadataDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */; }; 842611A21FCB769D0086A189 /* RSHTMLMetadata+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842611A11FCB769D0086A189 /* RSHTMLMetadata+Extension.swift */; }; 842E45CE1ED8C308000A8B52 /* AppNotifications.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842E45CD1ED8C308000A8B52 /* AppNotifications.swift */; }; 842E45DD1ED8C54B000A8B52 /* Browser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842E45DC1ED8C54B000A8B52 /* Browser.swift */; }; @@ -537,7 +534,6 @@ 84F9EAF3213660A100CF2DE4 /* testCurrentArticleIsNil.applescript in Sources */ = {isa = PBXBuildFile; fileRef = 84F9EAE0213660A100CF2DE4 /* testCurrentArticleIsNil.applescript */; }; 84F9EAF4213660A100CF2DE4 /* testGenericScript.applescript in Sources */ = {isa = PBXBuildFile; fileRef = 84F9EAE1213660A100CF2DE4 /* testGenericScript.applescript */; }; 84F9EAF5213660A100CF2DE4 /* establishMainWindowStartingState.applescript in Sources */ = {isa = PBXBuildFile; fileRef = 84F9EAE2213660A100CF2DE4 /* establishMainWindowStartingState.applescript */; }; - 84FF69B11FC3793300DC198E /* FaviconURLFinder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */; }; B24E9ADC245AB88400DA5718 /* NSAttributedString+NetNewsWire.swift in Sources */ = {isa = PBXBuildFile; fileRef = B24E9ABA245AB88300DA5718 /* NSAttributedString+NetNewsWire.swift */; }; B24E9ADE245AB88400DA5718 /* NSAttributedString+NetNewsWire.swift in Sources */ = {isa = PBXBuildFile; fileRef = B24E9ABA245AB88300DA5718 /* NSAttributedString+NetNewsWire.swift */; }; B27EEBF9244D15F3000932E6 /* stylesheet.css in Resources */ = {isa = PBXBuildFile; fileRef = B27EEBDF244D15F2000932E6 /* stylesheet.css */; }; @@ -1042,7 +1038,6 @@ 841ABA5F20145EC100980E11 /* BuiltinSmartFeedInspectorViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BuiltinSmartFeedInspectorViewController.swift; sourceTree = ""; }; 84216D0222128B9D0049B9B9 /* DetailWebViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DetailWebViewController.swift; sourceTree = ""; }; 842611891FCB67AA0086A189 /* WebFeedIconDownloader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebFeedIconDownloader.swift; sourceTree = ""; }; - 8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = HTMLMetadataDownloader.swift; sourceTree = ""; }; 8426119F1FCB72600086A189 /* FeaturedImageDownloader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeaturedImageDownloader.swift; sourceTree = ""; }; 842611A11FCB769D0086A189 /* RSHTMLMetadata+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "RSHTMLMetadata+Extension.swift"; sourceTree = ""; }; 842E45CD1ED8C308000A8B52 /* AppNotifications.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppNotifications.swift; sourceTree = ""; }; @@ -1182,7 +1177,6 @@ 84F9EAE1213660A100CF2DE4 /* testGenericScript.applescript */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.applescript; path = testGenericScript.applescript; sourceTree = ""; }; 84F9EAE2213660A100CF2DE4 /* establishMainWindowStartingState.applescript */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.applescript; path = establishMainWindowStartingState.applescript; sourceTree = ""; }; 84F9EAE4213660A100CF2DE4 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; - 84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FaviconURLFinder.swift; sourceTree = ""; }; B24E9ABA245AB88300DA5718 /* NSAttributedString+NetNewsWire.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "NSAttributedString+NetNewsWire.swift"; sourceTree = ""; }; B24EFD482330FF99006C6242 /* NetNewsWire-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "NetNewsWire-Bridging-Header.h"; sourceTree = ""; }; B24EFD5923310109006C6242 /* WKPreferencesPrivate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WKPreferencesPrivate.h; sourceTree = ""; }; @@ -1772,14 +1766,6 @@ path = "NetNewsWire-iOSTests"; sourceTree = ""; }; - 8426119C1FCB6ED40086A189 /* HTMLMetadata */ = { - isa = PBXGroup; - children = ( - 8426119D1FCB6ED40086A189 /* HTMLMetadataDownloader.swift */, - ); - path = HTMLMetadata; - sourceTree = ""; - }; 842E45E11ED8C681000A8B52 /* MainWindow */ = { isa = PBXGroup; children = ( @@ -1882,7 +1868,6 @@ 51EF0F78227716380050506E /* ColorHash.swift */, 848F6AE41FC29CFA002D422E /* FaviconDownloader.swift */, 51EF0F76227716200050506E /* FaviconGenerator.swift */, - 84FF69B01FC3793300DC198E /* FaviconURLFinder.swift */, 845A29081FC74B8E007B49E3 /* SingleFaviconDownloader.swift */, ); path = Favicons; @@ -2117,7 +2102,6 @@ 51B5C85A23F22A7A00032075 /* ShareExtension */, 848F6AE31FC29CFA002D422E /* Favicons */, 845213211FCA5B10003B6E93 /* Images */, - 8426119C1FCB6ED40086A189 /* HTMLMetadata */, 5183CCEA226F70350010922C /* Timer */, 512E08DD22687FA000BDCFDD /* Tree */, 849A97561ED9EB0D007D329B /* Extensions */, @@ -3216,7 +3200,6 @@ 512AF9DD236F05230066F8BE /* InteractiveLabel.swift in Sources */, 51E3EB3D229AB08300645299 /* ErrorHandler.swift in Sources */, 5183CCE5226F4DFA0010922C /* RefreshInterval.swift in Sources */, - 51C4529D22650A1000C03939 /* FaviconURLFinder.swift in Sources */, 5142192A23522B5500E07E2C /* ImageViewController.swift in Sources */, 516244E3241E19F000B61C47 /* ColorPaletteTableViewController.swift in Sources */, 51C45258226508CF00C03939 /* AppAssets.swift in Sources */, @@ -3290,7 +3273,6 @@ 5108F6D22375EED2001ABC45 /* TimelineCustomizerViewController.swift in Sources */, 519E743D22C663F900A78E47 /* SceneDelegate.swift in Sources */, FFD43E412340F488009E5CA3 /* MarkAsReadAlertController.swift in Sources */, - 51C452A322650A1E00C03939 /* HTMLMetadataDownloader.swift in Sources */, 51C4528D2265095F00C03939 /* AddFolderViewController.swift in Sources */, 8413C1382D050A1E002E3D0F /* UniformTypeIdentifiers+Extras.swift in Sources */, 51C452782265091600C03939 /* MasterTimelineCellData.swift in Sources */, @@ -3369,7 +3351,6 @@ 84A14FF320048CA70046AD9A /* SendToMicroBlogCommand.swift in Sources */, B2B8075E239C49D300F191E0 /* RSImage-AppIcons.swift in Sources */, 849A97891ED9ECEF007D329B /* ArticleTheme.swift in Sources */, - 84FF69B11FC3793300DC198E /* FaviconURLFinder.swift in Sources */, 84B7178C201E66580091657D /* SidebarViewController+ContextualMenus.swift in Sources */, 842611A21FCB769D0086A189 /* RSHTMLMetadata+Extension.swift in Sources */, 84A1500520048DDF0046AD9A /* SendToMarsEditCommand.swift in Sources */, @@ -3427,7 +3408,6 @@ 510C417F24E5D1AE008226FD /* ExtensionContainersFile.swift in Sources */, 84C9FC7A22629E1200D921D6 /* PreferencesTableViewBackgroundView.swift in Sources */, 84CAFCAF22BC8C35007694F0 /* FetchRequestOperation.swift in Sources */, - 8426119E1FCB6ED40086A189 /* HTMLMetadataDownloader.swift in Sources */, 849A976E1ED9EBC8007D329B /* TimelineViewController.swift in Sources */, 5154368B229404D1005E1CDF /* FaviconGenerator.swift in Sources */, 5183CCE6226F4E110010922C /* RefreshInterval.swift in Sources */, diff --git a/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift b/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift index a1508d2fc..33777ba50 100644 --- a/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift +++ b/RSWeb/Sources/RSWeb/HTMLMetadataCache.swift @@ -29,7 +29,7 @@ final class HTMLMetadataCache: Sendable { let dateCreated = Date() } - private let cache = Cache(timeToLive: TimeInterval(21 * 60 * 60), timeBetweenCleanups: TimeInterval(10 * 60 * 60)) + private let cache = Cache(timeToLive: TimeInterval(hours: 21), timeBetweenCleanups: TimeInterval(hours: 10)) subscript(_ url: String) -> RSHTMLMetadata? { get { diff --git a/RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift b/RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift new file mode 100644 index 000000000..4b30e5515 --- /dev/null +++ b/RSWeb/Sources/RSWeb/HTMLMetadataDownloader.swift @@ -0,0 +1,118 @@ +// +// HTMLMetadataDownloader.swift +// NetNewsWire +// +// Created by Brent Simmons on 11/26/17. +// Copyright © 2017 Ranchero Software. All rights reserved. +// + +import Foundation +import os +import RSParser + +public final class HTMLMetadataDownloader: Sendable { + + public static let shared = HTMLMetadataDownloader() + + private static let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "HTMLMetadataDownloader") + private static let debugLoggingEnabled = true + + private let cache = HTMLMetadataCache() + private let attemptDatesLock = OSAllocatedUnfairLock(initialState: [String: Date]()) + private let urlsReturning4xxsLock = OSAllocatedUnfairLock(initialState: Set()) + + public func cachedMetadata(for url: String) -> RSHTMLMetadata? { + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader requested cached metadata for \(url)") + } + + guard let htmlMetadata = cache[url] else { + downloadMetadataIfNeeded(url) + return nil + } + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader returning cached metadata for \(url)") + } + return htmlMetadata + } +} + +private extension HTMLMetadataDownloader { + + func downloadMetadataIfNeeded(_ url: String) { + + if urlShouldBeSkippedDueToPrevious4xxResponse(url) { + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an earlier request returned a 4xx response.") + } + return + } + + // We try a download once an hour at most. + let shouldDownload = attemptDatesLock.withLock { attemptDates in + + let currentDate = Date() + + if let attemptDate = attemptDates[url], attemptDate > currentDate.bySubtracting(hours: 1) { + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because an attempt was made less than an hour ago.") + } + return false + } + + attemptDates[url] = currentDate + return true + } + + if shouldDownload { + downloadMetadata(url) + } + } + + func downloadMetadata(_ url: String) { + + guard let actualURL = URL(unicodeString: url) else { + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader skipping download for \(url) because it couldn’t construct a URL.") + } + return + } + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader downloading for \(url)") + } + + Downloader.shared.download(actualURL) { data, response, error in + if let data, !data.isEmpty, let response, response.statusIsOK { + let urlToUse = response.url ?? actualURL + let parserData = ParserData(url: urlToUse.absoluteString, data: data) + let htmlMetadata = RSHTMLMetadataParser.htmlMetadata(with: parserData) + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader caching parsed metadata for \(url)") + } + self.cache[url] = htmlMetadata + return + } + + if let statusCode = response?.forcedStatusCode, (400...499).contains(statusCode) { + self.noteURLDidReturn4xx(url) + } + + if Self.debugLoggingEnabled { + Self.logger.debug("HTMLMetadataDownloader failed download for \(url)") + } + } + } + + func urlShouldBeSkippedDueToPrevious4xxResponse(_ url: String) -> Bool { + + urlsReturning4xxsLock.withLock { $0.contains(url) } + } + + func noteURLDidReturn4xx(_ url: String) { + + _ = urlsReturning4xxsLock.withLock { $0.insert(url) } + } +} diff --git a/Shared/Favicons/FaviconDownloader.swift b/Shared/Favicons/FaviconDownloader.swift index bc0712e5f..b0942f35b 100644 --- a/Shared/Favicons/FaviconDownloader.swift +++ b/Shared/Favicons/FaviconDownloader.swift @@ -11,6 +11,9 @@ import CoreServices import Articles import Account import RSCore +import RSWeb +import RSParser +import UniformTypeIdentifiers extension Notification.Name { @@ -132,15 +135,13 @@ final class FaviconDownloader { return favicon(with: faviconURL, homePageURL: url) } - findFaviconURLs(with: url) { (faviconURLs) in - if let faviconURLs = faviconURLs { - // If the site explicitly specifies favicon.ico, it will appear twice. - self.currentHomePageHasOnlyFaviconICO = faviconURLs.count == 1 + if let faviconURLs = findFaviconURLs(with: url) { + // If the site explicitly specifies favicon.ico, it will appear twice. + self.currentHomePageHasOnlyFaviconICO = faviconURLs.count == 1 - if let firstIconURL = faviconURLs.first { - let _ = self.favicon(with: firstIconURL, homePageURL: url) - self.remainingFaviconURLs[url] = faviconURLs.dropFirst() - } + if let firstIconURL = faviconURLs.first { + let _ = self.favicon(with: firstIconURL, homePageURL: url) + self.remainingFaviconURLs[url] = faviconURLs.dropFirst() } } @@ -196,31 +197,22 @@ private extension FaviconDownloader { static let localeForLowercasing = Locale(identifier: "en_US") - func findFaviconURLs(with homePageURL: String, _ completion: @escaping ([String]?) -> Void) { + func findFaviconURLs(with homePageURL: String) -> [String]? { - guard let url = URL(unicodeString: homePageURL) else { - completion(nil) - return + guard let url = URL(string: homePageURL) else { + return nil + } + guard let htmlMetadata = HTMLMetadataDownloader.shared.cachedMetadata(for: homePageURL) else { + return nil + } + let faviconURLs = htmlMetadata.usableFaviconURLs() ?? [String]() + + guard let scheme = url.scheme, let host = url.host else { + return faviconURLs.isEmpty ? nil : faviconURLs } - FaviconURLFinder.findFaviconURLs(with: homePageURL) { (faviconURLs) in - guard var faviconURLs = faviconURLs else { - completion(nil) - return - } - - var defaultFaviconURL: String? = nil - - if let scheme = url.scheme, let host = url.host { - defaultFaviconURL = "\(scheme)://\(host)/favicon.ico".lowercased(with: FaviconDownloader.localeForLowercasing) - } - - if let defaultFaviconURL = defaultFaviconURL { - faviconURLs.append(defaultFaviconURL) - } - - completion(faviconURLs) - } + let defaultFaviconURL = "\(scheme)://\(host)/favicon.ico".lowercased(with: FaviconDownloader.localeForLowercasing) + return faviconURLs + [defaultFaviconURL] } func faviconDownloader(withURL faviconURL: String, homePageURL: String?) -> SingleFaviconDownloader { @@ -311,5 +303,35 @@ private extension FaviconDownloader { assertionFailure(error.localizedDescription) } } - +} + +private extension RSHTMLMetadata { + + func usableFaviconURLs() -> [String]? { + + favicons.compactMap { favicon in + shouldAllowFavicon(favicon) ? favicon.urlString : nil + } + } + + static let ignoredTypes = [UTType.svg] + + private func shouldAllowFavicon(_ favicon: RSHTMLMetadataFavicon) -> Bool { + + // Check mime type. + if let mimeType = favicon.type, let utType = UTType(mimeType: mimeType) { + if Self.ignoredTypes.contains(utType) { + return false + } + } + + // Check file extension. + if let urlString = favicon.urlString, let url = URL(string: urlString), let utType = UTType(filenameExtension: url.pathExtension) { + if Self.ignoredTypes.contains(utType) { + return false + } + } + + return true + } } diff --git a/Shared/Favicons/FaviconURLFinder.swift b/Shared/Favicons/FaviconURLFinder.swift deleted file mode 100644 index c90b3ad4a..000000000 --- a/Shared/Favicons/FaviconURLFinder.swift +++ /dev/null @@ -1,66 +0,0 @@ -// -// FaviconURLFinder.swift -// NetNewsWire -// -// Created by Brent Simmons on 11/20/17. -// Copyright © 2017 Ranchero Software. All rights reserved. -// - -import Foundation -import CoreServices -import RSParser -import UniformTypeIdentifiers - -// The favicon URLs may be specified in the head section of the home page. - -struct FaviconURLFinder { - - /// Finds favicon URLs in a web page. - /// - Parameters: - /// - homePageURL: The page to search. - /// - completion: A closure called when the links have been found. - /// - urls: An array of favicon URLs as strings. - static func findFaviconURLs(with homePageURL: String, _ completion: @escaping (_ urls: [String]?) -> Void) { - - guard let _ = URL(unicodeString: homePageURL) else { - completion(nil) - return - } - - // If the favicon has an explicit type, check that for an ignored type; otherwise, check the file extension. - HTMLMetadataDownloader.downloadMetadata(for: homePageURL) { (htmlMetadata) in - - guard let favicons = htmlMetadata?.favicons else { - completion(nil) - return - } - - let faviconURLs = favicons.compactMap { - shouldAllowFavicon($0) ? $0.urlString : nil - } - - completion(faviconURLs) - } - } - - private static let ignoredTypes = [UTType.svg] - - private static func shouldAllowFavicon(_ favicon: RSHTMLMetadataFavicon) -> Bool { - - // Check mime type. - if let mimeType = favicon.type, let utType = UTType(mimeType: mimeType) { - if Self.ignoredTypes.contains(utType) { - return false - } - } - - // Check file extension. - if let urlString = favicon.urlString, let url = URL(string: urlString), let utType = UTType(filenameExtension: url.pathExtension) { - if Self.ignoredTypes.contains(utType) { - return false - } - } - - return true - } -} diff --git a/Shared/HTMLMetadata/HTMLMetadataDownloader.swift b/Shared/HTMLMetadata/HTMLMetadataDownloader.swift deleted file mode 100644 index 636233780..000000000 --- a/Shared/HTMLMetadata/HTMLMetadataDownloader.swift +++ /dev/null @@ -1,64 +0,0 @@ -// -// HTMLMetadataDownloader.swift -// NetNewsWire -// -// Created by Brent Simmons on 11/26/17. -// Copyright © 2017 Ranchero Software. All rights reserved. -// - -import Foundation -import RSWeb -import RSParser -import os - -struct HTMLMetadataDownloader { - - static let serialDispatchQueue = DispatchQueue(label: "HTMLMetadataDownloader") - - static let currentURLsLock = OSAllocatedUnfairLock(initialState: Set()) - - static func downloadMetadata(for url: String, _ completion: @escaping (RSHTMLMetadata?) -> Void) { - - guard let actualURL = URL(unicodeString: url) else { - completion(nil) - return - } - - let urlDownloadIsInProgress = currentURLsLock.withLock { currentURLs in - if currentURLs.contains(actualURL) { - return true - } - currentURLs.insert(actualURL) - return false - } - if urlDownloadIsInProgress { - completion(nil) - return - } - - Downloader.shared.download(actualURL) { (data, response, error) in - - _ = currentURLsLock.withLock { currentURLs in - currentURLs.remove(actualURL) - } - - if let data = data, !data.isEmpty, let response = response, response.statusIsOK, error == nil { - let urlToUse = response.url ?? actualURL - let parserData = ParserData(url: urlToUse.absoluteString, data: data) - parseMetadata(with: parserData, completion) - return - } - - completion(nil) - } - } - - private static func parseMetadata(with parserData: ParserData, _ completion: @escaping (RSHTMLMetadata?) -> Void) { - serialDispatchQueue.async { - let htmlMetadata = RSHTMLMetadataParser.htmlMetadata(with: parserData) - DispatchQueue.main.async { - completion(htmlMetadata) - } - } - } -} diff --git a/Shared/Images/WebFeedIconDownloader.swift b/Shared/Images/WebFeedIconDownloader.swift index 7790b6292..3f9ce8f1d 100644 --- a/Shared/Images/WebFeedIconDownloader.swift +++ b/Shared/Images/WebFeedIconDownloader.swift @@ -165,7 +165,18 @@ private extension WebFeedIconDownloader { return } - findIconURLForHomePageURL(homePageURL, feed: feed) + guard let metadata = HTMLMetadataDownloader.shared.cachedMetadata(for: homePageURL) else { + imageResultBlock(nil) + return + } + + if let url = metadata.bestWebsiteIconURL() { + cacheIconURL(for: homePageURL, url) + icon(forURL: url, feed: feed, imageResultBlock) + return + } + + homePagesWithNoIconURLCache.insert(homePageURL) } func icon(forURL url: String, feed: WebFeed, _ imageResultBlock: @escaping (RSImage?) -> Void) { @@ -197,23 +208,6 @@ private extension WebFeedIconDownloader { homePageToIconURLCacheDirty = true } - func findIconURLForHomePageURL(_ homePageURL: String, feed: WebFeed) { - - guard !urlsInProgress.contains(homePageURL) else { - return - } - urlsInProgress.insert(homePageURL) - - HTMLMetadataDownloader.downloadMetadata(for: homePageURL) { (metadata) in - - self.urlsInProgress.remove(homePageURL) - guard let metadata = metadata else { - return - } - self.pullIconURL(from: metadata, homePageURL: homePageURL, feed: feed) - } - } - func pullIconURL(from metadata: RSHTMLMetadata, homePageURL: String, feed: WebFeed) { if let url = metadata.bestWebsiteIconURL() {