diff --git a/Shared/Favicons/FaviconDownloader.swift b/Shared/Favicons/FaviconDownloader.swift
index eee115711..ccba7c6bb 100644
--- a/Shared/Favicons/FaviconDownloader.swift
+++ b/Shared/Favicons/FaviconDownloader.swift
@@ -131,20 +131,12 @@ final class FaviconDownloader {
 		}
 
 		findFaviconURLs(with: url) { (faviconURLs) in
-			var hasIcons = false
-
 			if let faviconURLs = faviconURLs {
 				if let firstIconURL = faviconURLs.first {
-					hasIcons = true
 					let _ = self.favicon(with: firstIconURL, homePageURL: url)
 					self.remainingFaviconURLs[url] = faviconURLs.dropFirst()
 				}
 			}
-
-			if (!hasIcons) {
-				self.homePageURLsWithNoFaviconURLCache.insert(url)
-				self.homePageURLsWithNoFaviconURLCacheDirty = true
-			}
 		}
 
 		return nil
@@ -167,6 +159,8 @@ final class FaviconDownloader {
 					remainingFaviconURLs[homePageURL] = faviconURLs.dropFirst();
 				} else {
 					remainingFaviconURLs[homePageURL] = nil
+					self.homePageURLsWithNoFaviconURLCache.insert(homePageURL)
+					self.homePageURLsWithNoFaviconURLCacheDirty = true
 				}
 			}
 			return