diff --git a/Frameworks/Account/Account.swift b/Frameworks/Account/Account.swift index 271677817..165bc9893 100644 --- a/Frameworks/Account/Account.swift +++ b/Frameworks/Account/Account.swift @@ -251,7 +251,8 @@ public final class Account: DisplayNameProvider, UnreadCountProvider, Container, self.dataFolder = dataFolder let databaseFilePath = (dataFolder as NSString).appendingPathComponent("DB.sqlite3") - self.database = ArticlesDatabase(databaseFilePath: databaseFilePath, accountID: accountID) + let retentionStyle: ArticlesDatabase.RetentionStyle = type == .onMyMac ? .feedBased : .syncSystem + self.database = ArticlesDatabase(databaseFilePath: databaseFilePath, accountID: accountID, retentionStyle: retentionStyle) switch type { case .onMyMac: @@ -676,56 +677,41 @@ public final class Account: DisplayNameProvider, UnreadCountProvider, Container, } func update(_ webFeed: WebFeed, with parsedFeed: ParsedFeed, _ completion: @escaping DatabaseCompletionBlock) { - // Used only by an On My Mac account. + // Used only by an On My Mac or iCloud account. + precondition(Thread.isMainThread) + precondition(type == .onMyMac) // TODO: allow iCloud + webFeed.takeSettings(from: parsedFeed) - let webFeedIDsAndItems = [webFeed.webFeedID: parsedFeed.items] - update(webFeedIDsAndItems: webFeedIDsAndItems, defaultRead: false, completion: completion) + let parsedItems = parsedFeed.items + guard !parsedItems.isEmpty else { + completion(nil) + return + } + + database.update(with: parsedItems, webFeedID: webFeed.webFeedID) { updateArticlesResult in + switch updateArticlesResult { + case .success(let newAndUpdatedArticles): + self.sendNotificationAbout(newAndUpdatedArticles) + completion(nil) + case .failure(let databaseError): + completion(databaseError) + } + } } func update(webFeedIDsAndItems: [String: Set], defaultRead: Bool, completion: @escaping DatabaseCompletionBlock) { + // Used only by syncing systems. precondition(Thread.isMainThread) + precondition(type != .onMyMac) // TODO: also make sure type != iCloud guard !webFeedIDsAndItems.isEmpty else { completion(nil) return } database.update(webFeedIDsAndItems: webFeedIDsAndItems, defaultRead: defaultRead) { updateArticlesResult in - - func sendNotificationAbout(newArticles: Set
?, updatedArticles: Set
?) { - var webFeeds = Set() - - if let newArticles = newArticles { - webFeeds.formUnion(Set(newArticles.compactMap { $0.webFeed })) - } - if let updatedArticles = updatedArticles { - webFeeds.formUnion(Set(updatedArticles.compactMap { $0.webFeed })) - } - - var shouldSendNotification = false - var userInfo = [String: Any]() - - if let newArticles = newArticles, !newArticles.isEmpty { - shouldSendNotification = true - userInfo[UserInfoKey.newArticles] = newArticles - self.updateUnreadCounts(for: webFeeds) { - NotificationCenter.default.post(name: .DownloadArticlesDidUpdateUnreadCounts, object: self, userInfo: nil) - } - } - - if let updatedArticles = updatedArticles, !updatedArticles.isEmpty { - shouldSendNotification = true - userInfo[UserInfoKey.updatedArticles] = updatedArticles - } - - if shouldSendNotification { - userInfo[UserInfoKey.webFeeds] = webFeeds - NotificationCenter.default.post(name: .AccountDidDownloadArticles, object: self, userInfo: userInfo) - } - } - switch updateArticlesResult { case .success(let newAndUpdatedArticles): - sendNotificationAbout(newArticles: newAndUpdatedArticles.newArticles, updatedArticles: newAndUpdatedArticles.updatedArticles) + self.sendNotificationAbout(newAndUpdatedArticles) completion(nil) case .failure(let databaseError): completion(databaseError) @@ -1245,6 +1231,38 @@ private extension Account { feed.unreadCount = unreadCount } } + + func sendNotificationAbout(_ newAndUpdatedArticles: NewAndUpdatedArticles) { + var webFeeds = Set() + + if let newArticles = newAndUpdatedArticles.newArticles { + webFeeds.formUnion(Set(newArticles.compactMap { $0.webFeed })) + } + if let updatedArticles = newAndUpdatedArticles.updatedArticles { + webFeeds.formUnion(Set(updatedArticles.compactMap { $0.webFeed })) + } + + var shouldSendNotification = false + var userInfo = [String: Any]() + + if let newArticles = newAndUpdatedArticles.newArticles, !newArticles.isEmpty { + shouldSendNotification = true + userInfo[UserInfoKey.newArticles] = newArticles + self.updateUnreadCounts(for: webFeeds) { + NotificationCenter.default.post(name: .DownloadArticlesDidUpdateUnreadCounts, object: self, userInfo: nil) + } + } + + if let updatedArticles = newAndUpdatedArticles.updatedArticles, !updatedArticles.isEmpty { + shouldSendNotification = true + userInfo[UserInfoKey.updatedArticles] = updatedArticles + } + + if shouldSendNotification { + userInfo[UserInfoKey.webFeeds] = webFeeds + NotificationCenter.default.post(name: .AccountDidDownloadArticles, object: self, userInfo: userInfo) + } + } } // MARK: - Container Overrides diff --git a/Frameworks/Account/Feedly/FeedlyAPICaller.swift b/Frameworks/Account/Feedly/FeedlyAPICaller.swift index dfd101325..d52c83bb8 100644 --- a/Frameworks/Account/Feedly/FeedlyAPICaller.swift +++ b/Frameworks/Account/Feedly/FeedlyAPICaller.swift @@ -302,7 +302,7 @@ final class FeedlyAPICaller { transport.send(request: request, resultType: [FeedlyFeed].self, dateDecoding: .millisecondsSince1970, keyDecoding: .convertFromSnakeCase) { result in switch result { - case .success(let httpResponse, _): + case .success((let httpResponse, _)): if httpResponse.statusCode == 200 { completion(.success(())) } else { @@ -364,7 +364,7 @@ extension FeedlyAPICaller: FeedlyAddFeedToCollectionService { transport.send(request: request, resultType: [FeedlyFeed].self, dateDecoding: .millisecondsSince1970, keyDecoding: .convertFromSnakeCase) { result in switch result { - case .success(_, let collectionFeeds): + case .success((_, let collectionFeeds)): if let feeds = collectionFeeds { completion(.success(feeds)) } else { diff --git a/Frameworks/ArticlesDatabase/ArticlesDatabase.swift b/Frameworks/ArticlesDatabase/ArticlesDatabase.swift index ec94b2e35..4ca73e177 100644 --- a/Frameworks/ArticlesDatabase/ArticlesDatabase.swift +++ b/Frameworks/ArticlesDatabase/ArticlesDatabase.swift @@ -43,14 +43,21 @@ public typealias ArticleStatusesResultBlock = (ArticleStatusesResult) -> Void public final class ArticlesDatabase { + public enum RetentionStyle { + case feedBased // Local and iCloud: article retention is defined by contents of feed + case syncSystem // Feedbin, Feedly, etc.: article retention is defined by external system + } + private let articlesTable: ArticlesTable private let queue: DatabaseQueue private let operationQueue = MainThreadOperationQueue() + private let retentionStyle: RetentionStyle - public init(databaseFilePath: String, accountID: String) { + public init(databaseFilePath: String, accountID: String, retentionStyle: RetentionStyle) { let queue = DatabaseQueue(databasePath: databaseFilePath) self.queue = queue - self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue) + self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue, retentionStyle: retentionStyle) + self.retentionStyle = retentionStyle try! queue.runCreateStatements(ArticlesDatabase.tableCreationStatements) queue.runInDatabase { databaseResult in @@ -62,7 +69,6 @@ public final class ArticlesDatabase { database.executeStatements("DROP TABLE if EXISTS tags;DROP INDEX if EXISTS tags_tagName_index;DROP INDEX if EXISTS articles_feedID_index;DROP INDEX if EXISTS statuses_read_index;DROP TABLE if EXISTS attachments;DROP TABLE if EXISTS attachmentsLookup;") } -// queue.vacuumIfNeeded(daysBetweenVacuums: 9) // TODO: restore this after we do database cleanups. DispatchQueue.main.async { self.articlesTable.indexUnindexedArticles() } @@ -183,8 +189,15 @@ public final class ArticlesDatabase { // MARK: - Saving and Updating Articles - /// Update articles and save new ones. + /// Update articles and save new ones — for feed-based systems (local and iCloud). + public func update(with parsedItems: Set, webFeedID: String, completion: @escaping UpdateArticlesCompletionBlock) { + precondition(retentionStyle == .feedBased) + articlesTable.update(parsedItems, webFeedID, completion) + } + + /// Update articles and save new ones — for sync systems (Feedbin, Feedly, etc.). public func update(webFeedIDsAndItems: [String: Set], defaultRead: Bool, completion: @escaping UpdateArticlesCompletionBlock) { + precondition(retentionStyle == .syncSystem) articlesTable.update(webFeedIDsAndItems, defaultRead, completion) } @@ -219,6 +232,7 @@ public final class ArticlesDatabase { articlesTable.createStatusesIfNeeded(articleIDs, completion) } +#if os(iOS) // MARK: - Suspend and Resume (for iOS) /// Cancel current operations and close the database. @@ -239,7 +253,8 @@ public final class ArticlesDatabase { queue.resume() operationQueue.resume() } - +#endif + // MARK: - Caches /// Call to free up some memory. Should be done when the app is backgrounded, for instance. @@ -254,7 +269,9 @@ public final class ArticlesDatabase { /// Calls the various clean-up functions. public func cleanupDatabaseAtStartup(subscribedToWebFeedIDs: Set) { - articlesTable.deleteOldArticles() + if retentionStyle == .syncSystem { + articlesTable.deleteOldArticles() + } articlesTable.deleteArticlesNotInSubscribedToFeedIDs(subscribedToWebFeedIDs) } } diff --git a/Frameworks/ArticlesDatabase/ArticlesTable.swift b/Frameworks/ArticlesDatabase/ArticlesTable.swift index 35b52d583..9aa7bc06b 100644 --- a/Frameworks/ArticlesDatabase/ArticlesTable.swift +++ b/Frameworks/ArticlesDatabase/ArticlesTable.swift @@ -19,6 +19,8 @@ final class ArticlesTable: DatabaseTable { private let queue: DatabaseQueue private let statusesTable: StatusesTable private let authorsLookupTable: DatabaseLookupTable + private let retentionStyle: ArticlesDatabase.RetentionStyle + private var articlesCache = [String: Article]() private lazy var searchTable: SearchTable = { @@ -30,13 +32,14 @@ final class ArticlesTable: DatabaseTable { private typealias ArticlesFetchMethod = (FMDatabase) -> Set
- init(name: String, accountID: String, queue: DatabaseQueue) { + init(name: String, accountID: String, queue: DatabaseQueue, retentionStyle: ArticlesDatabase.RetentionStyle) { self.name = name self.accountID = accountID self.queue = queue self.statusesTable = StatusesTable(queue: queue) - + self.retentionStyle = retentionStyle + let authorsTable = AuthorsTable(name: DatabaseTableName.authors) self.authorsLookupTable = DatabaseLookupTable(name: DatabaseTableName.authorsLookup, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.authorID, relatedTable: authorsTable, relationshipName: RelationshipName.authors) } @@ -169,7 +172,78 @@ final class ArticlesTable: DatabaseTable { // MARK: - Updating + func update(_ parsedItems: Set, _ webFeedID: String, _ completion: @escaping UpdateArticlesCompletionBlock) { + precondition(retentionStyle == .feedBased) + if parsedItems.isEmpty { + callUpdateArticlesCompletionBlock(nil, nil, completion) + return + } + + // 1. Ensure statuses for all the incoming articles. + // 2. Create incoming articles with parsedItems. + // 3. Ignore incoming articles that are userDeleted + // 4. Fetch all articles for the feed. + // 5. Create array of Articles not in database and save them. + // 6. Create array of updated Articles and save what’s changed. + // 7. Call back with new and updated Articles. + // 8. Delete Articles in database no longer present in the feed. + // 9. Update search index. + + self.queue.runInTransaction { (databaseResult) in + + func makeDatabaseCalls(_ database: FMDatabase) { + let articleIDs = parsedItems.articleIDs() + + let statusesDictionary = self.statusesTable.ensureStatusesForArticleIDs(articleIDs, false, database) //1 + assert(statusesDictionary.count == articleIDs.count) + + let allIncomingArticles = Article.articlesWithParsedItems(parsedItems, webFeedID, self.accountID, statusesDictionary) //2 + let incomingArticles = Set(allIncomingArticles.filter { !($0.status.userDeleted) }) //3 + if incomingArticles.isEmpty { + self.callUpdateArticlesCompletionBlock(nil, nil, completion) + return + } + + let fetchedArticles = self.fetchArticlesForFeedID(webFeedID, withLimits: false, database) //4 + let fetchedArticlesDictionary = fetchedArticles.dictionary() + + let newArticles = self.findAndSaveNewArticles(incomingArticles, fetchedArticlesDictionary, database) //5 + let updatedArticles = self.findAndSaveUpdatedArticles(incomingArticles, fetchedArticlesDictionary, database) //6 + + self.callUpdateArticlesCompletionBlock(newArticles, updatedArticles, completion) //7 + + self.addArticlesToCache(newArticles) + self.addArticlesToCache(updatedArticles) + + // 8. Delete articles no longer in feed. + let articleIDsToDelete = fetchedArticles.articleIDs().filter { !(articleIDs.contains($0)) } + if !articleIDsToDelete.isEmpty { + self.removeArticles(articleIDsToDelete, database) + self.removeArticleIDsFromCache(articleIDsToDelete) + } + + // 9. Update search index. + if let newArticles = newArticles { + self.searchTable.indexNewArticles(newArticles, database) + } + if let updatedArticles = updatedArticles { + self.searchTable.indexUpdatedArticles(updatedArticles, database) + } + } + + switch databaseResult { + case .success(let database): + makeDatabaseCalls(database) + case .failure(let databaseError): + DispatchQueue.main.async { + completion(.failure(databaseError)) + } + } + } + } + func update(_ webFeedIDsAndItems: [String: Set], _ read: Bool, _ completion: @escaping UpdateArticlesCompletionBlock) { + precondition(retentionStyle == .syncSystem) if webFeedIDsAndItems.isEmpty { callUpdateArticlesCompletionBlock(nil, nil, completion) return @@ -850,6 +924,12 @@ private extension ArticlesTable { } } + func removeArticleIDsFromCache(_ articleIDs: Set) { + for articleID in articleIDs { + articlesCache[articleID] = nil + } + } + func articleIsIgnorable(_ article: Article) -> Bool { // Ignorable articles: either userDeleted==1 or (not starred and arrival date > 4 months). if article.status.userDeleted { @@ -863,6 +943,7 @@ private extension ArticlesTable { func filterIncomingArticles(_ articles: Set
) -> Set
{ // Drop Articles that we can ignore. + precondition(retentionStyle == .syncSystem) return Set(articles.filter{ !articleIsIgnorable($0) }) } diff --git a/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift b/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift index 47b88caf7..d3e78c687 100644 --- a/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift +++ b/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift @@ -112,8 +112,12 @@ extension Article { // return Set(parsedItems.map{ Article(parsedItem: $0, maximumDateAllowed: maximumDateAllowed, accountID: accountID, feedID: feedID, status: statusesDictionary[$0.articleID]!) }) // } + private static func _maximumDateAllowed() -> Date { + return Date().addingTimeInterval(60 * 60 * 24) // Allow dates up to about 24 hours ahead of now + } + static func articlesWithWebFeedIDsAndItems(_ webFeedIDsAndItems: [String: Set], _ accountID: String, _ statusesDictionary: [String: ArticleStatus]) -> Set
{ - let maximumDateAllowed = Date().addingTimeInterval(60 * 60 * 24) // Allow dates up to about 24 hours ahead of now + let maximumDateAllowed = _maximumDateAllowed() var feedArticles = Set
() for (webFeedID, parsedItems) in webFeedIDsAndItems { for parsedItem in parsedItems { @@ -124,6 +128,11 @@ extension Article { } return feedArticles } + + static func articlesWithParsedItems(_ parsedItems: Set, _ webFeedID: String, _ accountID: String, _ statusesDictionary: [String: ArticleStatus]) -> Set
{ + let maximumDateAllowed = _maximumDateAllowed() + return Set(parsedItems.map{ Article(parsedItem: $0, maximumDateAllowed: maximumDateAllowed, accountID: accountID, webFeedID: webFeedID, status: statusesDictionary[$0.articleID]!) }) + } } extension Article: DatabaseObject {