From cd44f28accadeb9721c9ed189682a2c10c845935 Mon Sep 17 00:00:00 2001 From: Brent Simmons Date: Mon, 13 Jul 2020 18:29:30 -0700 Subject: [PATCH] Add cleanup code to ArticlesDatabase and ArticlesTable. --- .../ArticlesDatabase/ArticlesDatabase.swift | 43 +++++++++-- .../ArticlesDatabase/ArticlesTable.swift | 73 ++++++++++++++++++- 2 files changed, 109 insertions(+), 7 deletions(-) diff --git a/Frameworks/ArticlesDatabase/ArticlesDatabase.swift b/Frameworks/ArticlesDatabase/ArticlesDatabase.swift index 4d3b7efc0..717995bce 100644 --- a/Frameworks/ArticlesDatabase/ArticlesDatabase.swift +++ b/Frameworks/ArticlesDatabase/ArticlesDatabase.swift @@ -22,6 +22,11 @@ public typealias UpdateArticlesCompletionBlock = (Set
?, Set
?) public final class ArticlesDatabase { + public enum RetentionStyle { + case feedBased // Local and iCloud: article retention is defined by contents of feed + case syncSystem // Feedbin, Feedly, etc.: article retention is defined by external system + } + /// When ArticlesDatabase is suspended, database calls will crash the app. public var isSuspended: Bool { return queue.isSuspended @@ -29,11 +34,13 @@ public final class ArticlesDatabase { private let articlesTable: ArticlesTable private let queue: DatabaseQueue + private let retentionStyle: RetentionStyle - public init(databaseFilePath: String, accountID: String) { + public init(databaseFilePath: String, accountID: String, retentionStyle: RetentionStyle) { let queue = DatabaseQueue(databasePath: databaseFilePath) self.queue = queue - self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue) + self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue, retentionStyle: retentionStyle) + self.retentionStyle = retentionStyle queue.runCreateStatements(ArticlesDatabase.tableCreationStatements) queue.runInDatabase { database in @@ -175,11 +182,35 @@ public final class ArticlesDatabase { // MARK: - Cleanup - // These are to be used only at startup. These are to prevent the database from growing forever. + /// Calls the various clean-up functions. To be used only at startup. + /// + /// This prevents the database from growing forever. If we didn’t do this: + /// 1) The database would grow to an inordinate size, and + /// 2) the app would become very slow. + public func cleanupDatabaseAtStartup(subscribedToWebFeedIDs: Set) { + if retentionStyle == .syncSystem { + articlesTable.deleteOldArticles() + } + articlesTable.deleteArticlesNotInSubscribedToFeedIDs(subscribedToWebFeedIDs) + articlesTable.deleteOldStatuses() + } - /// Calls the various clean-up functions. - public func cleanupDatabaseAtStartup(subscribedToFeedIDs: Set) { - articlesTable.deleteArticlesNotInSubscribedToFeedIDs(subscribedToFeedIDs) + /// Do database cleanups made necessary by the retention policy change in April 2020. + /// + /// The retention policy for feed-based systems changed in April 2020: + /// we keep articles only for as long as they’re in the feed. + /// This change could result in a bunch of older articles suddenly + /// appearing as unread articles. + /// + /// These are articles that were in the database, + /// but weren’t appearing in the UI because they were beyond the 90-day window. + /// (The previous retention policy used a 90-day window.) + /// + /// This function marks everything as read that’s beyond that 90-day window. + /// It’s intended to be called only once on an account. + public func performApril2020RetentionPolicyChange() { + precondition(retentionStyle == .feedBased) + articlesTable.markOlderStatusesAsRead() } } diff --git a/Frameworks/ArticlesDatabase/ArticlesTable.swift b/Frameworks/ArticlesDatabase/ArticlesTable.swift index 14be4a591..6344a4a0e 100644 --- a/Frameworks/ArticlesDatabase/ArticlesTable.swift +++ b/Frameworks/ArticlesDatabase/ArticlesTable.swift @@ -19,6 +19,7 @@ final class ArticlesTable: DatabaseTable { private let queue: DatabaseQueue private let statusesTable: StatusesTable private let authorsLookupTable: DatabaseLookupTable + private let retentionStyle: ArticlesDatabase.RetentionStyle private var databaseArticlesCache = [String: DatabaseArticle]() private lazy var searchTable: SearchTable = { @@ -31,12 +32,13 @@ final class ArticlesTable: DatabaseTable { private typealias ArticlesFetchMethod = (FMDatabase) -> Set
- init(name: String, accountID: String, queue: DatabaseQueue) { + init(name: String, accountID: String, queue: DatabaseQueue, retentionStyle: ArticlesDatabase.RetentionStyle) { self.name = name self.accountID = accountID self.queue = queue self.statusesTable = StatusesTable(queue: queue) + self.retentionStyle = retentionStyle let authorsTable = AuthorsTable(name: DatabaseTableName.authors) self.authorsLookupTable = DatabaseLookupTable(name: DatabaseTableName.authorsLookup, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.authorID, relatedTable: authorsTable, relationshipName: RelationshipName.authors) @@ -422,6 +424,63 @@ final class ArticlesTable: DatabaseTable { // MARK: - Cleanup + /// Delete articles that we won’t show in the UI any longer + /// — their arrival date is before our 90-day recency window; + /// they are read; they are not starred. + /// + /// Because deleting articles might block the database for too long, + /// we do this in a careful way: delete articles older than a year, + /// check to see how much time has passed, then decide whether or not to continue. + /// Repeat for successively more-recent dates. + /// + /// Returns `true` if it deleted old articles all the way up to the 90 day cutoff date. + func deleteOldArticles() { + precondition(retentionStyle == .syncSystem) + + queue.runInTransaction { database in + func deleteOldArticles(cutoffDate: Date) { + let sql = "delete from articles where articleID in (select articleID from articles natural join statuses where dateArrived Bool { + let timeElapsed = Date().timeIntervalSince(startTime) + return timeElapsed > 2.0 + } + + let dayIntervals = [365, 300, 225, 150] + for dayInterval in dayIntervals { + deleteOldArticles(cutoffDate: startTime.bySubtracting(days: dayInterval)) + if tooMuchTimeHasPassed() { + return + } + } + deleteOldArticles(cutoffDate: self.articleCutoffDate) + } + } + + /// Delete old statuses. + func deleteOldStatuses() { + queue.runInTransaction { database in + let sql: String + let cutoffDate: Date + + switch self.retentionStyle { + case .syncSystem: + sql = "delete from statuses where dateArrived