Add cleanup code to ArticlesDatabase and ArticlesTable.

Frameworks/ArticlesDatabase/ArticlesDatabase.swift

@@ -22,6 +22,11 @@ public typealias UpdateArticlesCompletionBlock = (Set<Article>?, Set<Article>?)
 
 public final class ArticlesDatabase {
 
+	public enum RetentionStyle {
+		case feedBased // Local and iCloud: article retention is defined by contents of feed
+		case syncSystem // Feedbin, Feedly, etc.: article retention is defined by external system
+	}
+
 	/// When ArticlesDatabase is suspended, database calls will crash the app.
 	public var isSuspended: Bool {
 		return queue.isSuspended
@@ -29,11 +34,13 @@ public final class ArticlesDatabase {
 
 	private let articlesTable: ArticlesTable
 	private let queue: DatabaseQueue
+	private let retentionStyle: RetentionStyle
 
-	public init(databaseFilePath: String, accountID: String) {
+	public init(databaseFilePath: String, accountID: String, retentionStyle: RetentionStyle) {
 		let queue = DatabaseQueue(databasePath: databaseFilePath)
 		self.queue = queue
-		self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue)
+		self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue, retentionStyle: retentionStyle)
+		self.retentionStyle = retentionStyle
 
 		queue.runCreateStatements(ArticlesDatabase.tableCreationStatements)
 		queue.runInDatabase { database in
@@ -175,11 +182,35 @@ public final class ArticlesDatabase {
 
 	// MARK: - Cleanup
 
-	// These are to be used only at startup. These are to prevent the database from growing forever.
+	/// Calls the various clean-up functions. To be used only at startup.
+	///
+	/// This prevents the database from growing forever. If we didn’t do this:
+	/// 1) The database would grow to an inordinate size, and
+	/// 2) the app would become very slow.
+	public func cleanupDatabaseAtStartup(subscribedToWebFeedIDs: Set<String>) {
+		if retentionStyle == .syncSystem {
+			articlesTable.deleteOldArticles()
+		}
+		articlesTable.deleteArticlesNotInSubscribedToFeedIDs(subscribedToWebFeedIDs)
+		articlesTable.deleteOldStatuses()
+	}
 
-	/// Calls the various clean-up functions.
-	public func cleanupDatabaseAtStartup(subscribedToFeedIDs: Set<String>) {
-		articlesTable.deleteArticlesNotInSubscribedToFeedIDs(subscribedToFeedIDs)
+	/// Do database cleanups made necessary by the retention policy change in April 2020.
+	///
+	/// The retention policy for feed-based systems changed in April 2020:
+	/// we keep articles only for as long as they’re in the feed.
+	/// This change could result in a bunch of older articles suddenly
+	/// appearing as unread articles.
+	///
+	/// These are articles that were in the database,
+	/// but weren’t appearing in the UI because they were beyond the 90-day window.
+	/// (The previous retention policy used a 90-day window.)
+	///
+	/// This function marks everything as read that’s beyond that 90-day window.
+	/// It’s intended to be called only once on an account.
+	public func performApril2020RetentionPolicyChange() {
+		precondition(retentionStyle == .feedBased)
+		articlesTable.markOlderStatusesAsRead()
 	}
 }
 

Frameworks/ArticlesDatabase/ArticlesTable.swift

@@ -19,6 +19,7 @@ final class ArticlesTable: DatabaseTable {
 	private let queue: DatabaseQueue
 	private let statusesTable: StatusesTable
 	private let authorsLookupTable: DatabaseLookupTable
+	private let retentionStyle: ArticlesDatabase.RetentionStyle
 	private var databaseArticlesCache = [String: DatabaseArticle]()
 
 	private lazy var searchTable: SearchTable = {
@@ -31,12 +32,13 @@ final class ArticlesTable: DatabaseTable {
 
 	private typealias ArticlesFetchMethod = (FMDatabase) -> Set<Article>
 
-	init(name: String, accountID: String, queue: DatabaseQueue) {
+	init(name: String, accountID: String, queue: DatabaseQueue, retentionStyle: ArticlesDatabase.RetentionStyle) {
 
 		self.name = name
 		self.accountID = accountID
 		self.queue = queue
 		self.statusesTable = StatusesTable(queue: queue)
+		self.retentionStyle = retentionStyle
 
 		let authorsTable = AuthorsTable(name: DatabaseTableName.authors)
 		self.authorsLookupTable = DatabaseLookupTable(name: DatabaseTableName.authorsLookup, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.authorID, relatedTable: authorsTable, relationshipName: RelationshipName.authors)
@@ -422,6 +424,63 @@ final class ArticlesTable: DatabaseTable {
 
 	// MARK: - Cleanup
 
+	/// Delete articles that we won’t show in the UI any longer
+	/// — their arrival date is before our 90-day recency window;
+	/// they are read; they are not starred.
+	///
+	/// Because deleting articles might block the database for too long,
+	/// we do this in a careful way: delete articles older than a year,
+	/// check to see how much time has passed, then decide whether or not to continue.
+	/// Repeat for successively more-recent dates.
+	///
+	/// Returns `true` if it deleted old articles all the way up to the 90 day cutoff date.
+	func deleteOldArticles() {
+		precondition(retentionStyle == .syncSystem)
+
+		queue.runInTransaction { database in
+			func deleteOldArticles(cutoffDate: Date) {
+				let sql = "delete from articles where articleID in (select articleID from articles natural join statuses where dateArrived<? and read=1 and starred=0);"
+				let parameters = [cutoffDate] as [Any]
+				database.executeUpdate(sql, withArgumentsIn: parameters)
+			}
+
+			let startTime = Date()
+			func tooMuchTimeHasPassed() -> Bool {
+				let timeElapsed = Date().timeIntervalSince(startTime)
+				return timeElapsed > 2.0
+			}
+
+			let dayIntervals = [365, 300, 225, 150]
+			for dayInterval in dayIntervals {
+				deleteOldArticles(cutoffDate: startTime.bySubtracting(days: dayInterval))
+				if tooMuchTimeHasPassed() {
+					return
+				}
+			}
+			deleteOldArticles(cutoffDate: self.articleCutoffDate)
+		}
+	}
+
+	/// Delete old statuses.
+	func deleteOldStatuses() {
+		queue.runInTransaction { database in
+			let sql: String
+			let cutoffDate: Date
+
+			switch self.retentionStyle {
+			case .syncSystem:
+				sql = "delete from statuses where dateArrived<? and read=1 and starred=0 and articleID not in (select articleID from articles);"
+				cutoffDate = Date().bySubtracting(days: 180)
+			case .feedBased:
+				sql = "delete from statuses where dateArrived<? and starred=0 and articleID not in (select articleID from articles);"
+				cutoffDate = Date().bySubtracting(days: 30)
+			}
+
+			let parameters = [cutoffDate] as [Any]
+			database.executeUpdate(sql, withArgumentsIn: parameters)
+		}
+	}
+
 	/// Delete articles from feeds that are no longer in the current set of subscribed-to feeds.
 	/// This deletes from the articles and articleStatuses tables,
 	/// and, via a trigger, it also deletes from the search index.
@@ -444,6 +503,18 @@ final class ArticlesTable: DatabaseTable {
 			self.statusesTable.removeStatuses(articleIDs, database)
 		}
 	}
+
+	/// Mark statuses beyond the 90-day window as read.
+	///
+	/// This is not intended for wide use: this is part of implementing
+	/// the April 2020 retention policy change for feed-based accounts.
+	func markOlderStatusesAsRead() {
+		queue.runInTransaction { database in
+			let sql = "update statuses set read = true where dateArrived<?;"
+			let parameters = [self.articleCutoffDate] as [Any]
+			database.executeUpdate(sql, withArgumentsIn: parameters)
+		}
+	}
 }
 
 // MARK: - Private