diff --git a/Frameworks/Articles/Article.swift b/Frameworks/Articles/Article.swift index 1ecabe1cc..c54bb6624 100644 --- a/Frameworks/Articles/Article.swift +++ b/Frameworks/Articles/Article.swift @@ -23,13 +23,12 @@ public struct Article: Hashable { public let externalURL: String? public let summary: String? public let imageURL: String? - public let bannerImageURL: String? public let datePublished: Date? public let dateModified: Date? public let authors: Set? public let status: ArticleStatus - public init(accountID: String, articleID: String?, feedID: String, uniqueID: String, title: String?, contentHTML: String?, contentText: String?, url: String?, externalURL: String?, summary: String?, imageURL: String?, bannerImageURL: String?, datePublished: Date?, dateModified: Date?, authors: Set?, status: ArticleStatus) { + public init(accountID: String, articleID: String?, feedID: String, uniqueID: String, title: String?, contentHTML: String?, contentText: String?, url: String?, externalURL: String?, summary: String?, imageURL: String?, datePublished: Date?, dateModified: Date?, authors: Set?, status: ArticleStatus) { self.accountID = accountID self.feedID = feedID @@ -41,7 +40,6 @@ public struct Article: Hashable { self.externalURL = externalURL self.summary = summary self.imageURL = imageURL - self.bannerImageURL = bannerImageURL self.datePublished = datePublished self.dateModified = dateModified self.authors = authors diff --git a/Frameworks/ArticlesDatabase/ArticlesDatabase.swift b/Frameworks/ArticlesDatabase/ArticlesDatabase.swift index 57519be58..0b1a3355e 100644 --- a/Frameworks/ArticlesDatabase/ArticlesDatabase.swift +++ b/Frameworks/ArticlesDatabase/ArticlesDatabase.swift @@ -18,7 +18,14 @@ import Articles public typealias UnreadCountDictionary = [String: Int] // feedID: unreadCount public typealias UnreadCountCompletionBlock = (UnreadCountDictionary) -> Void -public typealias UpdateArticlesCompletionBlock = (Set
?, Set
?) -> Void //newArticles, updatedArticles + +public struct ArticleChanges { + public let newArticles: Set
? + public let updatedArticles: Set
? + public let deletedArticles: Set
? +} +public typealias UpdateArticlesCompletionBlock = (ArticleChanges) -> Void + public final class ArticlesDatabase { diff --git a/Frameworks/ArticlesDatabase/ArticlesDatabase.xcodeproj/project.pbxproj b/Frameworks/ArticlesDatabase/ArticlesDatabase.xcodeproj/project.pbxproj index d19d892fe..0da269edf 100644 --- a/Frameworks/ArticlesDatabase/ArticlesDatabase.xcodeproj/project.pbxproj +++ b/Frameworks/ArticlesDatabase/ArticlesDatabase.xcodeproj/project.pbxproj @@ -11,7 +11,6 @@ 841D4D742106B59F00DD04E6 /* Articles.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 841D4D732106B59F00DD04E6 /* Articles.framework */; }; 84288A001F6A3C4400395871 /* DatabaseObject+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842889FF1F6A3C4400395871 /* DatabaseObject+Database.swift */; }; 84288A021F6A3D8000395871 /* RelatedObjectsMap+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84288A011F6A3D8000395871 /* RelatedObjectsMap+Database.swift */; }; - 843577161F744FC800F460AE /* DatabaseArticle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 843577151F744FC800F460AE /* DatabaseArticle.swift */; }; 843577221F749C6200F460AE /* ArticleChangesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 843577211F749C6200F460AE /* ArticleChangesTests.swift */; }; 843702C31F70D15D00B18807 /* ParsedArticle+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 843702C21F70D15D00B18807 /* ParsedArticle+Database.swift */; }; 843CB9961F34174100EE6581 /* Author+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84F20F901F1810DD00D8E682 /* Author+Database.swift */; }; @@ -114,7 +113,6 @@ 841D4D732106B59F00DD04E6 /* Articles.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = Articles.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 842889FF1F6A3C4400395871 /* DatabaseObject+Database.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DatabaseObject+Database.swift"; sourceTree = ""; }; 84288A011F6A3D8000395871 /* RelatedObjectsMap+Database.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "RelatedObjectsMap+Database.swift"; sourceTree = ""; }; - 843577151F744FC800F460AE /* DatabaseArticle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DatabaseArticle.swift; sourceTree = ""; }; 843577211F749C6200F460AE /* ArticleChangesTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ArticleChangesTests.swift; sourceTree = ""; }; 843702C21F70D15D00B18807 /* ParsedArticle+Database.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = "ParsedArticle+Database.swift"; path = "Extensions/ParsedArticle+Database.swift"; sourceTree = ""; }; 844BEE371F0AB3AA004AB7CD /* ArticlesDatabase.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = ArticlesDatabase.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -175,7 +173,6 @@ 845580661F0AEBCD003CCFA1 /* Constants.swift */, 84E156EB1F0AB80E00F8CC05 /* ArticlesTable.swift */, 8477ACBB2221E76F00DF7F37 /* SearchTable.swift */, - 843577151F744FC800F460AE /* DatabaseArticle.swift */, 84E156ED1F0AB81400F8CC05 /* StatusesTable.swift */, 84F20F8E1F180D8700D8E682 /* AuthorsTable.swift */, 8461462A1F0AC44100870CB3 /* Extensions */, @@ -498,7 +495,6 @@ 84F20F8F1F180D8700D8E682 /* AuthorsTable.swift in Sources */, 84288A001F6A3C4400395871 /* DatabaseObject+Database.swift in Sources */, 8477ACBC2221E76F00DF7F37 /* SearchTable.swift in Sources */, - 843577161F744FC800F460AE /* DatabaseArticle.swift in Sources */, 843702C31F70D15D00B18807 /* ParsedArticle+Database.swift in Sources */, 84E156EC1F0AB80E00F8CC05 /* ArticlesTable.swift in Sources */, 84E156EE1F0AB81400F8CC05 /* StatusesTable.swift in Sources */, diff --git a/Frameworks/ArticlesDatabase/ArticlesTable.swift b/Frameworks/ArticlesDatabase/ArticlesTable.swift index a708abbc0..98290d149 100644 --- a/Frameworks/ArticlesDatabase/ArticlesTable.swift +++ b/Frameworks/ArticlesDatabase/ArticlesTable.swift @@ -20,7 +20,8 @@ final class ArticlesTable: DatabaseTable { private let statusesTable: StatusesTable private let authorsLookupTable: DatabaseLookupTable private let retentionStyle: ArticlesDatabase.RetentionStyle - private var databaseArticlesCache = [String: DatabaseArticle]() + + private var articlesCache = [String: Article]() private lazy var searchTable: SearchTable = { return SearchTable(queue: queue, articlesTable: self) @@ -28,7 +29,6 @@ final class ArticlesTable: DatabaseTable { // TODO: update articleCutoffDate as time passes and based on user preferences. let articleCutoffDate = Date().bySubtracting(days: 90) - private var maximumArticleCutoffDate = Date().bySubtracting(days: 4 * 31) private typealias ArticlesFetchMethod = (FMDatabase) -> Set
@@ -47,15 +47,11 @@ final class ArticlesTable: DatabaseTable { // MARK: - Fetching Articles for Feed func fetchArticles(_ feedID: String) -> Set
{ - return fetchArticles{ self.fetchArticlesForFeedID(feedID, withLimits: true, $0) } + return fetchArticles{ self.fetchArticlesForFeedID(feedID, $0) } } func fetchArticlesAsync(_ feedID: String, _ callback: @escaping ArticleSetBlock) { - fetchArticlesAsync({ self.fetchArticlesForFeedID(feedID, withLimits: true, $0) }, callback) - } - - private func fetchArticlesForFeedID(_ feedID: String, withLimits: Bool, _ database: FMDatabase) -> Set
{ - return fetchArticlesWithWhereClause(database, whereClause: "articles.feedID = ?", parameters: [feedID as AnyObject], withLimits: withLimits) + fetchArticlesAsync({ self.fetchArticlesForFeedID(feedID, $0) }, callback) } // MARK: - Fetching Articles by articleID @@ -68,16 +64,6 @@ final class ArticlesTable: DatabaseTable { return fetchArticlesAsync({ self.fetchArticles(articleIDs: articleIDs, $0) }, callback) } - private func fetchArticles(articleIDs: Set, _ database: FMDatabase) -> Set
{ - if articleIDs.isEmpty { - return Set
() - } - let parameters = articleIDs.map { $0 as AnyObject } - let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(articleIDs.count))! - let whereClause = "articleID in \(placeholders)" - return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters, withLimits: false) - } - // MARK: - Fetching Unread Articles func fetchUnreadArticles(_ feedIDs: Set) -> Set
{ @@ -88,17 +74,6 @@ final class ArticlesTable: DatabaseTable { fetchArticlesAsync({ self.fetchUnreadArticles(feedIDs, $0) }, callback) } - private func fetchUnreadArticles(_ feedIDs: Set, _ database: FMDatabase) -> Set
{ - // select * from articles natural join statuses where feedID in ('http://ranchero.com/xml/rss.xml') and read=0 - if feedIDs.isEmpty { - return Set
() - } - let parameters = feedIDs.map { $0 as AnyObject } - let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(feedIDs.count))! - let whereClause = "feedID in \(placeholders) and read=0" - return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters, withLimits: true) - } - // MARK: - Fetching Today Articles func fetchArticlesSince(_ feedIDs: Set, _ cutoffDate: Date) -> Set
{ @@ -109,19 +84,6 @@ final class ArticlesTable: DatabaseTable { fetchArticlesAsync({ self.fetchArticlesSince(feedIDs, cutoffDate, $0) }, callback) } - private func fetchArticlesSince(_ feedIDs: Set, _ cutoffDate: Date, _ database: FMDatabase) -> Set
{ - // select * from articles natural join statuses where feedID in ('http://ranchero.com/xml/rss.xml') and (datePublished > ? || (datePublished is null and dateArrived > ?) - // - // datePublished may be nil, so we fall back to dateArrived. - if feedIDs.isEmpty { - return Set
() - } - let parameters = feedIDs.map { $0 as AnyObject } + [cutoffDate as AnyObject, cutoffDate as AnyObject] - let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(feedIDs.count))! - let whereClause = "feedID in \(placeholders) and (datePublished > ? or (datePublished is null and dateArrived > ?))" - return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters, withLimits: false) - } - // MARK: - Fetching Starred Articles func fetchStarredArticles(_ feedIDs: Set) -> Set
{ @@ -132,17 +94,6 @@ final class ArticlesTable: DatabaseTable { fetchArticlesAsync({ self.fetchStarredArticles(feedIDs, $0) }, callback) } - private func fetchStarredArticles(_ feedIDs: Set, _ database: FMDatabase) -> Set
{ - // select * from articles natural join statuses where feedID in ('http://ranchero.com/xml/rss.xml') and starred = 1; - if feedIDs.isEmpty { - return Set
() - } - let parameters = feedIDs.map { $0 as AnyObject } - let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(feedIDs.count))! - let whereClause = "feedID in \(placeholders) and starred = 1" - return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters, withLimits: false) - } - // MARK: - Fetching Search Articles func fetchArticlesMatching(_ searchString: String, _ feedIDs: Set) -> Set
{ @@ -173,7 +124,7 @@ final class ArticlesTable: DatabaseTable { let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(searchRowIDs.count))! let whereClause = "searchRowID in \(placeholders)" let parameters: [AnyObject] = Array(searchRowIDs) as [AnyObject] - let articles = fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters, withLimits: true) + let articles = fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters) // TODO: include the feedIDs in the SQL rather than filtering here. return articles.filter{ feedIDs.contains($0.feedID) } } @@ -207,9 +158,74 @@ final class ArticlesTable: DatabaseTable { // MARK: - Updating + func update(_ parsedItems: Set, _ webFeedID: String, _ completion: @escaping UpdateArticlesCompletionBlock) { + precondition(retentionStyle == .feedBased) + if parsedItems.isEmpty { + callUpdateArticlesCompletionBlock(nil, nil, nil, completion) + return + } + + // 1. Ensure statuses for all the incoming articles. + // 2. Create incoming articles with parsedItems. + // 3. [Deleted - this step is no longer needed] + // 4. Fetch all articles for the feed. + // 5. Create array of Articles not in database and save them. + // 6. Create array of updated Articles and save what’s changed. + // 7. Call back with new and updated Articles. + // 8. Delete Articles in database no longer present in the feed. + // 9. Update search index. + + self.queue.runInTransaction { database in + + let articleIDs = parsedItems.articleIDs() + + let statusesDictionary = self.statusesTable.ensureStatusesForArticleIDs(articleIDs, false, database) //1 + assert(statusesDictionary.count == articleIDs.count) + + let incomingArticles = Article.articlesWithParsedItems(parsedItems, webFeedID, self.accountID, statusesDictionary) //2 + if incomingArticles.isEmpty { + self.callUpdateArticlesCompletionBlock(nil, nil, nil, completion) + return + } + + let fetchedArticles = self.fetchArticlesForFeedID(webFeedID, database) //4 + let fetchedArticlesDictionary = fetchedArticles.dictionary() + + let newArticles = self.findAndSaveNewArticles(incomingArticles, fetchedArticlesDictionary, database) //5 + let updatedArticles = self.findAndSaveUpdatedArticles(incomingArticles, fetchedArticlesDictionary, database) //6 + + // Articles to delete are 1) not starred and 2) older than 30 days and 3) no longer in feed. + let cutoffDate = Date().bySubtracting(days: 30) + let articlesToDelete = fetchedArticles.filter { (article) -> Bool in + return !article.status.starred && article.status.dateArrived < cutoffDate && !articleIDs.contains(article.articleID) + } + + self.callUpdateArticlesCompletionBlock(newArticles, updatedArticles, articlesToDelete, completion) //7 + + self.addArticlesToCache(newArticles) + self.addArticlesToCache(updatedArticles) + + // 8. Delete articles no longer in feed. + let articleIDsToDelete = articlesToDelete.articleIDs() + if !articleIDsToDelete.isEmpty { + self.removeArticles(articleIDsToDelete, database) + self.removeArticleIDsFromCache(articleIDsToDelete) + } + + // 9. Update search index. + if let newArticles = newArticles { + self.searchTable.indexNewArticles(newArticles, database) + } + if let updatedArticles = updatedArticles { + self.searchTable.indexUpdatedArticles(updatedArticles, database) + } + } + } + func update(_ feedIDsAndItems: [String: Set], _ read: Bool, _ completion: @escaping UpdateArticlesCompletionBlock) { + precondition(retentionStyle == .syncSystem) if feedIDsAndItems.isEmpty { - completion(nil, nil) + callUpdateArticlesCompletionBlock(nil, nil, nil, completion) return } @@ -233,13 +249,13 @@ final class ArticlesTable: DatabaseTable { let allIncomingArticles = Article.articlesWithFeedIDsAndItems(feedIDsAndItems, self.accountID, statusesDictionary) //2 if allIncomingArticles.isEmpty { - self.callUpdateArticlesCompletionBlock(nil, nil, completion) + self.callUpdateArticlesCompletionBlock(nil, nil, nil, completion) return } let incomingArticles = self.filterIncomingArticles(allIncomingArticles) //3 if incomingArticles.isEmpty { - self.callUpdateArticlesCompletionBlock(nil, nil, completion) + self.callUpdateArticlesCompletionBlock(nil, nil, nil, completion) return } @@ -250,22 +266,17 @@ final class ArticlesTable: DatabaseTable { let newArticles = self.findAndSaveNewArticles(incomingArticles, fetchedArticlesDictionary, database) //5 let updatedArticles = self.findAndSaveUpdatedArticles(incomingArticles, fetchedArticlesDictionary, database) //6 - self.callUpdateArticlesCompletionBlock(newArticles, updatedArticles, completion) //7 + self.callUpdateArticlesCompletionBlock(newArticles, updatedArticles, nil, completion) //7 + + self.addArticlesToCache(newArticles) + self.addArticlesToCache(updatedArticles) // 8. Update search index. - var articlesToIndex = Set
() if let newArticles = newArticles { - articlesToIndex.formUnion(newArticles) + self.searchTable.indexNewArticles(newArticles, database) } if let updatedArticles = updatedArticles { - articlesToIndex.formUnion(updatedArticles) - } - let articleIDsToIndex = articlesToIndex.articleIDs() - if articleIDsToIndex.isEmpty { - return - } - DispatchQueue.main.async { - self.searchTable.ensureIndexedArticles(for: articleIDsToIndex) + self.searchTable.indexUpdatedArticles(updatedArticles, database) } } } @@ -277,7 +288,7 @@ final class ArticlesTable: DatabaseTable { self.statusesTable.mark(statuses, statusKey, flag, database) } } - + // MARK: - Unread Counts func fetchUnreadCounts(_ feedIDs: Set, _ completion: @escaping UnreadCountCompletionBlock) { @@ -330,7 +341,7 @@ final class ArticlesTable: DatabaseTable { let cutoffDate = articleCutoffDate queue.runInDatabase { (database) in - let sql = "select distinct feedID, count(*) from articles natural join statuses where read=0 and (starred=1 or dateArrived>?) group by feedID;" + let sql = "select distinct feedID, count(*) from articles natural join statuses where read=0 group by feedID;" guard let resultSet = database.executeQuery(sql, withArgumentsIn: [cutoffDate]) else { DispatchQueue.main.async { @@ -418,7 +429,7 @@ final class ArticlesTable: DatabaseTable { func emptyCaches() { queue.runInDatabase { _ in - self.databaseArticlesCache = [String: DatabaseArticle]() + self.articlesCache = [String: Article]() } } @@ -541,110 +552,64 @@ private extension ArticlesTable { } func articlesWithResultSet(_ resultSet: FMResultSet, _ database: FMDatabase) -> Set
{ - // 1. Create DatabaseArticles without related objects. - // 2. Then fetch the related objects, given the set of articleIDs. - // 3. Then create set of Articles with DatabaseArticles and related objects and return it. + var cachedArticles = Set
() + var fetchedArticles = Set
() - // 1. Create databaseArticles (intermediate representations). + while resultSet.next() { - let databaseArticles = makeDatabaseArticles(with: resultSet) - if databaseArticles.isEmpty { - return Set
() - } - - let articleIDs = databaseArticles.articleIDs() - - // 2. Fetch related objects. - - let authorsMap = authorsLookupTable.fetchRelatedObjects(for: articleIDs, in: database) - - // 3. Create articles with related objects. - - let articles = databaseArticles.map { (databaseArticle) -> Article in - return articleWithDatabaseArticle(databaseArticle, authorsMap) - } - - return Set(articles) - } - - func articleWithDatabaseArticle(_ databaseArticle: DatabaseArticle, _ authorsMap: RelatedObjectsMap?) -> Article { - - let articleID = databaseArticle.articleID - let authors = authorsMap?.authors(for: articleID) - - return Article(databaseArticle: databaseArticle, accountID: accountID, authors: authors) - } - - func makeDatabaseArticles(with resultSet: FMResultSet) -> Set { - let articles = resultSet.mapToSet { (row) -> DatabaseArticle? in - - guard let articleID = row.string(forColumn: DatabaseKey.articleID) else { + guard let articleID = resultSet.string(forColumn: DatabaseKey.articleID) else { assertionFailure("Expected articleID.") - return nil + continue } - // Articles are removed from the cache when they’re updated. - // See saveUpdatedArticles. - if let databaseArticle = databaseArticlesCache[articleID] { - return databaseArticle + if let article = articlesCache[articleID] { + cachedArticles.insert(article) + continue } // The resultSet is a result of a JOIN query with the statuses table, // so we can get the statuses at the same time and avoid additional database lookups. guard let status = statusesTable.statusWithRow(resultSet, articleID: articleID) else { assertionFailure("Expected status.") - return nil - } - guard let feedID = row.string(forColumn: DatabaseKey.feedID) else { - assertionFailure("Expected feedID.") - return nil - } - guard let uniqueID = row.string(forColumn: DatabaseKey.uniqueID) else { - assertionFailure("Expected uniqueID.") - return nil + continue } - let title = row.string(forColumn: DatabaseKey.title) - let contentHTML = row.string(forColumn: DatabaseKey.contentHTML) - let contentText = row.string(forColumn: DatabaseKey.contentText) - let url = row.string(forColumn: DatabaseKey.url) - let externalURL = row.string(forColumn: DatabaseKey.externalURL) - let summary = row.string(forColumn: DatabaseKey.summary) - let imageURL = row.string(forColumn: DatabaseKey.imageURL) - let bannerImageURL = row.string(forColumn: DatabaseKey.bannerImageURL) - let datePublished = row.date(forColumn: DatabaseKey.datePublished) - let dateModified = row.date(forColumn: DatabaseKey.dateModified) + guard let article = Article(accountID: accountID, row: resultSet, status: status) else { + continue + } + fetchedArticles.insert(article) + } + resultSet.close() - let databaseArticle = DatabaseArticle(articleID: articleID, feedID: feedID, uniqueID: uniqueID, title: title, contentHTML: contentHTML, contentText: contentText, url: url, externalURL: externalURL, summary: summary, imageURL: imageURL, bannerImageURL: bannerImageURL, datePublished: datePublished, dateModified: dateModified, status: status) - databaseArticlesCache[articleID] = databaseArticle - return databaseArticle + if fetchedArticles.isEmpty { + return cachedArticles } - return articles + // Fetch authors for non-cached articles. (Articles from the cache already have authors.) + let fetchedArticleIDs = fetchedArticles.articleIDs() + let authorsMap = authorsLookupTable.fetchRelatedObjects(for: fetchedArticleIDs, in: database) + let articlesWithFetchedAuthors = fetchedArticles.map { (article) -> Article in + if let authors = authorsMap?.authors(for: article.articleID) { + return article.byAdding(authors) + } + return article + } + + // Add fetchedArticles to cache, now that they have attached authors. + for article in articlesWithFetchedAuthors { + articlesCache[article.articleID] = article + } + + return cachedArticles.union(articlesWithFetchedAuthors) } - func fetchArticlesWithWhereClause(_ database: FMDatabase, whereClause: String, parameters: [AnyObject], withLimits: Bool) -> Set
{ - // Don’t fetch articles that shouldn’t appear in the UI. The rules: - // * Must not be deleted. - // * Must be either 1) starred or 2) dateArrived must be newer than cutoff date. - - if withLimits { - let sql = "select * from articles natural join statuses where \(whereClause) and (starred=1 or dateArrived>?);" - return articlesWithSQL(sql, parameters + [articleCutoffDate as AnyObject], database) - } - else { - let sql = "select * from articles natural join statuses where \(whereClause);" - return articlesWithSQL(sql, parameters, database) - } + func fetchArticlesWithWhereClause(_ database: FMDatabase, whereClause: String, parameters: [AnyObject]) -> Set
{ + let sql = "select * from articles natural join statuses where \(whereClause);" + return articlesWithSQL(sql, parameters, database) } func fetchUnreadCount(_ feedID: String, _ database: FMDatabase) -> Int { - // Count only the articles that would appear in the UI. - // * Must be unread. - // * Must not be deleted. - // * Must be either 1) starred or 2) dateArrived must be newer than cutoff date. - - let sql = "select count(*) from articles natural join statuses where feedID=? and read=0 and (starred=1 or dateArrived>?);" + let sql = "select count(*) from articles natural join statuses where feedID=? and read=0;" return numberWithSQLAndParameters(sql, [feedID, articleCutoffDate], in: database) } @@ -663,7 +628,7 @@ private extension ArticlesTable { let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(searchRowIDs.count))! let whereClause = "searchRowID in \(placeholders)" let parameters: [AnyObject] = Array(searchRowIDs) as [AnyObject] - return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters, withLimits: true) + return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters) } func sqliteSearchString(with searchString: String) -> String { @@ -688,14 +653,64 @@ private extension ArticlesTable { return articlesWithResultSet(resultSet, database) } + func fetchUnreadArticles(_ feedIDs: Set, _ database: FMDatabase) -> Set
{ + // select * from articles natural join statuses where feedID in ('http://ranchero.com/xml/rss.xml') and read=0 + if feedIDs.isEmpty { + return Set
() + } + let parameters = feedIDs.map { $0 as AnyObject } + let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(feedIDs.count))! + let whereClause = "feedID in \(placeholders) and read=0" + return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters) + } + + func fetchArticlesForFeedID(_ feedID: String, _ database: FMDatabase) -> Set
{ + return fetchArticlesWithWhereClause(database, whereClause: "articles.feedID = ?", parameters: [feedID as AnyObject]) + } + + func fetchArticles(articleIDs: Set, _ database: FMDatabase) -> Set
{ + if articleIDs.isEmpty { + return Set
() + } + let parameters = articleIDs.map { $0 as AnyObject } + let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(articleIDs.count))! + let whereClause = "articleID in \(placeholders)" + return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters) + } + + func fetchArticlesSince(_ feedIDs: Set, _ cutoffDate: Date, _ database: FMDatabase) -> Set
{ + // select * from articles natural join statuses where feedID in ('http://ranchero.com/xml/rss.xml') and (datePublished > ? || (datePublished is null and dateArrived > ?) + // + // datePublished may be nil, so we fall back to dateArrived. + if feedIDs.isEmpty { + return Set
() + } + let parameters = feedIDs.map { $0 as AnyObject } + [cutoffDate as AnyObject, cutoffDate as AnyObject] + let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(feedIDs.count))! + let whereClause = "feedID in \(placeholders) and (datePublished > ? or (datePublished is null and dateArrived > ?))" + return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters) + } + + func fetchStarredArticles(_ feedIDs: Set, _ database: FMDatabase) -> Set
{ + // select * from articles natural join statuses where feedID in ('http://ranchero.com/xml/rss.xml') and starred = 1; + if feedIDs.isEmpty { + return Set
() + } + let parameters = feedIDs.map { $0 as AnyObject } + let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(feedIDs.count))! + let whereClause = "feedID in \(placeholders) and starred=1" + return fetchArticlesWithWhereClause(database, whereClause: whereClause, parameters: parameters) + } + // MARK: - Saving Parsed Items - func callUpdateArticlesCompletionBlock(_ newArticles: Set
?, _ updatedArticles: Set
?, _ completion: @escaping UpdateArticlesCompletionBlock) { + func callUpdateArticlesCompletionBlock(_ newArticles: Set
?, _ updatedArticles: Set
?, _ deletedArticles: Set
?, _ completion: @escaping UpdateArticlesCompletionBlock) { + let articleChanges = ArticleChanges(newArticles: newArticles, updatedArticles: updatedArticles, deletedArticles: deletedArticles) DispatchQueue.main.async { - completion(newArticles, updatedArticles) + completion(articleChanges) } } - + // MARK: - Saving New Articles func findNewArticles(_ incomingArticles: Set
, _ fetchedArticlesDictionary: [String: Article]) -> Set
? { @@ -770,7 +785,6 @@ private extension ArticlesTable { func saveUpdatedArticles(_ updatedArticles: Set
, _ fetchedArticles: [String: Article], _ database: FMDatabase) { - removeArticlesFromDatabaseArticlesCache(updatedArticles) saveUpdatedRelatedObjects(updatedArticles, fetchedArticles, database) for updatedArticle in updatedArticles { @@ -795,24 +809,32 @@ private extension ArticlesTable { updateRowsWithDictionary(changesDictionary, whereKey: DatabaseKey.articleID, matches: updatedArticle.articleID, database: database) } - func removeArticlesFromDatabaseArticlesCache(_ updatedArticles: Set
) { - let articleIDs = updatedArticles.articleIDs() - for articleID in articleIDs { - databaseArticlesCache[articleID] = nil + func addArticlesToCache(_ articles: Set
?) { + guard let articles = articles else { + return + } + for article in articles { + articlesCache[article.articleID] = article } } - func statusIndicatesArticleIsIgnorable(_ status: ArticleStatus) -> Bool { - // Ignorable articles: not starred and arrival date > 4 months. - if status.starred { + func removeArticleIDsFromCache(_ articleIDs: Set) { + for articleID in articleIDs { + articlesCache[articleID] = nil + } + } + + func articleIsIgnorable(_ article: Article) -> Bool { + if article.status.starred || !article.status.read { return false } - return status.dateArrived < maximumArticleCutoffDate + return article.status.dateArrived < articleCutoffDate } func filterIncomingArticles(_ articles: Set
) -> Set
{ // Drop Articles that we can ignore. - return Set(articles.filter{ !statusIndicatesArticleIsIgnorable($0.status) }) + precondition(retentionStyle == .syncSystem) + return Set(articles.filter{ !articleIsIgnorable($0) }) } func removeArticles(_ articleIDs: Set, _ database: FMDatabase) { diff --git a/Frameworks/ArticlesDatabase/DatabaseArticle.swift b/Frameworks/ArticlesDatabase/DatabaseArticle.swift deleted file mode 100644 index 7ac09c90c..000000000 --- a/Frameworks/ArticlesDatabase/DatabaseArticle.swift +++ /dev/null @@ -1,44 +0,0 @@ -// -// DatabaseArticle.swift -// NetNewsWire -// -// Created by Brent Simmons on 9/21/17. -// Copyright © 2017 Ranchero Software. All rights reserved. -// - -import Foundation -import Articles - -// Intermediate representation of an Article. Doesn’t include related objects. -// Used by ArticlesTable as part of fetching articles. - -struct DatabaseArticle: Hashable { - - let articleID: String - let feedID: String - let uniqueID: String - let title: String? - let contentHTML: String? - let contentText: String? - let url: String? - let externalURL: String? - let summary: String? - let imageURL: String? - let bannerImageURL: String? - let datePublished: Date? - let dateModified: Date? - let status: ArticleStatus - - // MARK: - Hashable - - public func hash(into hasher: inout Hasher) { - hasher.combine(articleID) - } -} - -extension Set where Element == DatabaseArticle { - - func articleIDs() -> Set { - return Set(map { $0.articleID }) - } -} diff --git a/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift b/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift index 3b9b43898..2b206a901 100644 --- a/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift +++ b/Frameworks/ArticlesDatabase/Extensions/Article+Database.swift @@ -13,8 +13,31 @@ import RSParser extension Article { - init(databaseArticle: DatabaseArticle, accountID: String, authors: Set?) { - self.init(accountID: accountID, articleID: databaseArticle.articleID, feedID: databaseArticle.feedID, uniqueID: databaseArticle.uniqueID, title: databaseArticle.title, contentHTML: databaseArticle.contentHTML, contentText: databaseArticle.contentText, url: databaseArticle.url, externalURL: databaseArticle.externalURL, summary: databaseArticle.summary, imageURL: databaseArticle.imageURL, bannerImageURL: databaseArticle.bannerImageURL, datePublished: databaseArticle.datePublished, dateModified: databaseArticle.dateModified, authors: authors, status: databaseArticle.status) + init?(accountID: String, row: FMResultSet, status: ArticleStatus) { + guard let articleID = row.string(forColumn: DatabaseKey.articleID) else { + assertionFailure("Expected articleID.") + return nil + } + guard let webFeedID = row.string(forColumn: DatabaseKey.feedID) else { + assertionFailure("Expected feedID.") + return nil + } + guard let uniqueID = row.string(forColumn: DatabaseKey.uniqueID) else { + assertionFailure("Expected uniqueID.") + return nil + } + + let title = row.string(forColumn: DatabaseKey.title) + let contentHTML = row.string(forColumn: DatabaseKey.contentHTML) + let contentText = row.string(forColumn: DatabaseKey.contentText) + let url = row.string(forColumn: DatabaseKey.url) + let externalURL = row.string(forColumn: DatabaseKey.externalURL) + let summary = row.string(forColumn: DatabaseKey.summary) + let imageURL = row.string(forColumn: DatabaseKey.imageURL) + let datePublished = row.date(forColumn: DatabaseKey.datePublished) + let dateModified = row.date(forColumn: DatabaseKey.dateModified) + + self.init(accountID: accountID, articleID: articleID, feedID: webFeedID, uniqueID: uniqueID, title: title, contentHTML: contentHTML, contentText: contentText, url: url, externalURL: externalURL, summary: summary, imageURL: imageURL, datePublished: datePublished, dateModified: dateModified, authors: nil, status: status) } init(parsedItem: ParsedItem, maximumDateAllowed: Date, accountID: String, feedID: String, status: ArticleStatus) { @@ -34,7 +57,7 @@ extension Article { dateModified = nil } - self.init(accountID: accountID, articleID: parsedItem.syncServiceID, feedID: feedID, uniqueID: parsedItem.uniqueID, title: parsedItem.title, contentHTML: parsedItem.contentHTML, contentText: parsedItem.contentText, url: parsedItem.url, externalURL: parsedItem.externalURL, summary: parsedItem.summary, imageURL: parsedItem.imageURL, bannerImageURL: parsedItem.bannerImageURL, datePublished: datePublished, dateModified: dateModified, authors: authors, status: status) + self.init(accountID: accountID, articleID: parsedItem.syncServiceID, feedID: feedID, uniqueID: parsedItem.uniqueID, title: parsedItem.title, contentHTML: parsedItem.contentHTML, contentText: parsedItem.contentText, url: parsedItem.url, externalURL: parsedItem.externalURL, summary: parsedItem.summary, imageURL: parsedItem.imageURL, datePublished: datePublished, dateModified: dateModified, authors: authors, status: status) } private func addPossibleStringChangeWithKeyPath(_ comparisonKeyPath: KeyPath, _ otherArticle: Article, _ key: String, _ dictionary: inout DatabaseDictionary) { @@ -43,6 +66,13 @@ extension Article { } } + func byAdding(_ authors: Set) -> Article { + if authors.isEmpty { + return self + } + return Article(accountID: self.accountID, articleID: self.articleID, feedID: self.feedID, uniqueID: self.uniqueID, title: self.title, contentHTML: self.contentHTML, contentText: self.contentText, url: self.url, externalURL: self.externalURL, summary: self.summary, imageURL: self.imageURL, datePublished: self.datePublished, dateModified: self.dateModified, authors: authors, status: self.status) + } + func changesFrom(_ existingArticle: Article) -> DatabaseDictionary? { if self == existingArticle { return nil @@ -60,7 +90,6 @@ extension Article { addPossibleStringChangeWithKeyPath(\Article.externalURL, existingArticle, DatabaseKey.externalURL, &d) addPossibleStringChangeWithKeyPath(\Article.summary, existingArticle, DatabaseKey.summary, &d) addPossibleStringChangeWithKeyPath(\Article.imageURL, existingArticle, DatabaseKey.imageURL, &d) - addPossibleStringChangeWithKeyPath(\Article.bannerImageURL, existingArticle, DatabaseKey.bannerImageURL, &d) // If updated versions of dates are nil, and we have existing dates, keep the existing dates. // This is data that’s good to have, and it’s likely that a feed removing dates is doing so in error. @@ -78,10 +107,9 @@ extension Article { return d.count < 1 ? nil : d } -// static func articlesWithParsedItems(_ parsedItems: Set, _ accountID: String, _ feedID: String, _ statusesDictionary: [String: ArticleStatus]) -> Set
{ -// let maximumDateAllowed = Date().addingTimeInterval(60 * 60 * 24) // Allow dates up to about 24 hours ahead of now -// return Set(parsedItems.map{ Article(parsedItem: $0, maximumDateAllowed: maximumDateAllowed, accountID: accountID, feedID: feedID, status: statusesDictionary[$0.articleID]!) }) -// } + private static func _maximumDateAllowed() -> Date { + return Date().addingTimeInterval(60 * 60 * 24) // Allow dates up to about 24 hours ahead of now + } static func articlesWithFeedIDsAndItems(_ feedIDsAndItems: [String: Set], _ accountID: String, _ statusesDictionary: [String: ArticleStatus]) -> Set
{ let maximumDateAllowed = Date().addingTimeInterval(60 * 60 * 24) // Allow dates up to about 24 hours ahead of now @@ -92,6 +120,11 @@ extension Article { } return articles } + + static func articlesWithParsedItems(_ parsedItems: Set, _ feedID: String, _ accountID: String, _ statusesDictionary: [String: ArticleStatus]) -> Set
{ + let maximumDateAllowed = _maximumDateAllowed() + return Set(parsedItems.map{ Article(parsedItem: $0, maximumDateAllowed: maximumDateAllowed, accountID: accountID, feedID: feedID, status: statusesDictionary[$0.articleID]!) }) + } } extension Article: DatabaseObject { @@ -124,9 +157,6 @@ extension Article: DatabaseObject { if let imageURL = imageURL { d[DatabaseKey.imageURL] = imageURL } - if let bannerImageURL = bannerImageURL { - d[DatabaseKey.bannerImageURL] = bannerImageURL - } if let datePublished = datePublished { d[DatabaseKey.datePublished] = datePublished }