From fb121f8a8cdbcb6d18746aebc3eed49029acc0a9 Mon Sep 17 00:00:00 2001 From: Brent Simmons Date: Sat, 2 Sep 2017 16:08:02 -0700 Subject: [PATCH] Make more progress on saving/updating articles. --- Frameworks/Data/DatabaseID.swift | 1 + Frameworks/Database/ArticlesTable.swift | 10 +++- .../Extensions/ParsedItem+Database.swift | 7 +-- Frameworks/Database/StatusesTable.swift | 46 ++++++++++++++++--- Frameworks/RSCore/RSCore/NSString+RSCore.h | 2 +- Frameworks/RSCore/RSCore/NSString+RSCore.m | 4 +- .../RSParser/Feeds/JSON/JSONFeedParser.swift | 2 +- .../RSParser/Feeds/JSON/RSSInJSONParser.swift | 2 +- Frameworks/RSParser/Feeds/ParsedItem.swift | 2 +- .../Feeds/XML/RSParsedFeedTransformer.swift | 2 +- 10 files changed, 61 insertions(+), 17 deletions(-) diff --git a/Frameworks/Data/DatabaseID.swift b/Frameworks/Data/DatabaseID.swift index a7558e184..2d3ae1304 100644 --- a/Frameworks/Data/DatabaseID.swift +++ b/Frameworks/Data/DatabaseID.swift @@ -7,6 +7,7 @@ // import Foundation +import RSCore // MD5 works because: // * It’s fast diff --git a/Frameworks/Database/ArticlesTable.swift b/Frameworks/Database/ArticlesTable.swift index 83afb289c..0815a8885 100644 --- a/Frameworks/Database/ArticlesTable.swift +++ b/Frameworks/Database/ArticlesTable.swift @@ -254,7 +254,15 @@ private extension ArticlesTable { func updateArticles(_ articlesDictionary: [String: Article], _ parsedItemsDictionary: [String: ParsedItem], _ feed: Feed, _ completion: @escaping RSVoidCompletionBlock) { - + let parsedItemArticleIDs = Set(parsedItemsDictionary.keys) + + queue.update { (database) in + + self.statusesTable.ensureStatusesForArticleIDs(parsedItemArticleIDs, database) + + } + + } } diff --git a/Frameworks/Database/Extensions/ParsedItem+Database.swift b/Frameworks/Database/Extensions/ParsedItem+Database.swift index 45dec0d1b..d02c98052 100644 --- a/Frameworks/Database/Extensions/ParsedItem+Database.swift +++ b/Frameworks/Database/Extensions/ParsedItem+Database.swift @@ -8,10 +8,11 @@ import Foundation import RSParser +import Data extension ParsedItem { - private func databaseIdentifierWithFeed(_ feed: Feed) -> String { + func databaseIdentifierWithFeed(_ feed: Feed) -> String { if let identifier = syncServiceID { return identifier @@ -28,8 +29,8 @@ extension ParsedFeed { var d = [String: ParsedItem]() - for parsedItem in parsedItems { - let identifier = identifierForParsedItem(parsedItem, feed) + for parsedItem in items { + let identifier = parsedItem.databaseIdentifierWithFeed(feed) d[identifier] = parsedItem } diff --git a/Frameworks/Database/StatusesTable.swift b/Frameworks/Database/StatusesTable.swift index 08ce8d902..2d37fd4a6 100644 --- a/Frameworks/Database/StatusesTable.swift +++ b/Frameworks/Database/StatusesTable.swift @@ -40,7 +40,7 @@ final class StatusesTable: DatabaseTable { return articleStatus } - // MARK: Creating + // MARK: Creating/Updating func ensureStatusesForArticles(_ articles: Set
, _ database: FMDatabase) { @@ -49,12 +49,32 @@ final class StatusesTable: DatabaseTable { return } - createAndSaveStatusesForArticles(articlesNeedingStatuses, database) + let articleIDs = articlesNeedingStatuses.articleIDs() + ensureStatusesForArticleIDs(articleIDs, database) attachCachedStatuses(articlesNeedingStatuses) assert(articles.eachHasAStatus()) } - + + func ensureStatusesForArticleIDs(_ articleIDs: Set, _ database: FMDatabase) { + + // Check cache. + let articleIDsMissingCachedStatus = articleIDsWithNoCachedStatus(articleIDs) + if articleIDsMissingCachedStatus.isEmpty { + return + } + + // Check database. + fetchAndCacheStatusesForArticleIDs(articleIDsMissingCachedStatus, database) + let articleIDsNeedingStatus = articleIDsWithNoCachedStatus(articleIDs) + if articleIDsNeedingStatus.isEmpty { + return + } + + // Create new statuses. + createAndSaveStatusesForArticleIDs(articleIDsNeedingStatus, database) + } + // MARK: Marking func markArticleIDs(_ articleIDs: Set, _ statusKey: String, _ flag: Bool, _ database: FMDatabase) { @@ -74,6 +94,11 @@ private extension StatusesTable { } } + func articleIDsWithNoCachedStatus(_ articleIDs: Set) -> Set { + + return Set(articleIDs.filter { cache[$0] == nil }) + } + // MARK: Creating func saveStatuses(_ statuses: Set, _ database: FMDatabase) { @@ -82,7 +107,7 @@ private extension StatusesTable { insertRows(statusArray, insertType: .orIgnore, in: database) } - func cacheStatuses(_ statuses: [ArticleStatus]) { + func cacheStatuses(_ statuses: Set) { let databaseObjects = statuses.map { $0 as DatabaseObject } cache.addObjectsNotCached(databaseObjects) @@ -97,10 +122,19 @@ private extension StatusesTable { func createAndSaveStatusesForArticleIDs(_ articleIDs: Set, _ database: FMDatabase) { let now = Date() - let statuses = articleIDs.map { ArticleStatus(articleID: $0, dateArrived: now) } + let statuses = Set(articleIDs.map { ArticleStatus(articleID: $0, dateArrived: now) }) cacheStatuses(statuses) - saveStatuses(Set(statuses), database) + saveStatuses(statuses, database) } + func fetchAndCacheStatusesForArticleIDs(_ articleIDs: Set, _ database: FMDatabase) { + + guard let resultSet = selectRowsWhere(key: DatabaseKey.articleID, inValues: Array(articleIDs), in: database) else { + return + } + + let statuses = resultSet.mapToSet(statusWithRow) + cacheStatuses(statuses) + } } diff --git a/Frameworks/RSCore/RSCore/NSString+RSCore.h b/Frameworks/RSCore/RSCore/NSString+RSCore.h index 3027e702b..1f8e9354c 100755 --- a/Frameworks/RSCore/RSCore/NSString+RSCore.h +++ b/Frameworks/RSCore/RSCore/NSString+RSCore.h @@ -23,7 +23,7 @@ NSString *RSStringReplaceAll(NSString *stringToSearch, NSString *searchFor, NSSt /*The hashed data is a UTF-8 encoded version of the string.*/ -- (NSData *)rs_md5Hash; +- (NSData *)rs_md5HashData; - (NSString *)rs_md5HashString; diff --git a/Frameworks/RSCore/RSCore/NSString+RSCore.m b/Frameworks/RSCore/RSCore/NSString+RSCore.m index ec38c3027..c28a7e9c0 100755 --- a/Frameworks/RSCore/RSCore/NSString+RSCore.m +++ b/Frameworks/RSCore/RSCore/NSString+RSCore.m @@ -45,7 +45,7 @@ NSString *RSStringReplaceAll(NSString *stringToSearch, NSString *searchFor, NSSt @implementation NSString (RSCore) -- (NSData *)rs_md5Hash { +- (NSData *)rs_md5HashData { NSData *data = [self dataUsingEncoding:NSUTF8StringEncoding]; return [data rs_md5Hash]; @@ -54,7 +54,7 @@ NSString *RSStringReplaceAll(NSString *stringToSearch, NSString *searchFor, NSSt - (NSString *)rs_md5HashString { - NSData *d = [self rs_md5Hash]; + NSData *d = [self rs_md5HashData]; return [d rs_hexadecimalString]; } diff --git a/Frameworks/RSParser/Feeds/JSON/JSONFeedParser.swift b/Frameworks/RSParser/Feeds/JSON/JSONFeedParser.swift index 69364686f..0e33c3f2b 100644 --- a/Frameworks/RSParser/Feeds/JSON/JSONFeedParser.swift +++ b/Frameworks/RSParser/Feeds/JSON/JSONFeedParser.swift @@ -114,7 +114,7 @@ private extension JSONFeedParser { let tags = itemDictionary["tags"] as? [String] let attachments = parseAttachments(itemDictionary) - return ParsedItem(uniqueID: uniqueID, feedURL: feedURL, url: url, externalURL: externalURL, title: title, contentHTML: contentHTML, contentText: contentText, summary: summary, imageURL: imageURL, bannerImageURL: bannerImageURL, datePublished: datePublished, dateModified: dateModified, authors: authors, tags: tags, attachments: attachments) + return ParsedItem(syncServiceID: nil, uniqueID: uniqueID, feedURL: feedURL, url: url, externalURL: externalURL, title: title, contentHTML: contentHTML, contentText: contentText, summary: summary, imageURL: imageURL, bannerImageURL: bannerImageURL, datePublished: datePublished, dateModified: dateModified, authors: authors, tags: tags, attachments: attachments) } static func parseUniqueID(_ itemDictionary: JSONDictionary) -> String? { diff --git a/Frameworks/RSParser/Feeds/JSON/RSSInJSONParser.swift b/Frameworks/RSParser/Feeds/JSON/RSSInJSONParser.swift index 4d75f6cf4..3d0a600ce 100644 --- a/Frameworks/RSParser/Feeds/JSON/RSSInJSONParser.swift +++ b/Frameworks/RSParser/Feeds/JSON/RSSInJSONParser.swift @@ -127,7 +127,7 @@ private extension RSSInJSONParser { } if let uniqueID = uniqueID { - return ParsedItem(uniqueID: uniqueID, feedURL: feedURL, url: nil, externalURL: externalURL, title: title, contentHTML: contentHTML, contentText: contentText, summary: nil, imageURL: nil, bannerImageURL: nil, datePublished: datePublished, dateModified: nil, authors: authors, tags: tags, attachments: attachments) + return ParsedItem(syncServiceID: nil, uniqueID: uniqueID, feedURL: feedURL, url: nil, externalURL: externalURL, title: title, contentHTML: contentHTML, contentText: contentText, summary: nil, imageURL: nil, bannerImageURL: nil, datePublished: datePublished, dateModified: nil, authors: authors, tags: tags, attachments: attachments) } return nil } diff --git a/Frameworks/RSParser/Feeds/ParsedItem.swift b/Frameworks/RSParser/Feeds/ParsedItem.swift index a96ca6539..15dea8caf 100644 --- a/Frameworks/RSParser/Feeds/ParsedItem.swift +++ b/Frameworks/RSParser/Feeds/ParsedItem.swift @@ -46,7 +46,7 @@ public struct ParsedItem: Hashable { self.authors = authors self.tags = tags self.attachments = attachments - self.hashValue = articleID.hashValue + self.hashValue = feedURL.hashValue ^ uniqueID.hashValue } public static func ==(lhs: ParsedItem, rhs: ParsedItem) -> Bool { diff --git a/Frameworks/RSParser/Feeds/XML/RSParsedFeedTransformer.swift b/Frameworks/RSParser/Feeds/XML/RSParsedFeedTransformer.swift index aa42b3934..c5d68b40f 100644 --- a/Frameworks/RSParser/Feeds/XML/RSParsedFeedTransformer.swift +++ b/Frameworks/RSParser/Feeds/XML/RSParsedFeedTransformer.swift @@ -46,7 +46,7 @@ private extension RSParsedFeedTransformer { let dateModified = parsedArticle.dateModified let authors = parsedAuthors(parsedArticle.author) - return ParsedItem(uniqueID: uniqueID, feedURL: parsedArticle.feedURL, url: url, externalURL: externalURL, title: title, contentHTML: contentHTML, contentText: nil, summary: nil, imageURL: nil, bannerImageURL: nil, datePublished: datePublished, dateModified: dateModified, authors: authors, tags: nil, attachments: nil) + return ParsedItem(syncServiceID: nil, uniqueID: uniqueID, feedURL: parsedArticle.feedURL, url: url, externalURL: externalURL, title: title, contentHTML: contentHTML, contentText: nil, summary: nil, imageURL: nil, bannerImageURL: nil, datePublished: datePublished, dateModified: dateModified, authors: authors, tags: nil, attachments: nil) } static func parsedAuthors(_ authorEmailAddress: String?) -> [ParsedAuthor]? {