From cca3089218305429f5a693306b8ef916b23afa2a Mon Sep 17 00:00:00 2001 From: Brent Simmons Date: Fri, 27 Sep 2019 21:56:16 -0700 Subject: [PATCH 1/5] Update RSParser. --- submodules/RSParser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/RSParser b/submodules/RSParser index d3fe846f8..f01129d76 160000 --- a/submodules/RSParser +++ b/submodules/RSParser @@ -1 +1 @@ -Subproject commit d3fe846f8969f8914d233242b711f4314ec1cb17 +Subproject commit f01129d762eba20cd11a680bbde651ca75639ef3 From 07a631309cddff6adf71a9a3dffdb46f5e63bf6d Mon Sep 17 00:00:00 2001 From: Brent Simmons Date: Fri, 27 Sep 2019 22:59:15 -0700 Subject: [PATCH 2/5] Update RSWeb. --- submodules/RSWeb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/RSWeb b/submodules/RSWeb index d140c97a1..9cb7ca961 160000 --- a/submodules/RSWeb +++ b/submodules/RSWeb @@ -1 +1 @@ -Subproject commit d140c97a13799a38fe62c09d1719c3963c3df3ce +Subproject commit 9cb7ca96182b3320882522708a2b4dcdaafb07f6 From d7b45a14137725c9b55dfa5c66e979c3b976b931 Mon Sep 17 00:00:00 2001 From: Brent Simmons Date: Fri, 27 Sep 2019 23:01:31 -0700 Subject: [PATCH 3/5] =?UTF-8?q?Change=20parseDatePublished()=20to=20a=20la?= =?UTF-8?q?zy=20var=20parsedDatePublished=20=E2=80=94=20it=20appeared=20th?= =?UTF-8?q?at=20it=20was=20getting=20called=20more=20than=20once,=20and=20?= =?UTF-8?q?date=20parsing=20is=20expensive.=20Also:=20use=20RSDateWithStri?= =?UTF-8?q?ng=20rather=20than=20an=20NSDateFormatter,=20since=20NSDateForm?= =?UTF-8?q?atter=20is=20so=20massively=20slow.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Feedbin/FeedbinAccountDelegate.swift | 2 +- Frameworks/Account/Feedbin/FeedbinEntry.swift | 28 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Frameworks/Account/Feedbin/FeedbinAccountDelegate.swift b/Frameworks/Account/Feedbin/FeedbinAccountDelegate.swift index f7b801b8c..bbd185238 100644 --- a/Frameworks/Account/Feedbin/FeedbinAccountDelegate.swift +++ b/Frameworks/Account/Feedbin/FeedbinAccountDelegate.swift @@ -1109,7 +1109,7 @@ private extension FeedbinAccountDelegate { let parsedItems: [ParsedItem] = entries.map { entry in let authors = Set([ParsedAuthor(name: entry.authorName, url: entry.jsonFeed?.jsonFeedAuthor?.url, avatarURL: entry.jsonFeed?.jsonFeedAuthor?.avatarURL, emailAddress: nil)]) - return ParsedItem(syncServiceID: String(entry.articleID), uniqueID: String(entry.articleID), feedURL: String(entry.feedID), url: nil, externalURL: entry.url, title: entry.title, contentHTML: entry.contentHTML, contentText: nil, summary: entry.summary, imageURL: nil, bannerImageURL: nil, datePublished: entry.parseDatePublished(), dateModified: nil, authors: authors, tags: nil, attachments: nil) + return ParsedItem(syncServiceID: String(entry.articleID), uniqueID: String(entry.articleID), feedURL: String(entry.feedID), url: nil, externalURL: entry.url, title: entry.title, contentHTML: entry.contentHTML, contentText: nil, summary: entry.summary, imageURL: nil, bannerImageURL: nil, datePublished: entry.parsedDatePublished, dateModified: nil, authors: authors, tags: nil, attachments: nil) } return Set(parsedItems) diff --git a/Frameworks/Account/Feedbin/FeedbinEntry.swift b/Frameworks/Account/Feedbin/FeedbinEntry.swift index 8d73a51da..82b40e186 100644 --- a/Frameworks/Account/Feedbin/FeedbinEntry.swift +++ b/Frameworks/Account/Feedbin/FeedbinEntry.swift @@ -10,7 +10,7 @@ import Foundation import RSParser import RSCore -struct FeedbinEntry: Codable { +final class FeedbinEntry: Codable { let articleID: Int let feedID: Int @@ -23,6 +23,19 @@ struct FeedbinEntry: Codable { let dateArrived: String? let jsonFeed: FeedbinEntryJSONFeed? + // Feedbin dates can't be decoded by the JSONDecoding 8601 decoding strategy. Feedbin + // requires a very specific date formatter to work and even then it fails occasionally. + // Rather than loose all the entries we only lose the one date by decoding as a string + // and letting the one date fail when parsed. + lazy var parsedDatePublished: Date? = { + if let datePublished = datePublished { + return RSDateWithString(datePublished) + } + else { + return nil + } + }() + enum CodingKeys: String, CodingKey { case articleID = "id" case feedID = "feed_id" @@ -35,19 +48,6 @@ struct FeedbinEntry: Codable { case dateArrived = "created_at" case jsonFeed = "json_feed" } - - // Feedbin dates can't be decoded by the JSONDecoding 8601 decoding strategy. Feedbin - // requires a very specific date formatter to work and even then it fails occasionally. - // Rather than loose all the entries we only lose the one date by decoding as a string - // and letting the one date fail when parsed. - func parseDatePublished() -> Date? { - if datePublished != nil { - return FeedbinDate.formatter.date(from: datePublished!) - } else { - return nil - } - } - } struct FeedbinEntryJSONFeed: Codable { From 2b491217f3fbb33b3d03a962bc58eaf3c8f66881 Mon Sep 17 00:00:00 2001 From: Brent Simmons Date: Sat, 28 Sep 2019 12:18:08 -0700 Subject: [PATCH 4/5] =?UTF-8?q?Create=20statusWithRow(=5F=20row:=20FMResul?= =?UTF-8?q?tSet,=20articleID:=20String)=20=E2=80=94=C2=A0it=20allows=20us?= =?UTF-8?q?=20to=20avoid=20pulling=20articleID=20from=20the=20row=20twice?= =?UTF-8?q?=20every=20time=20we=E2=80=99re=20creating=20a=20DatabaseArticl?= =?UTF-8?q?e.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Frameworks/ArticlesDatabase/ArticlesTable.swift | 13 ++++++------- Frameworks/ArticlesDatabase/StatusesTable.swift | 8 ++++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Frameworks/ArticlesDatabase/ArticlesTable.swift b/Frameworks/ArticlesDatabase/ArticlesTable.swift index 3ac4de861..a92354045 100644 --- a/Frameworks/ArticlesDatabase/ArticlesTable.swift +++ b/Frameworks/ArticlesDatabase/ArticlesTable.swift @@ -475,16 +475,15 @@ private extension ArticlesTable { func makeDatabaseArticles(with resultSet: FMResultSet) -> Set { let articles = resultSet.mapToSet { (row) -> DatabaseArticle? in - // The resultSet is a result of a JOIN query with the statuses table, - // so we can get the statuses at the same time and avoid additional database lookups. - - guard let status = statusesTable.statusWithRow(resultSet) else { - assertionFailure("Expected status.") + guard let articleID = row.string(forColumn: DatabaseKey.articleID) else { + assertionFailure("Expected articleID.") return nil } - guard let articleID = row.string(forColumn: DatabaseKey.articleID) else { - assertionFailure("Expected articleID.") + // The resultSet is a result of a JOIN query with the statuses table, + // so we can get the statuses at the same time and avoid additional database lookups. + guard let status = statusesTable.statusWithRow(resultSet, articleID: articleID) else { + assertionFailure("Expected status.") return nil } guard let feedID = row.string(forColumn: DatabaseKey.feedID) else { diff --git a/Frameworks/ArticlesDatabase/StatusesTable.swift b/Frameworks/ArticlesDatabase/StatusesTable.swift index 0bb18377d..fa5a741ab 100644 --- a/Frameworks/ArticlesDatabase/StatusesTable.swift +++ b/Frameworks/ArticlesDatabase/StatusesTable.swift @@ -105,17 +105,21 @@ final class StatusesTable: DatabaseTable { guard let articleID = row.string(forColumn: DatabaseKey.articleID) else { return nil } + return statusWithRow(row, articleID: articleID) + } + + func statusWithRow(_ row: FMResultSet, articleID: String) ->ArticleStatus? { if let cachedStatus = cache[articleID] { return cachedStatus } - + guard let dateArrived = row.date(forColumn: DatabaseKey.dateArrived) else { return nil } let articleStatus = ArticleStatus(articleID: articleID, dateArrived: dateArrived, row: row) cache.addStatusIfNotCached(articleStatus) - + return articleStatus } From 37c9818cad8511b9c507bbdafdca87a7e1bcafeb Mon Sep 17 00:00:00 2001 From: Brent Simmons Date: Sat, 28 Sep 2019 13:51:33 -0700 Subject: [PATCH 5/5] =?UTF-8?q?Create=20and=20use=20a=20cache=20for=20Data?= =?UTF-8?q?baseArticle=20=E2=80=94=20this=20will=20make=20fetches=20faster?= =?UTF-8?q?,=20since=20we=20can=20skip=20pulling=20the=20same=20data=20fro?= =?UTF-8?q?m=20the=20database=20over=20and=20over.=20Articles=20in=20the?= =?UTF-8?q?=20cache=20are=20removed=20when=20articles=20are=20updated,=20s?= =?UTF-8?q?o=20the=20cache=20is=20never=20stale.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ArticlesDatabase/ArticlesTable.swift | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Frameworks/ArticlesDatabase/ArticlesTable.swift b/Frameworks/ArticlesDatabase/ArticlesTable.swift index a92354045..8f06afdc8 100644 --- a/Frameworks/ArticlesDatabase/ArticlesTable.swift +++ b/Frameworks/ArticlesDatabase/ArticlesTable.swift @@ -20,6 +20,7 @@ final class ArticlesTable: DatabaseTable { private let statusesTable: StatusesTable private let authorsLookupTable: DatabaseLookupTable private let attachmentsLookupTable: DatabaseLookupTable + private var databaseArticlesCache = [String: DatabaseArticle]() private lazy var searchTable: SearchTable = { return SearchTable(queue: queue, articlesTable: self) @@ -480,6 +481,12 @@ private extension ArticlesTable { return nil } + // Articles are removed from the cache when they’re updated. + // See saveUpdatedArticles. + if let databaseArticle = databaseArticlesCache[articleID] { + return databaseArticle + } + // The resultSet is a result of a JOIN query with the statuses table, // so we can get the statuses at the same time and avoid additional database lookups. guard let status = statusesTable.statusWithRow(resultSet, articleID: articleID) else { @@ -506,7 +513,9 @@ private extension ArticlesTable { let datePublished = row.date(forColumn: DatabaseKey.datePublished) let dateModified = row.date(forColumn: DatabaseKey.dateModified) - return DatabaseArticle(articleID: articleID, feedID: feedID, uniqueID: uniqueID, title: title, contentHTML: contentHTML, contentText: contentText, url: url, externalURL: externalURL, summary: summary, imageURL: imageURL, bannerImageURL: bannerImageURL, datePublished: datePublished, dateModified: dateModified, status: status) + let databaseArticle = DatabaseArticle(articleID: articleID, feedID: feedID, uniqueID: uniqueID, title: title, contentHTML: contentHTML, contentText: contentText, url: url, externalURL: externalURL, summary: summary, imageURL: imageURL, bannerImageURL: bannerImageURL, datePublished: datePublished, dateModified: dateModified, status: status) + databaseArticlesCache[articleID] = databaseArticle + return databaseArticle } return articles @@ -662,6 +671,7 @@ private extension ArticlesTable { func saveUpdatedArticles(_ updatedArticles: Set
, _ fetchedArticles: [String: Article], _ database: FMDatabase) { + removeArticlesFromDatabaseArticlesCache(updatedArticles) saveUpdatedRelatedObjects(updatedArticles, fetchedArticles, database) for updatedArticle in updatedArticles { @@ -682,10 +692,17 @@ private extension ArticlesTable { // Not unexpected. There may be no changes. return } - + updateRowsWithDictionary(changesDictionary, whereKey: DatabaseKey.articleID, matches: updatedArticle.articleID, database: database) } - + + func removeArticlesFromDatabaseArticlesCache(_ updatedArticles: Set
) { + let articleIDs = updatedArticles.articleIDs() + for articleID in articleIDs { + databaseArticlesCache[articleID] = nil + } + } + func statusIndicatesArticleIsIgnorable(_ status: ArticleStatus) -> Bool { // Ignorable articles: either userDeleted==1 or (not starred and arrival date > 4 months). if status.userDeleted {