From 175cd0e7987724a16962a3265ab3ad36fc7d5f16 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Fri, 13 Mar 2020 18:18:47 -0400 Subject: [PATCH] Parse articles from story river --- .../Account/Account.xcodeproj/project.pbxproj | 8 +++ .../NewsBlur/Models/NewsBlurArticle.swift | 55 ++++++++--------- .../NewsBlur/Models/NewsBlurDate.swift | 20 +++++++ .../Models/NewsBlurUnreadArticle.swift | 50 ++++++++++++++++ .../Account/NewsBlur/NewsBlurAPICaller.swift | 34 ++++++++--- .../NewsBlur/NewsBlurAccountDelegate.swift | 60 ++++++++++++++++++- 6 files changed, 186 insertions(+), 41 deletions(-) create mode 100644 Frameworks/Account/NewsBlur/Models/NewsBlurDate.swift create mode 100644 Frameworks/Account/NewsBlur/Models/NewsBlurUnreadArticle.swift diff --git a/Frameworks/Account/Account.xcodeproj/project.pbxproj b/Frameworks/Account/Account.xcodeproj/project.pbxproj index 2ae45d270..83dd24f7e 100644 --- a/Frameworks/Account/Account.xcodeproj/project.pbxproj +++ b/Frameworks/Account/Account.xcodeproj/project.pbxproj @@ -10,6 +10,8 @@ 179DB02FFBC17AC9798F0EBC /* NewsBlurArticle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB7399814F6FB3247825C /* NewsBlurArticle.swift */; }; 179DB28CF49F73A945EBF5DB /* NewsBlurLoginResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB088236E3236010462E8 /* NewsBlurLoginResponse.swift */; }; 179DB49A960F8B78C4924458 /* NewsBlurGenericCodingKeys.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB66D933E976C29159DEE /* NewsBlurGenericCodingKeys.swift */; }; + 179DB61D33CD8DC94C90F7ED /* NewsBlurDate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DBB17C42E6E434EDC29FA /* NewsBlurDate.swift */; }; + 179DBED55C9B4D6A413486C1 /* NewsBlurUnreadArticle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB818180A51098A9816B2 /* NewsBlurUnreadArticle.swift */; }; 179DBF4DE2562D4C532F6008 /* NewsBlurSubscription.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB1B909672E0E807B5E8C /* NewsBlurSubscription.swift */; }; 3B3A33E7238D3D6800314204 /* Secrets.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B3A33E6238D3D6800314204 /* Secrets.swift */; }; 3B826DA72385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B826D9E2385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift */; }; @@ -230,6 +232,8 @@ 179DB1B909672E0E807B5E8C /* NewsBlurSubscription.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurSubscription.swift; sourceTree = ""; }; 179DB66D933E976C29159DEE /* NewsBlurGenericCodingKeys.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurGenericCodingKeys.swift; sourceTree = ""; }; 179DB7399814F6FB3247825C /* NewsBlurArticle.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurArticle.swift; sourceTree = ""; }; + 179DB818180A51098A9816B2 /* NewsBlurUnreadArticle.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurUnreadArticle.swift; sourceTree = ""; }; + 179DBB17C42E6E434EDC29FA /* NewsBlurDate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurDate.swift; sourceTree = ""; }; 3B3A33E6238D3D6800314204 /* Secrets.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Secrets.swift; path = ../../Shared/Secrets.swift; sourceTree = ""; }; 3B826D9E2385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedWranglerAuthorizationResult.swift; sourceTree = ""; }; 3B826D9F2385C81C00FC1ADB /* FeedWranglerFeedItem.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedWranglerFeedItem.swift; sourceTree = ""; }; @@ -454,6 +458,8 @@ 179DB1B909672E0E807B5E8C /* NewsBlurSubscription.swift */, 179DB7399814F6FB3247825C /* NewsBlurArticle.swift */, 179DB66D933E976C29159DEE /* NewsBlurGenericCodingKeys.swift */, + 179DB818180A51098A9816B2 /* NewsBlurUnreadArticle.swift */, + 179DBB17C42E6E434EDC29FA /* NewsBlurDate.swift */, ); path = Models; sourceTree = ""; @@ -1147,6 +1153,8 @@ 179DBF4DE2562D4C532F6008 /* NewsBlurSubscription.swift in Sources */, 179DB02FFBC17AC9798F0EBC /* NewsBlurArticle.swift in Sources */, 179DB49A960F8B78C4924458 /* NewsBlurGenericCodingKeys.swift in Sources */, + 179DBED55C9B4D6A413486C1 /* NewsBlurUnreadArticle.swift in Sources */, + 179DB61D33CD8DC94C90F7ED /* NewsBlurDate.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Frameworks/Account/NewsBlur/Models/NewsBlurArticle.swift b/Frameworks/Account/NewsBlur/Models/NewsBlurArticle.swift index 8f4f88986..39a1d54bf 100644 --- a/Frameworks/Account/NewsBlur/Models/NewsBlurArticle.swift +++ b/Frameworks/Account/NewsBlur/Models/NewsBlurArticle.swift @@ -10,41 +10,36 @@ import Foundation import RSCore import RSParser -typealias NewsBlurArticleHash = NewsBlurUnreadArticleHashesResponse.ArticleHash +typealias NewsBlurArticle = NewsBlurArticlesResponse.Article -struct NewsBlurUnreadArticleHashesResponse: Decodable { - let subscriptions: [String: [ArticleHash]] +struct NewsBlurArticlesResponse: Decodable { + let articles: [Article] - struct ArticleHash: Hashable, Codable { - var hash: String - var timestamp: Date + struct Article: Decodable { + let articleId: String + let feedId: Int + let title: String? + let url: String? + let authorName: String? + let contentHTML: String? + let datePublished: Date } } -extension NewsBlurUnreadArticleHashesResponse { +extension NewsBlurArticlesResponse { private enum CodingKeys: String, CodingKey { - case feeds = "unread_feed_story_hashes" - } - - init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - - // Parse subscriptions - var subscriptions: [String: [ArticleHash]] = [:] - let subscriptionContainer = try container.nestedContainer(keyedBy: NewsBlurGenericCodingKeys.self, forKey: .feeds) - try subscriptionContainer.allKeys.forEach { key in - subscriptions[key.stringValue] = [] - var hashArrayContainer = try subscriptionContainer.nestedUnkeyedContainer(forKey: key) - while !hashArrayContainer.isAtEnd { - var hashContainer = try hashArrayContainer.nestedUnkeyedContainer() - let hash = try hashContainer.decode(String.self) - let timestamp = try hashContainer.decode(Date.self) - let articleHash = ArticleHash(hash: hash, timestamp: timestamp) - - subscriptions[key.stringValue]?.append(articleHash) - } - } - - self.subscriptions = subscriptions + case articles = "stories" + } +} + +extension NewsBlurArticlesResponse.Article { + private enum CodingKeys: String, CodingKey { + case articleId = "story_hash" + case feedId = "story_feed_id" + case title = "story_title" + case url = "story_permalink" + case authorName = "story_authors" + case contentHTML = "story_content" + case datePublished = "story_date" } } diff --git a/Frameworks/Account/NewsBlur/Models/NewsBlurDate.swift b/Frameworks/Account/NewsBlur/Models/NewsBlurDate.swift new file mode 100644 index 000000000..d7731fbce --- /dev/null +++ b/Frameworks/Account/NewsBlur/Models/NewsBlurDate.swift @@ -0,0 +1,20 @@ +// +// NewsBlurDate.swift +// Account +// +// Created by Anh Quang Do on 2020-03-13. +// Copyright (c) 2020 Ranchero Software, LLC. All rights reserved. +// + +import Foundation + +struct NewsBlurDate { + static let yyyyMMddHHmmss: DateFormatter = { + let formatter = DateFormatter() + formatter.calendar = Calendar(identifier: .iso8601) + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(abbreviation: "GMT") + formatter.dateFormat = "yyyy-MM-dd HH:mm:ss" + return formatter + }() +} diff --git a/Frameworks/Account/NewsBlur/Models/NewsBlurUnreadArticle.swift b/Frameworks/Account/NewsBlur/Models/NewsBlurUnreadArticle.swift new file mode 100644 index 000000000..ad316d0a8 --- /dev/null +++ b/Frameworks/Account/NewsBlur/Models/NewsBlurUnreadArticle.swift @@ -0,0 +1,50 @@ +// +// NewsBlurUnreadArticle.swift +// Account +// +// Created by Anh Quang Do on 2020-03-13. +// Copyright (c) 2020 Ranchero Software, LLC. All rights reserved. +// + +import Foundation +import RSCore +import RSParser + +typealias NewsBlurArticleHash = NewsBlurUnreadArticleHashesResponse.ArticleHash + +struct NewsBlurUnreadArticleHashesResponse: Decodable { + let subscriptions: [String: [ArticleHash]] + + struct ArticleHash: Hashable, Codable { + var hash: String + var timestamp: Date + } +} + +extension NewsBlurUnreadArticleHashesResponse { + private enum CodingKeys: String, CodingKey { + case feeds = "unread_feed_story_hashes" + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + // Parse subscriptions + var subscriptions: [String: [ArticleHash]] = [:] + let subscriptionContainer = try container.nestedContainer(keyedBy: NewsBlurGenericCodingKeys.self, forKey: .feeds) + try subscriptionContainer.allKeys.forEach { key in + subscriptions[key.stringValue] = [] + var hashArrayContainer = try subscriptionContainer.nestedUnkeyedContainer(forKey: key) + while !hashArrayContainer.isAtEnd { + var hashContainer = try hashArrayContainer.nestedUnkeyedContainer() + let hash = try hashContainer.decode(String.self) + let timestamp = try hashContainer.decode(Date.self) + let articleHash = ArticleHash(hash: hash, timestamp: timestamp) + + subscriptions[key.stringValue]?.append(articleHash) + } + } + + self.subscriptions = subscriptions + } +} diff --git a/Frameworks/Account/NewsBlur/NewsBlurAPICaller.swift b/Frameworks/Account/NewsBlur/NewsBlurAPICaller.swift index a24cbc741..c9b5e2c48 100644 --- a/Frameworks/Account/NewsBlur/NewsBlurAPICaller.swift +++ b/Frameworks/Account/NewsBlur/NewsBlurAPICaller.swift @@ -84,7 +84,7 @@ final class NewsBlurAPICaller: NSObject { } } - func retrieveSubscriptions(completion: @escaping (Result<[NewsBlurSubscription], Error>) -> Void) { + func retrieveSubscriptions(completion: @escaping (Result<[NewsBlurSubscription]?, Error>) -> Void) { let url = baseURL .appendingPathComponent("reader/feeds") .appendingQueryItems([ @@ -101,14 +101,14 @@ final class NewsBlurAPICaller: NSObject { transport.send(request: request, resultType: NewsBlurFeedsResponse.self) { result in switch result { case .success((_, let payload)): - completion(.success(payload?.subscriptions ?? [])) + completion(.success(payload?.subscriptions)) case .failure(let error): completion(.failure(error)) } } } - func retrieveUnreadArticleHashes(completion: @escaping (Result<[NewsBlurArticleHash], Error>) -> Void) { + func retrieveUnreadArticleHashes(completion: @escaping (Result<[NewsBlurArticleHash]?, Error>) -> Void) { let url = baseURL .appendingPathComponent("reader/unread_story_hashes") .appendingQueryItems([ @@ -124,11 +124,29 @@ final class NewsBlurAPICaller: NSObject { transport.send(request: request, resultType: NewsBlurUnreadArticleHashesResponse.self, dateDecoding: .secondsSince1970) { result in switch result { case .success((_, let payload)): - guard let subscriptions = payload?.subscriptions else { - completion(.success([])) - return - } - completion(.success(subscriptions.values.flatMap { $0 })) + completion(.success(payload?.subscriptions.values.flatMap { $0 })) + case .failure(let error): + completion(.failure(error)) + } + } + } + + func retrieveArticles(hashes: [NewsBlurArticleHash], completion: @escaping (Result<[NewsBlurArticle]?, Error>) -> Void) { + let url = baseURL + .appendingPathComponent("reader/river_stories") + .appendingQueryItem(.init(name: "include_hidden", value: "true"))? + .appendingQueryItems(hashes.map { URLQueryItem(name: "h", value: $0.hash) }) + + guard let callURL = url else { + completion(.failure(TransportError.noURL)) + return + } + + let request = URLRequest(url: callURL, credentials: credentials) + transport.send(request: request, resultType: NewsBlurArticlesResponse.self, dateDecoding: .formatted(NewsBlurDate.yyyyMMddHHmmss)) { result in + switch result { + case .success((_, let payload)): + completion(.success(payload?.articles)) case .failure(let error): completion(.failure(error)) } diff --git a/Frameworks/Account/NewsBlur/NewsBlurAccountDelegate.swift b/Frameworks/Account/NewsBlur/NewsBlurAccountDelegate.swift index 203400bbf..2b08ee7b3 100644 --- a/Frameworks/Account/NewsBlur/NewsBlurAccountDelegate.swift +++ b/Frameworks/Account/NewsBlur/NewsBlurAccountDelegate.swift @@ -123,15 +123,50 @@ final class NewsBlurAccountDelegate: AccountDelegate { completion(.success(())) } - func refreshArticles(for account: Account, completion: @escaping (Result<[NewsBlurArticleHash], Error>) -> Void) { + func refreshArticles(for account: Account, completion: @escaping (Result) -> Void) { os_log(.debug, log: log, "Refreshing articles...") + os_log(.debug, log: log, "Refreshing unread articles...") caller.retrieveUnreadArticleHashes { result in switch result { case .success(let articleHashes): - print(articleHashes) + self.refreshProgress.completeTask() + + self.refreshUnreadArticles(for: account, hashes: articleHashes, updateFetchDate: nil, completion: completion) case .failure(let error): - break + completion(.failure(error)) + } + } + } + + func refreshUnreadArticles(for account: Account, hashes: [NewsBlurArticleHash]?, updateFetchDate: Date?, completion: @escaping (Result) -> Void) { + guard let hashes = hashes, !hashes.isEmpty else { + if let lastArticleFetch = updateFetchDate { + self.accountMetadata?.lastArticleFetchStartTime = lastArticleFetch + self.accountMetadata?.lastArticleFetchEndTime = Date() + } + completion(.success(())) + return + } + + let numberOfArticles = min(hashes.count, 100) // api limit + let hashesToFetch = Array(hashes[..)-> Void) { completion(.success(())) } + + func processArticles(account: Account, articles: [NewsBlurArticle]?, completion: @escaping DatabaseCompletionBlock) { + let parsedItems = mapArticlesToParsedItems(articles: articles) + let webFeedIDsAndItems = Dictionary(grouping: parsedItems, by: { item in item.feedURL } ).mapValues { Set($0) } + account.update(webFeedIDsAndItems: webFeedIDsAndItems, defaultRead: true, completion: completion) + } + + func mapArticlesToParsedItems(articles: [NewsBlurArticle]?) -> Set { + guard let articles = articles else { + return Set() + } + + let parsedItems: [ParsedItem] = articles.map { article in + let author = Set([ParsedAuthor(name: article.authorName, url: nil, avatarURL: nil, emailAddress: nil)]) + return ParsedItem(syncServiceID: article.articleId, uniqueID: String(article.articleId), feedURL: String(article.feedId), url: article.url, externalURL: nil, title: article.title, contentHTML: article.contentHTML, contentText: nil, summary: nil, imageURL: nil, bannerImageURL: nil, datePublished: article.datePublished, dateModified: nil, authors: author, tags: nil, attachments: nil) + } + + return Set(parsedItems) + } func importOPML(for account: Account, opmlFile: URL, completion: @escaping (Result) -> ()) { completion(.success(()))