From 9144ee71e53b0a982a1ec581183e4b3917f5fb93 Mon Sep 17 00:00:00 2001 From: Jeremy Beker Date: Mon, 10 Jun 2019 16:53:35 -0400 Subject: [PATCH] Request article IDs and content. --- .../GoogleReaderCompatibleAPICaller.swift | 230 +++++++++++++++--- ...oogleReaderCompatibleAccountDelegate.swift | 43 ++-- .../GoogleReaderCompatibleEntry.swift | 121 ++++++--- .../GoogleReaderCompatibleUnreadEntry.swift | 16 +- submodules/RSWeb | 2 +- 5 files changed, 314 insertions(+), 98 deletions(-) diff --git a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAPICaller.swift b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAPICaller.swift index e8e206e83..d1bc13c56 100644 --- a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAPICaller.swift +++ b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAPICaller.swift @@ -108,6 +108,36 @@ final class GoogleReaderCompatibleAPICaller: NSObject { } + func requestAuthorizationToken(endpoint: URL, completion: @escaping (Result) -> Void) { + guard let credentials = credentials else { + completion(.failure(CredentialsError.incompleteCredentials)) + return + } + + let request = URLRequest(url: endpoint.appendingPathComponent("/reader/api/0/token"), credentials: credentials) + + transport.send(request: request) { result in + switch result { + case .success(let (_, data)): + guard let resultData = data else { + completion(.failure(TransportError.noData)) + break + } + + // Convert the return data to UTF8 and then parse out the Auth token + guard let rawData = String(data: resultData, encoding: .utf8) else { + completion(.failure(TransportError.noData)) + break + } + + + completion(.success(rawData)) + case .failure(let error): + completion(.failure(error)) + } + } + } + func importOPML(opmlData: Data, completion: @escaping (Result) -> Void) { let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("imports.json") @@ -412,24 +442,51 @@ final class GoogleReaderCompatibleAPICaller: NSObject { return } - let concatIDs = articleIDs.reduce("") { param, articleID in return param + ",\(articleID)" } - let paramIDs = String(concatIDs.dropFirst()) + guard let baseURL = APIBaseURL else { + completion(.failure(CredentialsError.incompleteCredentials)) + return + } - var callComponents = URLComponents(url: GoogleReaderCompatibleBaseURL.appendingPathComponent("entries.json"), resolvingAgainstBaseURL: false)! - callComponents.queryItems = [URLQueryItem(name: "ids", value: paramIDs), URLQueryItem(name: "mode", value: "extended")] - let request = URLRequest(url: callComponents.url!, credentials: credentials) - - transport.send(request: request, resultType: [GoogleReaderCompatibleEntry].self) { result in - + self.requestAuthorizationToken(endpoint: baseURL) { (result) in switch result { - case .success(let (_, entries)): - completion(.success((entries))) + case .success(let token): + // Do POST asking for data about all the new articles + var request = URLRequest(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/contents"), credentials: self.credentials) + request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") + request.httpMethod = "POST" + + // Get ids from above into hex representation of value + let idsToFetch = articleIDs.map({ (reference) -> String in + return "i=\(reference)" + }).joined(separator:"&") + + let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8) + //let postData = "T=\(token)&output=json&i=1349530380539369".data(using: String.Encoding.utf8) + + self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: GoogleReaderCompatibleEntryWrapper.self, completion: { (result) in + switch result { + case .success(let (response, entryWrapper)): + guard let entryWrapper = entryWrapper else { + completion(.failure(GoogleReaderCompatibleAccountDelegateError.invalidResponse)) + return + } + + let dateInfo = HTTPDateInfo(urlResponse: response) + self.accountMetadata?.lastArticleFetch = dateInfo?.date + + + completion(.success((entryWrapper.entries))) + case .failure(let error): + completion(.failure(error)) + } + }) + + case .failure(let error): completion(.failure(error)) } - } - + } func retrieveEntries(feedID: String, completion: @escaping (Result<([GoogleReaderCompatibleEntry]?, String?), Error>) -> Void) { @@ -459,30 +516,96 @@ final class GoogleReaderCompatibleAPICaller: NSObject { func retrieveEntries(completion: @escaping (Result<([GoogleReaderCompatibleEntry]?, String?, Int?), Error>) -> Void) { + guard let baseURL = APIBaseURL else { + completion(.failure(CredentialsError.incompleteCredentials)) + return + } + let since: Date = { - if let lastArticleFetch = accountMetadata?.lastArticleFetch { + if let lastArticleFetch = self.accountMetadata?.lastArticleFetch { return lastArticleFetch } else { return Calendar.current.date(byAdding: .month, value: -3, to: Date()) ?? Date() } }() - let sinceString = GoogleReaderCompatibleDate.formatter.string(from: since) - var callComponents = URLComponents(url: GoogleReaderCompatibleBaseURL.appendingPathComponent("entries.json"), resolvingAgainstBaseURL: false)! - callComponents.queryItems = [URLQueryItem(name: "since", value: sinceString), URLQueryItem(name: "per_page", value: "100"), URLQueryItem(name: "mode", value: "extended")] - let request = URLRequest(url: callComponents.url!, credentials: credentials) + let sinceString = since.timeIntervalSince1970 - transport.send(request: request, resultType: [GoogleReaderCompatibleEntry].self) { result in + // Add query string for getting JSON (probably should break this out as I will be doing it a lot) + guard var components = URLComponents(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/ids"), resolvingAgainstBaseURL: false) else { + completion(.failure(TransportError.noURL)) + return + } + + components.queryItems = [ + URLQueryItem(name: "o", value: String(sinceString)), + URLQueryItem(name: "n", value: "10000"), + URLQueryItem(name: "output", value: "json"), + URLQueryItem(name: "xt", value: "user/-/state/com.google/read"), + URLQueryItem(name: "s", value: "user/-/state/com.google/reading-list") + ] + + guard let callURL = components.url else { + completion(.failure(TransportError.noURL)) + return + } + + let conditionalGet = accountMetadata?.conditionalGetInfo[ConditionalGetKeys.unreadEntries] + let request = URLRequest(url: callURL, credentials: credentials, conditionalGet: conditionalGet) + + self.transport.send(request: request, resultType: GoogleReaderCompatibleReferenceWrapper.self) { result in switch result { - case .success(let (response, entries)): + case .success(let (_, entries)): - let dateInfo = HTTPDateInfo(urlResponse: response) - self.accountMetadata?.lastArticleFetch = dateInfo?.date + guard let entries = entries else { + completion(.failure(GoogleReaderCompatibleAccountDelegateError.invalidResponse)) + return + } + + self.requestAuthorizationToken(endpoint: baseURL) { (result) in + switch result { + case .success(let token): + // Do POST asking for data about all the new articles + var request = URLRequest(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/contents"), credentials: self.credentials) + request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") + request.httpMethod = "POST" + + // Get ids from above into hex representation of value + let idsToFetch = entries.itemRefs.map({ (reference) -> String in + let idValue = Int(reference.itemId)! + let idHexString = String(idValue, radix: 16, uppercase: false) + return "i=\(idHexString)" + }).joined(separator:"&") + + let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8) + //let postData = "T=\(token)&output=json&i=1349530380539369".data(using: String.Encoding.utf8) - let pagingInfo = HTTPLinkPagingInfo(urlResponse: response) - let lastPageNumber = self.extractPageNumber(link: pagingInfo.lastPage) - completion(.success((entries, pagingInfo.nextPage, lastPageNumber))) + self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: GoogleReaderCompatibleEntryWrapper.self, completion: { (result) in + switch result { + case .success(let (response, entryWrapper)): + guard let entryWrapper = entryWrapper else { + completion(.failure(GoogleReaderCompatibleAccountDelegateError.invalidResponse)) + return + } + + let dateInfo = HTTPDateInfo(urlResponse: response) + self.accountMetadata?.lastArticleFetch = dateInfo?.date + + + completion(.success((entryWrapper.entries, nil, nil))) + case .failure(let error): + completion(.failure(error)) + } + }) + + + case .failure(let error): + completion(.failure(error)) + } + } + + //completion(.success((entries, pagingInfo.nextPage, lastPageNumber))) case .failure(let error): self.accountMetadata?.lastArticleFetch = nil @@ -491,6 +614,13 @@ final class GoogleReaderCompatibleAPICaller: NSObject { } + + + + + + + } func retrieveEntries(page: String, completion: @escaping (Result<([GoogleReaderCompatibleEntry]?, String?), Error>) -> Void) { @@ -522,16 +652,46 @@ final class GoogleReaderCompatibleAPICaller: NSObject { func retrieveUnreadEntries(completion: @escaping (Result<[Int]?, Error>) -> Void) { - let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json") + guard let baseURL = APIBaseURL else { + completion(.failure(CredentialsError.incompleteCredentials)) + return + } + + // Add query string for getting JSON (probably should break this out as I will be doing it a lot) + guard var components = URLComponents(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/ids"), resolvingAgainstBaseURL: false) else { + completion(.failure(TransportError.noURL)) + return + } + + components.queryItems = [ + URLQueryItem(name: "s", value: "user/-/state/com.google/reading-list"), + URLQueryItem(name: "n", value: "10000"), + URLQueryItem(name: "xt", value: "user/-/state/com.google/read"), + URLQueryItem(name: "output", value: "json") + ] + + guard let callURL = components.url else { + completion(.failure(TransportError.noURL)) + return + } + let conditionalGet = accountMetadata?.conditionalGetInfo[ConditionalGetKeys.unreadEntries] let request = URLRequest(url: callURL, credentials: credentials, conditionalGet: conditionalGet) - transport.send(request: request, resultType: [Int].self) { result in + transport.send(request: request, resultType: GoogleReaderCompatibleReferenceWrapper.self) { result in switch result { case .success(let (response, unreadEntries)): + + guard let itemRefs = unreadEntries?.itemRefs else { + completion(.success([])) + return + } + + let itemIds = itemRefs.map{ Int($0.itemId)! } + self.storeConditionalGet(key: ConditionalGetKeys.unreadEntries, headers: response.allHeaderFields) - completion(.success(unreadEntries)) + completion(.success(itemIds)) case .failure(let error): completion(.failure(error)) } @@ -541,17 +701,17 @@ final class GoogleReaderCompatibleAPICaller: NSObject { } func createUnreadEntries(entries: [Int], completion: @escaping (Result) -> Void) { - let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json") - let request = URLRequest(url: callURL, credentials: credentials) - let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries) - transport.send(request: request, method: HTTPMethod.post, payload: payload, completion: completion) +// let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json") +// let request = URLRequest(url: callURL, credentials: credentials) +// let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries) +// transport.send(request: request, method: HTTPMethod.post, payload: payload, completion: completion) } func deleteUnreadEntries(entries: [Int], completion: @escaping (Result) -> Void) { - let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json") - let request = URLRequest(url: callURL, credentials: credentials) - let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries) - transport.send(request: request, method: HTTPMethod.delete, payload: payload, completion: completion) +// let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json") +// let request = URLRequest(url: callURL, credentials: credentials) +// let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries) +// transport.send(request: request, method: HTTPMethod.delete, payload: payload, completion: completion) } func retrieveStarredEntries(completion: @escaping (Result<[Int]?, Error>) -> Void) { diff --git a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAccountDelegate.swift b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAccountDelegate.swift index df677389d..8f192cd17 100644 --- a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAccountDelegate.swift +++ b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAccountDelegate.swift @@ -21,6 +21,7 @@ import os.log public enum GoogleReaderCompatibleAccountDelegateError: String, Error { case invalidParameter = "There was an invalid parameter passed." + case invalidResponse = "There was an invalid response from the server." } final class GoogleReaderCompatibleAccountDelegate: AccountDelegate { @@ -98,14 +99,14 @@ final class GoogleReaderCompatibleAccountDelegate: AccountDelegate { self.refreshArticles(account) { -// self.refreshArticleStatus(for: account) { -// self.refreshMissingArticles(account) { + self.refreshArticleStatus(for: account) { + self.refreshMissingArticles(account) { self.refreshProgress.clear() DispatchQueue.main.async { completion(.success(())) } -// } -// } + } + } } case .failure(let error): @@ -178,18 +179,18 @@ final class GoogleReaderCompatibleAccountDelegate: AccountDelegate { } - group.enter() - caller.retrieveStarredEntries() { result in - switch result { - case .success(let articleIDs): - self.syncArticleStarredState(account: account, articleIDs: articleIDs) - group.leave() - case .failure(let error): - os_log(.info, log: self.log, "Retrieving starred entries failed: %@.", error.localizedDescription) - group.leave() - } - - } +// group.enter() +// caller.retrieveStarredEntries() { result in +// switch result { +// case .success(let articleIDs): +// self.syncArticleStarredState(account: account, articleIDs: articleIDs) +// group.leave() +// case .failure(let error): +// os_log(.info, log: self.log, "Retrieving starred entries failed: %@.", error.localizedDescription) +// group.leave() +// } +// +// } group.notify(queue: DispatchQueue.main) { os_log(.debug, log: self.log, "Done refreshing article statuses.") @@ -970,7 +971,7 @@ private extension GoogleReaderCompatibleAccountDelegate { func processEntries(account: Account, entries: [GoogleReaderCompatibleEntry]?, completion: @escaping (() -> Void)) { - let parsedItems = mapEntriesToParsedItems(entries: entries) + let parsedItems = mapEntriesToParsedItems(account: account, entries: entries) let parsedMap = Dictionary(grouping: parsedItems, by: { item in item.feedURL } ) let group = DispatchGroup() @@ -997,15 +998,17 @@ private extension GoogleReaderCompatibleAccountDelegate { } - func mapEntriesToParsedItems(entries: [GoogleReaderCompatibleEntry]?) -> Set { + func mapEntriesToParsedItems(account: Account, entries: [GoogleReaderCompatibleEntry]?) -> Set { guard let entries = entries else { return Set() } let parsedItems: [ParsedItem] = entries.map { entry in - let authors = Set([ParsedAuthor(name: entry.authorName, url: entry.jsonFeed?.jsonFeedAuthor?.url, avatarURL: entry.jsonFeed?.jsonFeedAuthor?.avatarURL, emailAddress: nil)]) - return ParsedItem(syncServiceID: String(entry.articleID), uniqueID: String(entry.articleID), feedURL: String(entry.feedID), url: nil, externalURL: entry.url, title: entry.title, contentHTML: entry.contentHTML, contentText: nil, summary: entry.summary, imageURL: nil, bannerImageURL: nil, datePublished: entry.parseDatePublished(), dateModified: nil, authors: authors, tags: nil, attachments: nil) + // let authors = Set([ParsedAuthor(name: entry.authorName, url: entry.jsonFeed?.jsonFeedAuthor?.url, avatarURL: entry.jsonFeed?.jsonFeedAuthor?.avatarURL, emailAddress: nil)]) + // let feed = account.idToFeedDictionary[entry.origin.streamId!]! // TODO clean this up + + return ParsedItem(syncServiceID: String(entry.articleID), uniqueID: String(entry.articleID), feedURL: entry.origin.streamId!, url: nil, externalURL: entry.alternates.first?.url, title: entry.title, contentHTML: entry.summary.content, contentText: nil, summary: entry.summary.content, imageURL: nil, bannerImageURL: nil, datePublished: entry.parseDatePublished(), dateModified: nil, authors: nil, tags: nil, attachments: nil) } return Set(parsedItems) diff --git a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleEntry.swift b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleEntry.swift index 24fabd9dc..c98212774 100644 --- a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleEntry.swift +++ b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleEntry.swift @@ -10,58 +10,103 @@ import Foundation import RSParser import RSCore +struct GoogleReaderCompatibleEntryWrapper: Codable { + let id: String + let updated: Int + let entries: [GoogleReaderCompatibleEntry] + + + enum CodingKeys: String, CodingKey { + case id = "id" + case updated = "updated" + case entries = "items" + } +} + +/* { +"id": "tag:google.com,2005:reader/item/00058a3b5197197b", +"crawlTimeMsec": "1559362260113", +"timestampUsec": "1559362260113787", +"published": 1554845280, +"title": "", +"summary": { +"content": "\n

Found an old screenshot of NetNewsWire 1.0 for iPhone!

\n\n

\"Netnewswire

\n" +}, +"alternate": [ +{ +"href": "https://nnw.ranchero.com/2019/04/09/found-an-old.html" +} +], +"categories": [ +"user/-/state/com.google/reading-list", +"user/-/label/Uncategorized" +], +"origin": { +"streamId": "feed/130", +"title": "NetNewsWire" +} +} +*/ struct GoogleReaderCompatibleEntry: Codable { - let articleID: Int - let feedID: Int + let articleID: String let title: String? - let url: String? - let authorName: String? - let contentHTML: String? - let summary: String? - let datePublished: String? - let dateArrived: String? - let jsonFeed: GoogleReaderCompatibleEntryJSONFeed? + + let publishedTimestamp: Double? + let crawledTimestamp: String? + let timestampUsec: String? + + let summary: GoogleReaderCompatibleArticleSummary + let alternates: [GoogleReaderCompatibleAlternateLocation] + let categories: [String] + let origin: GoogleReaderCompatibleEntryOrigin enum CodingKeys: String, CodingKey { case articleID = "id" - case feedID = "feed_id" case title = "title" - case url = "url" - case authorName = "author" - case contentHTML = "content" case summary = "summary" - case datePublished = "published" - case dateArrived = "created_at" - case jsonFeed = "json_feed" - } - - // GoogleReaderCompatible dates can't be decoded by the JSONDecoding 8601 decoding strategy. GoogleReaderCompatible - // requires a very specific date formatter to work and even then it fails occasionally. - // Rather than loose all the entries we only lose the one date by decoding as a string - // and letting the one date fail when parsed. - func parseDatePublished() -> Date? { - if datePublished != nil { - return GoogleReaderCompatibleDate.formatter.date(from: datePublished!) - } else { - return nil - } + case alternates = "alternate" + case categories = "categories" + case publishedTimestamp = "published" + case crawledTimestamp = "crawlTimeMsec" + case origin = "origin" + case timestampUsec = "timestampUsec" } -} - -struct GoogleReaderCompatibleEntryJSONFeed: Codable { - let jsonFeedAuthor: GoogleReaderCompatibleEntryJSONFeedAuthor? - enum CodingKeys: String, CodingKey { - case jsonFeedAuthor = "author" + func parseDatePublished() -> Date? { + + guard let unixTime = publishedTimestamp else { + return nil + } + + return Date(timeIntervalSince1970: unixTime) } } -struct GoogleReaderCompatibleEntryJSONFeedAuthor: Codable { +struct GoogleReaderCompatibleArticleSummary: Codable { + let content: String? + + enum CodingKeys: String, CodingKey { + case content = "content" + } +} + +struct GoogleReaderCompatibleAlternateLocation: Codable { let url: String? - let avatarURL: String? + enum CodingKeys: String, CodingKey { - case url = "url" - case avatarURL = "avatar" + case url = "href" } } + + +struct GoogleReaderCompatibleEntryOrigin: Codable { + let streamId: String? + let title: String? + + enum CodingKeys: String, CodingKey { + case streamId = "streamId" + case title = "title" + } +} + diff --git a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleUnreadEntry.swift b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleUnreadEntry.swift index a5ce66ae0..b08073bfc 100644 --- a/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleUnreadEntry.swift +++ b/Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleUnreadEntry.swift @@ -8,12 +8,20 @@ import Foundation -struct GoogleReaderCompatibleUnreadEntry: Codable { - - let unreadEntries: [Int] +struct GoogleReaderCompatibleReferenceWrapper: Codable { + let itemRefs: [GoogleReaderCompatibleReference] enum CodingKeys: String, CodingKey { - case unreadEntries = "unread_entries" + case itemRefs = "itemRefs" + } +} + +struct GoogleReaderCompatibleReference: Codable { + + let itemId: String + + enum CodingKeys: String, CodingKey { + case itemId = "id" } } diff --git a/submodules/RSWeb b/submodules/RSWeb index cf3a30eb3..142cb8ccc 160000 --- a/submodules/RSWeb +++ b/submodules/RSWeb @@ -1 +1 @@ -Subproject commit cf3a30eb3833d9dd423fed003393e6e3c1a360d4 +Subproject commit 142cb8ccc491201e3de35c0b5d76d23d785f1978