From 6903db357a9573a92fc726b2297cdb20a91fb9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Oliveira?= Date: Thu, 11 Feb 2021 00:26:26 +0100 Subject: [PATCH 1/3] Update ArticlesDatabase.swift Delete code that forced rebuilding the search index --- .../Sources/ArticlesDatabase/ArticlesDatabase.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift b/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift index fa01b7a3d..f0243c47f 100644 --- a/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift +++ b/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift @@ -85,7 +85,7 @@ public final class ArticlesDatabase { database.executeStatements("CREATE INDEX if not EXISTS articles_searchRowID on articles(searchRowID);") database.executeStatements("DROP TABLE if EXISTS tags;DROP INDEX if EXISTS tags_tagName_index;DROP INDEX if EXISTS articles_feedID_index;DROP INDEX if EXISTS statuses_read_index;DROP TABLE if EXISTS attachments;DROP TABLE if EXISTS attachmentsLookup;") if !self.searchTable.containsColumn("authors", in: database) { - database.executeStatements("DROP TABLE if EXISTS search;UPDATE articles SET searchRowID = null;") + database.executeStatements("DROP TABLE if EXISTS search;") database.executeStatements(ArticlesDatabase.searchTableCreationStatements) } } From b4e150044bfafbb809941b641003cde57cc176e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Oliveira?= Date: Thu, 11 Feb 2021 01:31:36 +0100 Subject: [PATCH 2/3] Add author information to indexed body - Restore old code for database model - Modified `bodyForIndex` function to return `authorsNames` in it --- .../ArticlesDatabase/ArticlesDatabase.swift | 17 ++++------------- .../Sources/ArticlesDatabase/SearchTable.swift | 14 ++++++++++---- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift b/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift index f0243c47f..ddb576ba9 100644 --- a/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift +++ b/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift @@ -63,7 +63,6 @@ public final class ArticlesDatabase { } private let articlesTable: ArticlesTable - private let searchTable: SearchTable private let queue: DatabaseQueue private let operationQueue = MainThreadOperationQueue() private let retentionStyle: RetentionStyle @@ -72,11 +71,9 @@ public final class ArticlesDatabase { let queue = DatabaseQueue(databasePath: databaseFilePath) self.queue = queue self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue, retentionStyle: retentionStyle) - self.searchTable = SearchTable(queue: queue, articlesTable: self.articlesTable) self.retentionStyle = retentionStyle try! queue.runCreateStatements(ArticlesDatabase.tableCreationStatements) - try! queue.runCreateStatements(ArticlesDatabase.searchTableCreationStatements) queue.runInDatabase { databaseResult in let database = databaseResult.database! if !self.articlesTable.containsColumn("searchRowID", in: database) { @@ -84,10 +81,6 @@ public final class ArticlesDatabase { } database.executeStatements("CREATE INDEX if not EXISTS articles_searchRowID on articles(searchRowID);") database.executeStatements("DROP TABLE if EXISTS tags;DROP INDEX if EXISTS tags_tagName_index;DROP INDEX if EXISTS articles_feedID_index;DROP INDEX if EXISTS statuses_read_index;DROP TABLE if EXISTS attachments;DROP TABLE if EXISTS attachmentsLookup;") - if !self.searchTable.containsColumn("authors", in: database) { - database.executeStatements("DROP TABLE if EXISTS search;") - database.executeStatements(ArticlesDatabase.searchTableCreationStatements) - } } DispatchQueue.main.async { @@ -338,14 +331,12 @@ private extension ArticlesDatabase { CREATE INDEX if not EXISTS articles_feedID_datePublished_articleID on articles (feedID, datePublished, articleID); CREATE INDEX if not EXISTS statuses_starred_index on statuses (starred); + + CREATE VIRTUAL TABLE if not EXISTS search using fts4(title, body); + + CREATE TRIGGER if not EXISTS articles_after_delete_trigger_delete_search_text after delete on articles begin delete from search where rowid = OLD.searchRowID; end; """ - static let searchTableCreationStatements = """ - CREATE VIRTUAL TABLE if not EXISTS search using fts4(title, body, authors); - - CREATE TRIGGER if not EXISTS articles_after_delete_trigger_delete_search_text after delete on articles begin delete from search where rowid = OLD.searchRowID; end; - """ - func todayCutoffDate() -> Date { // 24 hours previous. This is used by the Today smart feed, which should not actually empty out at midnight. return Date(timeIntervalSinceNow: -(60 * 60 * 24)) // This does not need to be more precise. diff --git a/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift b/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift index 7aca4ae5a..100eabd3f 100644 --- a/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift +++ b/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift @@ -35,7 +35,13 @@ final class ArticleSearchInfo: Hashable { lazy var bodyForIndex: String = { let s = preferredText.rsparser_stringByDecodingHTMLEntities() - return s.strippingHTML().collapsingWhitespace + let sanitizedBody = s.strippingHTML().collapsingWhitespace + + if let authorsNames = authorsNames { + return sanitizedBody.appending(" \(authorsNames)") + } else { + return sanitizedBody + } }() init(articleID: String, title: String?, authorsNames: String?, contentHTML: String?, contentText: String?, summary: String?, searchRowID: Int?) { @@ -49,7 +55,7 @@ final class ArticleSearchInfo: Hashable { } convenience init(article: Article) { - let authorsNames = article.authors?.map({ $0.name }).reduce("", { $0.appending("").appending($1 ?? "") }) + let authorsNames = article.authors?.map({ $0.name }).reduce("", { $0.appending("").appending($1 ?? "") }).collapsingWhitespace self.init(articleID: article.articleID, title: article.title, authorsNames: authorsNames, contentHTML: article.contentHTML, contentText: article.contentText, summary: article.summary, searchRowID: nil) } @@ -130,7 +136,7 @@ private extension SearchTable { } func insert(_ article: ArticleSearchInfo, _ database: FMDatabase) -> Int { - let rowDictionary: DatabaseDictionary = [DatabaseKey.body: article.bodyForIndex, DatabaseKey.title: article.title ?? "", DatabaseKey.authors: article.authorsNames ?? ""] + let rowDictionary: DatabaseDictionary = [DatabaseKey.body: article.bodyForIndex, DatabaseKey.title: article.title ?? ""] insertRow(rowDictionary, insertType: .normal, in: database) return Int(database.lastInsertRowId()) } @@ -204,7 +210,7 @@ private extension SearchTable { return nil } let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(searchRowIDs.count))! - let sql = "select rowid, title, body, authors from \(name) where rowid in \(placeholders);" + let sql = "select rowid, title, body from \(name) where rowid in \(placeholders);" guard let resultSet = database.executeQuery(sql, withArgumentsIn: searchRowIDs) else { return nil } From ddf9f4114519bdc2ff440ce33c02c1fb06568ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Oliveira?= Date: Thu, 11 Feb 2021 22:59:45 +0100 Subject: [PATCH 3/3] Re add code and clean up - Re add code reverted earlier - Clean up code that builds `authorsNames` based on model objects - Applied the right indentation --- .../ArticlesDatabase/ArticlesTable.swift | 25 +++++++++++++--- .../ArticlesDatabase/SearchTable.swift | 29 ++++++++++++------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesTable.swift b/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesTable.swift index 61c2845ef..c495f08aa 100644 --- a/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesTable.swift +++ b/ArticlesDatabase/Sources/ArticlesDatabase/ArticlesTable.swift @@ -145,19 +145,36 @@ final class ArticlesTable: DatabaseTable { } // MARK: - Fetching Articles for Indexer + private func articleSearchInfosQuery(with placeholders: String) -> String { + return """ + SELECT + art.articleID, + art.title, + art.contentHTML, + art.contentText, + art.summary, + art.searchRowID, + (SELECT GROUP_CONCAT(name, ' ') + FROM authorsLookup as autL + JOIN authors as aut ON autL.authorID = aut.authorID + WHERE art.articleID = autL.articleID + GROUP BY autl.articleID) as authors + FROM articles as art + WHERE articleID in \(placeholders); + """ + } func fetchArticleSearchInfos(_ articleIDs: Set, in database: FMDatabase) -> Set? { let parameters = articleIDs.map { $0 as AnyObject } let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(articleIDs.count))! - let sql = "select articleID, title, contentHTML, contentText, summary, searchRowID from articles where articleID in \(placeholders);"; - - if let resultSet = database.executeQuery(sql, withArgumentsIn: parameters) { + if let resultSet = database.executeQuery(self.articleSearchInfosQuery(with: placeholders), withArgumentsIn: parameters) { return resultSet.mapToSet { (row) -> ArticleSearchInfo? in let articleID = row.string(forColumn: DatabaseKey.articleID)! let title = row.string(forColumn: DatabaseKey.title) let contentHTML = row.string(forColumn: DatabaseKey.contentHTML) let contentText = row.string(forColumn: DatabaseKey.contentText) let summary = row.string(forColumn: DatabaseKey.summary) + let authorsNames = row.string(forColumn: DatabaseKey.authors) let searchRowIDObject = row.object(forColumnName: DatabaseKey.searchRowID) var searchRowID: Int? = nil @@ -165,7 +182,7 @@ final class ArticlesTable: DatabaseTable { searchRowID = Int(row.longLongInt(forColumn: DatabaseKey.searchRowID)) } - return ArticleSearchInfo(articleID: articleID, title: title, contentHTML: contentHTML, contentText: contentText, summary: summary, searchRowID: searchRowID) + return ArticleSearchInfo(articleID: articleID, title: title, contentHTML: contentHTML, contentText: contentText, summary: summary, authorsNames: authorsNames, searchRowID: searchRowID) } } return nil diff --git a/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift b/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift index 15bcb0cdd..ed08439f4 100644 --- a/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift +++ b/ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift @@ -20,6 +20,7 @@ final class ArticleSearchInfo: Hashable { let contentHTML: String? let contentText: String? let summary: String? + let authorsNames: String? let searchRowID: Int? var preferredText: String { @@ -34,18 +35,19 @@ final class ArticleSearchInfo: Hashable { lazy var bodyForIndex: String = { let s = preferredText.rsparser_stringByDecodingHTMLEntities() - let sanitizedBody = s.strippingHTML().collapsingWhitespace - - if let authorsNames = authorsNames { - return sanitizedBody.appending(" \(authorsNames)") - } else { - return sanitizedBody - } + let sanitizedBody = s.strippingHTML().collapsingWhitespace + + if let authorsNames = authorsNames { + return sanitizedBody.appending(" \(authorsNames)") + } else { + return sanitizedBody + } }() - init(articleID: String, title: String?, contentHTML: String?, contentText: String?, summary: String?, searchRowID: Int?) { + init(articleID: String, title: String?, contentHTML: String?, contentText: String?, summary: String?, authorsNames: String?, searchRowID: Int?) { self.articleID = articleID self.title = title + self.authorsNames = authorsNames self.contentHTML = contentHTML self.contentText = contentText self.summary = summary @@ -53,8 +55,13 @@ final class ArticleSearchInfo: Hashable { } convenience init(article: Article) { - let authorsNames = article.authors?.map({ $0.name }).reduce("", { $0.appending("").appending($1 ?? "") }).collapsingWhitespace - self.init(articleID: article.articleID, title: article.title, authorsNames: authorsNames, contentHTML: article.contentHTML, contentText: article.contentText, summary: article.summary, searchRowID: nil) + let authorsNames: String? + if let authors = article.authors { + authorsNames = authors.compactMap({ $0.name }).joined(separator: " ") + } else { + authorsNames = nil + } + self.init(articleID: article.articleID, title: article.title, contentHTML: article.contentHTML, contentText: article.contentText, summary: article.summary, authorsNames: authorsNames, searchRowID: nil) } // MARK: Hashable @@ -66,7 +73,7 @@ final class ArticleSearchInfo: Hashable { // MARK: Equatable static func == (lhs: ArticleSearchInfo, rhs: ArticleSearchInfo) -> Bool { - return lhs.articleID == rhs.articleID && lhs.title == rhs.title && lhs.contentHTML == rhs.contentHTML && lhs.contentText == rhs.contentText && lhs.summary == rhs.summary && lhs.searchRowID == rhs.searchRowID + return lhs.articleID == rhs.articleID && lhs.title == rhs.title && lhs.contentHTML == rhs.contentHTML && lhs.contentText == rhs.contentText && lhs.summary == rhs.summary && lhs.authorsNames == rhs.authorsNames && lhs.searchRowID == rhs.searchRowID } }