From 7d992ce1753cdd6a643be05a6e9a4dd4375d5806 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 4 Aug 2023 12:05:29 +0800 Subject: [PATCH] :sparkles: Support for searching asset content https://github.com/siyuan-note/siyuan/issues/8874 --- app/appearance/langs/en_US.json | 4 +- app/appearance/langs/es_ES.json | 4 +- app/appearance/langs/fr_FR.json | 4 +- app/appearance/langs/zh_CHT.json | 4 +- app/appearance/langs/zh_CN.json | 4 +- kernel/cache/asset.go | 2 +- kernel/go.mod | 2 +- kernel/go.sum | 2 + kernel/job/cron.go | 1 + kernel/main.go | 1 + kernel/mobile/kernel.go | 1 + kernel/model/asset_content.go | 155 ++++++++++++++++++++++++++++++ kernel/model/index.go | 12 +++ kernel/sql/asset_content.go | 96 ++++++++++++++++++ kernel/sql/database.go | 109 ++++++++++++++++++--- kernel/sql/queue_asset_content.go | 147 ++++++++++++++++++++++++++++ kernel/sql/queue_history.go | 28 +----- kernel/task/queue.go | 30 +++--- kernel/util/runtime.go | 3 +- kernel/util/working.go | 34 ++++--- kernel/util/working_mobile.go | 1 + 21 files changed, 568 insertions(+), 76 deletions(-) create mode 100644 kernel/model/asset_content.go create mode 100644 kernel/sql/asset_content.go create mode 100644 kernel/sql/queue_asset_content.go diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 818798c3e..52230ce99 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -1260,6 +1260,8 @@ "212": "There are some defects in the current version of cloud data sync, please upgrade to the latest version. Sorry for the inconvenience", "213": "Cloud verification failed, please try to upgrade to the latest version and log in again before syncing", "214": "This function needs to be signed in to use", - "215": "Save failed: The target file is being used by another program" + "215": "Save failed: The target file is being used by another program", + "216": "Rebuilding asset content data index, please wait...", + "217": "[%d/%d] Created asset content data index" } } diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 34cb88c21..b40a635ad 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -1260,6 +1260,8 @@ "212": "Hay algunos defectos en la versi\u00f3n actual de sincronizaci\u00f3n de datos en la nube, actualice a la versi\u00f3n m\u00e1s reciente. Disculpe las molestias", "213": "La verificaci\u00f3n en la nube fall\u00f3, intente actualizar a la versi\u00f3n m\u00e1s reciente e inicie sesi\u00f3n de nuevo antes de sincronizar", "214": "Esta función requiere iniciar sesión en la cuenta antes de poder usarla", - "215": "Error al guardar: el archivo de destino está siendo utilizado por otro programa" + "215": "Error al guardar: el archivo de destino está siendo utilizado por otro programa", + "216": "Reconstruyendo el índice de datos de contenido de recursos, espere...", + "217": "[%d/%d] Índice de datos de contenido de activos creado" } } diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 41255c4a0..cb5d12a72 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -1260,6 +1260,8 @@ "212": "Il y a quelques défauts dans la version actuelle de la synchronisation des données cloud, veuillez mettre à niveau vers la dernière version. Désolé pour le désagrément", "213": "Échec de la vérification cloud, veuillez essayer de mettre à niveau vers la dernière version et de vous reconnecter avant de synchroniser", "214": "La fonctionnalité nécessite un numéro de compte de connexion avant de pouvoir être utilisée", - "215": "Échec de l'enregistrement : le fichier de destination est utilisé par un autre programme" + "215": "Échec de l'enregistrement : le fichier de destination est utilisé par un autre programme", + "216": "Reconstruction de l'index des données du contenu des ressources, veuillez patienter...", + "217": "[%d/%d] Création d'un index de données de contenu d'actif" } } diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 2b98db1f1..eeea014a6 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -1260,6 +1260,8 @@ "212": "當前版本雲端數據同步存在一些缺陷，請升級到最新版，帶來不便，敬請諒解", "213": "雲端校驗失敗，請嘗試升級到最新版並重新登錄後再進行同步", "214": "該功能需要登錄賬號後才能使用", - "215": "保存失敗：目標文件正在被其他程序佔用" + "215": "保存失敗：目標文件正在被其他程序佔用", + "216": "正在重建資源文件內容數據索引，請稍等...", + "217": "[%d/%d] 已經建立條資源文件內容數據索引" } } diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 65f050aec..ce81da7a7 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -1260,6 +1260,8 @@ "212": "当前版本云端数据同步存在一些缺陷，请升级到最新版，带来不便，敬请谅解", "213": "云端校验失败，请尝试升级到最新版并重新登录后再进行同步", "214": "该功能需要登录账号后才能使用", - "215": "保存失败：目标文件并且正在被其他程序占用" + "215": "保存失败：目标文件并且正在被其他程序占用", + "216": "正在重建资源文件内容数据索引，请稍等...", + "217": "[%d/%d] 已经建立条资源文件内容数据索引" } } diff --git a/kernel/cache/asset.go b/kernel/cache/asset.go index 5052d3563..3af22e8e4 100644 --- a/kernel/cache/asset.go +++ b/kernel/cache/asset.go @@ -79,7 +79,7 @@ func LoadAssets() { assetsCache[path] = &Asset{ HName: hName, Path: path, - Updated: info.ModTime().UnixMilli(), + Updated: info.ModTime().Unix(), } return nil }) diff --git a/kernel/go.mod b/kernel/go.mod index 6c5358483..aa131217b 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -47,7 +47,7 @@ require ( github.com/shirou/gopsutil/v3 v3.23.6 github.com/siyuan-note/dejavu v0.0.0-20230801123133-2edc24064c33 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 - github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112 + github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524 github.com/siyuan-note/httpclient v0.0.0-20230728124841-53922bac2be2 github.com/siyuan-note/logging v0.0.0-20230327073243-ebe83aec1493 diff --git a/kernel/go.sum b/kernel/go.sum index 6f1100956..8a1339fee 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -296,6 +296,8 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112 h1:lb+8C+XEEEn/lcBtoXlrf5mZEoe0y0KlqiIGG93Gozc= github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= +github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 h1:XghjHKJd+SiL0DkGYFVC+UGUDFtnR4v9gkAbPeh9Eq8= +github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524 h1:ZuxN5gwqtUOd1NkOkNhM4OlVWfjujY98zsR+zFi4x9g= github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524/go.mod h1:jK5lCYfPbFOrW23/HMeU7kmpLdEd5GkennF+kUpy7Vs= github.com/siyuan-note/httpclient v0.0.0-20230728124841-53922bac2be2 h1:z6vYbmEOVoytf30Ny6YDjyZTYdCPmazeAl4BN67+308= diff --git a/kernel/job/cron.go b/kernel/job/cron.go index 578e5fff0..7026d92be 100644 --- a/kernel/job/cron.go +++ b/kernel/job/cron.go @@ -38,6 +38,7 @@ func StartCron() { go every(50*time.Millisecond, model.FlushTxJob) go every(util.SQLFlushInterval, sql.FlushTxJob) go every(util.SQLFlushInterval, sql.FlushHistoryTxJob) + go every(util.SQLFlushInterval, sql.FlushAssetContentTxJob) go every(10*time.Minute, model.FixIndexJob) go every(10*time.Minute, model.IndexEmbedBlockJob) go every(10*time.Minute, model.CacheVirtualBlockRefJob) diff --git a/kernel/main.go b/kernel/main.go index 59ef67049..a1021717a 100644 --- a/kernel/main.go +++ b/kernel/main.go @@ -35,6 +35,7 @@ func main() { model.InitAppearance() sql.InitDatabase(false) sql.InitHistoryDatabase(false) + sql.InitAssetContentDatabase(false) sql.SetCaseSensitive(model.Conf.Search.CaseSensitive) sql.SetIndexAssetPath(model.Conf.Search.IndexAssetPath) diff --git a/kernel/mobile/kernel.go b/kernel/mobile/kernel.go index ae89d2953..2d5b7f89c 100644 --- a/kernel/mobile/kernel.go +++ b/kernel/mobile/kernel.go @@ -49,6 +49,7 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang model.InitAppearance() sql.InitDatabase(false) sql.InitHistoryDatabase(false) + sql.InitAssetContentDatabase(false) sql.SetCaseSensitive(model.Conf.Search.CaseSensitive) sql.SetIndexAssetPath(model.Conf.Search.IndexAssetPath) diff --git a/kernel/model/asset_content.go b/kernel/model/asset_content.go new file mode 100644 index 000000000..d609bef7c --- /dev/null +++ b/kernel/model/asset_content.go @@ -0,0 +1,155 @@ +// SiYuan - Refactor your thinking +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package model + +import ( + "github.com/88250/gulu" + "github.com/88250/lute/ast" + "github.com/siyuan-note/eventbus" + "github.com/siyuan-note/filelock" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" + "github.com/siyuan-note/siyuan/kernel/util" + "io/fs" + "path/filepath" + "strings" +) + +func ReindexAssetContent() { + task.AppendTask(task.AssetContentDatabaseIndexFull, fullReindexAssetContent) + return +} + +func fullReindexAssetContent() { + util.PushMsg(Conf.Language(216), 7*1000) + sql.InitAssetContentDatabase(true) + + assetsSearch := NewAssetsSearcher() + assetsSearch.Index() + return +} + +func init() { + subscribeSQLAssetContentEvents() +} + +func subscribeSQLAssetContentEvents() { + eventbus.Subscribe(util.EvtSQLAssetContentRebuild, func() { + ReindexAssetContent() + }) +} + +var ( + AssetsSearchEnabled = true +) + +type AssetsSearcher struct { + AssetsDir string + Parsers map[string]AssetParser +} + +func (searcher *AssetsSearcher) Index() { + assetsDir := searcher.AssetsDir + if !gulu.File.IsDir(assetsDir) { + return + } + + var results []*AssetParseResult + filepath.Walk(assetsDir, func(absPath string, info fs.FileInfo, err error) error { + if nil != err { + logging.LogErrorf("walk dir [%s] failed: %s", absPath, err) + return err + } + + if info.IsDir() { + return nil + } + + ext := strings.ToLower(filepath.Ext(absPath)) + parser, found := searcher.Parsers[ext] + if !found { + return nil + } + + result := parser.Parse(absPath) + if nil == result { + return nil + } + + result.Path = "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir)) + result.Size = info.Size() + result.Updated = info.ModTime().Unix() + results = append(results, result) + return nil + }) + + var assetContents []*sql.AssetContent + for _, result := range results { + assetContents = append(assetContents, &sql.AssetContent{ + ID: ast.NewNodeID(), + Name: filepath.Base(result.Path), + Ext: filepath.Ext(result.Path), + Path: result.Path, + Size: result.Size, + Updated: result.Updated, + Content: result.Content, + }) + } + + sql.IndexAssetContentsQueue(assetContents) +} + +func NewAssetsSearcher() *AssetsSearcher { + return &AssetsSearcher{ + AssetsDir: util.GetDataAssetsAbsPath(), + Parsers: map[string]AssetParser{ + ".txt": &TxtAssetParser{}, + }, + } +} + +type AssetParseResult struct { + Path string + Size int64 + Updated int64 + Content string +} + +type AssetParser interface { + Parse(absPath string) *AssetParseResult +} + +type TxtAssetParser struct { +} + +func (parser *TxtAssetParser) Parse(absPath string) (ret *AssetParseResult) { + if !strings.HasSuffix(strings.ToLower(absPath), ".txt") { + return + } + + data, err := filelock.ReadFile(absPath) + if nil != err { + logging.LogErrorf("read file [%s] failed: %s", absPath, err) + return + } + + ret = &AssetParseResult{ + Content: string(data), + } + return +} diff --git a/kernel/model/index.go b/kernel/model/index.go index 8ac5202c4..19e6d9b8f 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -303,4 +303,16 @@ func subscribeSQLEvents() { util.SetBootDetails(msg) util.ContextPushMsg(context, msg) }) + + eventbus.Subscribe(eventbus.EvtSQLInsertAssetContent, func(context map[string]interface{}) { + if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container { + return + } + + current := context["current"].(int) + total := context["total"] + msg := fmt.Sprintf(Conf.Language(217), current, total) + util.SetBootDetails(msg) + util.ContextPushMsg(context, msg) + }) } diff --git a/kernel/sql/asset_content.go b/kernel/sql/asset_content.go new file mode 100644 index 000000000..19ceab244 --- /dev/null +++ b/kernel/sql/asset_content.go @@ -0,0 +1,96 @@ +// SiYuan - Refactor your thinking +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/siyuan-note/eventbus" +) + +type AssetContent struct { + ID string + Name string + Ext string + Path string + Size int64 + Updated int64 + Content string +} + +const ( + AssetContentsFTSCaseInsensitiveInsert = "INSERT INTO asset_contents_fts_case_insensitive (id, name, ext, path, size, updated, content) VALUES %s" + AssetContentsPlaceholder = "(?, ?, ?, ?, ?, ?, ?)" +) + +func insertAssetContents(tx *sql.Tx, assetContents []*AssetContent, context map[string]interface{}) (err error) { + if 1 > len(assetContents) { + return + } + + var bulk []*AssetContent + for _, assetContent := range assetContents { + bulk = append(bulk, assetContent) + if 512 > len(bulk) { + continue + } + + if err = insertAssetContents0(tx, bulk, context); nil != err { + return + } + bulk = []*AssetContent{} + } + if 0 < len(bulk) { + if err = insertAssetContents0(tx, bulk, context); nil != err { + return + } + } + return +} + +func insertAssetContents0(tx *sql.Tx, bulk []*AssetContent, context map[string]interface{}) (err error) { + valueStrings := make([]string, 0, len(bulk)) + valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetContentsPlaceholder, "?")) + for _, b := range bulk { + valueStrings = append(valueStrings, AssetContentsPlaceholder) + valueArgs = append(valueArgs, b.ID) + valueArgs = append(valueArgs, b.Name) + valueArgs = append(valueArgs, b.Ext) + valueArgs = append(valueArgs, b.Path) + valueArgs = append(valueArgs, b.Size) + valueArgs = append(valueArgs, b.Updated) + valueArgs = append(valueArgs, b.Content) + } + + stmt := fmt.Sprintf(AssetContentsFTSCaseInsensitiveInsert, strings.Join(valueStrings, ",")) + if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err { + return + } + + eventbus.Publish(eventbus.EvtSQLInsertAssetContent, context) + return +} + +func deleteAssetContentsByPath(tx *sql.Tx, path string, context map[string]interface{}) (err error) { + stmt := "DELETE FROM asset_contents_fts_case_insensitive WHERE path = ?" + if err = execStmtTx(tx, stmt, path); nil != err { + return + } + return +} diff --git a/kernel/sql/database.go b/kernel/sql/database.go index e6b93756a..afec892fa 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -45,8 +45,9 @@ import ( ) var ( - db *sql.DB - historyDB *sql.DB + db *sql.DB + historyDB *sql.DB + assetContentDB *sql.DB ) func init() { @@ -193,7 +194,36 @@ func initDBTables() { } } +func initDBConnection() { + if nil != db { + closeDatabase() + } + dsn := util.DBPath + "?_journal_mode=WAL" + + "&_synchronous=OFF" + + "&_mmap_size=2684354560" + + "&_secure_delete=OFF" + + "&_cache_size=-20480" + + "&_page_size=32768" + + "&_busy_timeout=7000" + + "&_ignore_check_constraints=ON" + + "&_temp_store=MEMORY" + + "&_case_sensitive_like=OFF" + var err error + db, err = sql.Open("sqlite3_extended", dsn) + if nil != err { + logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err) + } + db.SetMaxIdleConns(20) + db.SetMaxOpenConns(20) + db.SetConnMaxLifetime(365 * 24 * time.Hour) +} + +var initHistoryDatabaseLock = sync.Mutex{} + func InitHistoryDatabase(forceRebuild bool) { + initHistoryDatabaseLock.Lock() + defer initHistoryDatabaseLock.Unlock() + initHistoryDBConnection() if !forceRebuild && gulu.File.IsExist(util.HistoryDBPath) { @@ -228,7 +258,7 @@ func initHistoryDBConnection() { var err error historyDB, err = sql.Open("sqlite3_extended", dsn) if nil != err { - logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err) + logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create history database failed: %s", err) } historyDB.SetMaxIdleConns(3) historyDB.SetMaxOpenConns(3) @@ -243,11 +273,34 @@ func initHistoryDBTables() { } } -func initDBConnection() { - if nil != db { - closeDatabase() +var initAssetContentDatabaseLock = sync.Mutex{} + +func InitAssetContentDatabase(forceRebuild bool) { + initAssetContentDatabaseLock.Lock() + defer initAssetContentDatabaseLock.Unlock() + + initAssetContentDBConnection() + + if !forceRebuild && gulu.File.IsExist(util.AssetContentDBPath) { + return } - dsn := util.DBPath + "?_journal_mode=WAL" + + + assetContentDB.Close() + if err := os.RemoveAll(util.AssetContentDBPath); nil != err { + logging.LogErrorf("remove assets database file [%s] failed: %s", util.AssetContentDBPath, err) + return + } + + initAssetContentDBConnection() + initAssetContentDBTables() +} + +func initAssetContentDBConnection() { + if nil != assetContentDB { + assetContentDB.Close() + } + + dsn := util.AssetContentDBPath + "?_journal_mode=WAL" + "&_synchronous=OFF" + "&_mmap_size=2684354560" + "&_secure_delete=OFF" + @@ -258,13 +311,21 @@ func initDBConnection() { "&_temp_store=MEMORY" + "&_case_sensitive_like=OFF" var err error - db, err = sql.Open("sqlite3_extended", dsn) + assetContentDB, err = sql.Open("sqlite3_extended", dsn) if nil != err { - logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err) + logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create assets database failed: %s", err) + } + assetContentDB.SetMaxIdleConns(3) + assetContentDB.SetMaxOpenConns(3) + assetContentDB.SetConnMaxLifetime(365 * 24 * time.Hour) +} + +func initAssetContentDBTables() { + assetContentDB.Exec("DROP TABLE asset_contents_fts_case_insensitive") + _, err := assetContentDB.Exec("CREATE VIRTUAL TABLE asset_contents_fts_case_insensitive USING fts5(id UNINDEXED, name, ext, path, size UNINDEXED, updated UNINDEXED, content, tokenize=\"siyuan case_insensitive\")") + if nil != err { + logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [asset_contents_fts_case_insensitive] failed: %s", err) } - db.SetMaxIdleConns(20) - db.SetMaxOpenConns(20) - db.SetConnMaxLifetime(365 * 24 * time.Hour) } var ( @@ -1161,6 +1222,18 @@ func beginTx() (tx *sql.Tx, err error) { return } +func commitTx(tx *sql.Tx) (err error) { + if nil == tx { + logging.LogErrorf("tx is nil") + return errors.New("tx is nil") + } + + if err = tx.Commit(); nil != err { + logging.LogErrorf("commit tx failed: %s\n %s", err, logging.ShortStack()) + } + return +} + func beginHistoryTx() (tx *sql.Tx, err error) { if tx, err = historyDB.Begin(); nil != err { logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack()) @@ -1183,7 +1256,17 @@ func commitHistoryTx(tx *sql.Tx) (err error) { return } -func commitTx(tx *sql.Tx) (err error) { +func beginAssetContentTx() (tx *sql.Tx, err error) { + if tx, err = assetContentDB.Begin(); nil != err { + logging.LogErrorf("begin asset content tx failed: %s\n %s", err, logging.ShortStack()) + if strings.Contains(err.Error(), "database is locked") { + os.Exit(logging.ExitCodeReadOnlyDatabase) + } + } + return +} + +func commitAssetContentTx(tx *sql.Tx) (err error) { if nil == tx { logging.LogErrorf("tx is nil") return errors.New("tx is nil") diff --git a/kernel/sql/queue_asset_content.go b/kernel/sql/queue_asset_content.go new file mode 100644 index 000000000..22357634e --- /dev/null +++ b/kernel/sql/queue_asset_content.go @@ -0,0 +1,147 @@ +// SiYuan - Refactor your thinking +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "database/sql" + "errors" + "fmt" + "runtime/debug" + "sync" + "time" + + "github.com/siyuan-note/eventbus" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/task" + "github.com/siyuan-note/siyuan/kernel/util" +) + +var ( + assetContentOperationQueue []*assetContentDBQueueOperation + assetContentDBQueueLock = sync.Mutex{} + + assetContentTxLock = sync.Mutex{} +) + +type assetContentDBQueueOperation struct { + inQueueTime time.Time + action string // index/deletePath + + assetContents []*AssetContent // index + path string // deletePath +} + +func FlushAssetContentTxJob() { + task.AppendTask(task.AssetContentDatabaseIndexCommit, FlushAssetContentQueue) +} + +func FlushAssetContentQueue() { + ops := getAssetContentOperations() + if 1 > len(ops) { + return + } + + assetContentTxLock.Lock() + defer assetContentTxLock.Unlock() + start := time.Now() + + groupOpsTotal := map[string]int{} + for _, op := range ops { + groupOpsTotal[op.action]++ + } + + context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} + groupOpsCurrent := map[string]int{} + for i, op := range ops { + if util.IsExiting { + return + } + + tx, err := beginAssetContentTx() + if nil != err { + return + } + + groupOpsCurrent[op.action]++ + context["current"] = groupOpsCurrent[op.action] + context["total"] = groupOpsTotal[op.action] + + if err = execAssetContentOp(op, tx, context); nil != err { + tx.Rollback() + logging.LogErrorf("queue operation failed: %s", err) + eventbus.Publish(util.EvtSQLAssetContentRebuild) + return + } + + if err = commitAssetContentTx(tx); nil != err { + logging.LogErrorf("commit tx failed: %s", err) + return + } + + if 16 < i && 0 == i%128 { + debug.FreeOSMemory() + } + } + + if 128 < len(ops) { + debug.FreeOSMemory() + } + + elapsed := time.Now().Sub(start).Milliseconds() + if 7000 < elapsed { + logging.LogInfof("database asset content op tx [%dms]", elapsed) + } +} + +func execAssetContentOp(op *assetContentDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) { + switch op.action { + case "index": + err = insertAssetContents(tx, op.assetContents, context) + case "delete": + err = deleteAssetContentsByPath(tx, op.path, context) + default: + msg := fmt.Sprintf("unknown asset content operation [%s]", op.action) + logging.LogErrorf(msg) + err = errors.New(msg) + } + return +} + +func DeleteAssetContentsByPathQueue(path string) { + assetContentTxLock.Lock() + defer assetContentTxLock.Unlock() + + newOp := &assetContentDBQueueOperation{inQueueTime: time.Now(), action: "deletePath", path: path} + assetContentOperationQueue = append(assetContentOperationQueue, newOp) +} + +func IndexAssetContentsQueue(assetContents []*AssetContent) { + assetContentTxLock.Lock() + defer assetContentTxLock.Unlock() + + newOp := &assetContentDBQueueOperation{inQueueTime: time.Now(), action: "index", assetContents: assetContents} + assetContentOperationQueue = append(assetContentOperationQueue, newOp) +} + +func getAssetContentOperations() (ops []*assetContentDBQueueOperation) { + assetContentTxLock.Lock() + defer assetContentTxLock.Unlock() + + ops = assetContentOperationQueue + assetContentOperationQueue = nil + return +} diff --git a/kernel/sql/queue_history.go b/kernel/sql/queue_history.go index 1c7c4efc9..32a3a3be6 100644 --- a/kernel/sql/queue_history.go +++ b/kernel/sql/queue_history.go @@ -55,8 +55,8 @@ func FlushHistoryQueue() { return } - txLock.Lock() - defer txLock.Unlock() + historyTxLock.Lock() + defer historyTxLock.Unlock() start := time.Now() groupOpsTotal := map[string]int{} @@ -145,27 +145,3 @@ func getHistoryOperations() (ops []*historyDBQueueOperation) { historyOperationQueue = nil return } - -func WaitForWritingHistoryDatabase() { - var printLog bool - var lastPrintLog bool - for i := 0; isWritingHistoryDatabase(); i++ { - time.Sleep(50 * time.Millisecond) - if 200 < i && !printLog { // 10s 后打日志 - logging.LogWarnf("history database is writing: \n%s", logging.ShortStack()) - printLog = true - } - if 1200 < i && !lastPrintLog { // 60s 后打日志 - logging.LogWarnf("history database is still writing") - lastPrintLog = true - } - } -} - -func isWritingHistoryDatabase() bool { - time.Sleep(util.SQLFlushInterval + 50*time.Millisecond) - if 0 < len(historyOperationQueue) || util.IsMutexLocked(&historyTxLock) { - return true - } - return false -} diff --git a/kernel/task/queue.go b/kernel/task/queue.go index 727fc9ea0..06dc2c8d4 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -82,19 +82,21 @@ func getCurrentActions() (ret []string) { } const ( - RepoCheckout = "task.repo.checkout" // 从快照中检出 - DatabaseIndexFull = "task.database.index.full" // 重建索引 - DatabaseIndex = "task.database.index" // 数据库索引 - DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 - DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 - DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 - OCRImage = "task.ocr.image" // 图片 OCR 提取文本 - HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 - HistoryDatabaseIndexFull = "task.history.database.index.full" // 历史数据库重建索引 - HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交 - DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 - ReloadUI = "task.reload.ui" // 重载 UI - UpgradeUserGuide = "task.upgrade.userGuide" // 升级用户指南文档笔记本 + RepoCheckout = "task.repo.checkout" // 从快照中检出 + DatabaseIndexFull = "task.database.index.full" // 重建索引 + DatabaseIndex = "task.database.index" // 数据库索引 + DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 + DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 + DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 + OCRImage = "task.ocr.image" // 图片 OCR 提取文本 + HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 + HistoryDatabaseIndexFull = "task.history.database.index.full" // 历史数据库重建索引 + HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交 + DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 + ReloadUI = "task.reload.ui" // 重载 UI + UpgradeUserGuide = "task.upgrade.userGuide" // 升级用户指南文档笔记本 + AssetContentDatabaseIndexFull = "task.asset.database.index.full" // 资源文件数据库重建索引 + AssetContentDatabaseIndexCommit = "task.asset.database.index.commit" // 资源文件数据库索引提交 ) // uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。 @@ -107,6 +109,8 @@ var uniqueActions = []string{ HistoryDatabaseIndexFull, HistoryDatabaseIndexCommit, DatabaseIndexEmbedBlock, + AssetContentDatabaseIndexFull, + AssetContentDatabaseIndexCommit, } func Contain(action string, moreActions ...string) bool { diff --git a/kernel/util/runtime.go b/kernel/util/runtime.go index b9acfe2e1..2d005bbb1 100644 --- a/kernel/util/runtime.go +++ b/kernel/util/runtime.go @@ -408,5 +408,6 @@ func existAvailabilityStatus(workspaceAbsPath string) bool { const ( EvtConfPandocInitialized = "conf.pandoc.initialized" - EvtSQLHistoryRebuild = "sql.history.rebuild" + EvtSQLHistoryRebuild = "sql.history.rebuild" + EvtSQLAssetContentRebuild = "sql.assetContent.rebuild" ) diff --git a/kernel/util/working.go b/kernel/util/working.go index df22baddc..5c1c3a43d 100644 --- a/kernel/util/working.go +++ b/kernel/util/working.go @@ -158,22 +158,23 @@ var ( HomeDir, _ = gulu.OS.Home() WorkingDir, _ = os.Getwd() - WorkspaceDir string // 工作空间目录路径 - WorkspaceLock *flock.Flock // 工作空间锁 - ConfDir string // 配置目录路径 - DataDir string // 数据目录路径 - RepoDir string // 仓库目录路径 - HistoryDir string // 数据历史目录路径 - TempDir string // 临时目录路径 - LogPath string // 配置目录下的日志文件 siyuan.log 路径 - DBName = "siyuan.db" // SQLite 数据库文件名 - DBPath string // SQLite 数据库文件路径 - HistoryDBPath string // SQLite 历史数据库文件路径 - BlockTreePath string // 区块树文件路径 - AppearancePath string // 配置目录下的外观目录 appearance/ 路径 - ThemesPath string // 配置目录下的外观目录下的 themes/ 路径 - IconsPath string // 配置目录下的外观目录下的 icons/ 路径 - SnippetsPath string // 数据目录下的 snippets/ 路径 + WorkspaceDir string // 工作空间目录路径 + WorkspaceLock *flock.Flock // 工作空间锁 + ConfDir string // 配置目录路径 + DataDir string // 数据目录路径 + RepoDir string // 仓库目录路径 + HistoryDir string // 数据历史目录路径 + TempDir string // 临时目录路径 + LogPath string // 配置目录下的日志文件 siyuan.log 路径 + DBName = "siyuan.db" // SQLite 数据库文件名 + DBPath string // SQLite 数据库文件路径 + HistoryDBPath string // SQLite 历史数据库文件路径 + AssetContentDBPath string // SQLite 资源文件内容数据库文件路径 + BlockTreePath string // 区块树文件路径 + AppearancePath string // 配置目录下的外观目录 appearance/ 路径 + ThemesPath string // 配置目录下的外观目录下的 themes/ 路径 + IconsPath string // 配置目录下的外观目录下的 icons/ 路径 + SnippetsPath string // 数据目录下的 snippets/ 路径 UIProcessIDs = sync.Map{} // UI 进程 ID ) @@ -247,6 +248,7 @@ func initWorkspaceDir(workspaceArg string) { os.Setenv("TMP", osTmpDir) DBPath = filepath.Join(TempDir, DBName) HistoryDBPath = filepath.Join(TempDir, "history.db") + AssetContentDBPath = filepath.Join(TempDir, "asset_content.db") BlockTreePath = filepath.Join(TempDir, "blocktree") SnippetsPath = filepath.Join(DataDir, "snippets") } diff --git a/kernel/util/working_mobile.go b/kernel/util/working_mobile.go index 4c11c4e14..f53222309 100644 --- a/kernel/util/working_mobile.go +++ b/kernel/util/working_mobile.go @@ -159,6 +159,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) { os.Setenv("TMP", osTmpDir) DBPath = filepath.Join(TempDir, DBName) HistoryDBPath = filepath.Join(TempDir, "history.db") + AssetContentDBPath = filepath.Join(TempDir, "asset_content.db") BlockTreePath = filepath.Join(TempDir, "blocktree") SnippetsPath = filepath.Join(DataDir, "snippets")