From 3fe8767c44051dba47fcd33e325f07a24ececcf1 Mon Sep 17 00:00:00 2001 From: Lykin <137850705+tiny-craft@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:44:08 +0800 Subject: [PATCH] feat: support import keys from csv file --- backend/services/browser_service.go | 135 ++++++++++++++++-- backend/services/system_service.go | 8 +- frontend/src/App.vue | 2 + .../src/components/common/FileOpenInput.vue | 5 +- .../components/dialogs/ExportKeyDialog.vue | 13 +- .../components/dialogs/ImportKeyDialog.vue | 122 ++++++++++++++++ frontend/src/components/icons/Export.vue | 48 +++---- frontend/src/components/icons/Import.vue | 6 +- .../src/components/new_value/AddHashValue.vue | 2 +- .../src/components/new_value/NewZSetValue.vue | 2 +- .../src/components/sidebar/BrowserPane.vue | 9 ++ frontend/src/langs/en-us.json | 24 +++- frontend/src/langs/zh-cn.json | 23 ++- frontend/src/stores/browser.js | 65 +++++++-- frontend/src/stores/dialog.js | 20 +++ 15 files changed, 411 insertions(+), 73 deletions(-) create mode 100644 frontend/src/components/dialogs/ImportKeyDialog.vue diff --git a/backend/services/browser_service.go b/backend/services/browser_service.go index bd746a2..6aae3ad 100644 --- a/backend/services/browser_service.go +++ b/backend/services/browser_service.go @@ -1999,7 +1999,7 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo st var deletedKeys = make([]any, 0, total) var mutex sync.Mutex del := func(ctx context.Context, cli redis.UniversalClient) error { - startTime := time.Now() + startTime := time.Now().Add(-10 * time.Second) for i, k := range ks { // emit progress per second param := map[string]any{ @@ -2010,6 +2010,8 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo st if i >= total-1 || time.Now().Sub(startTime).Milliseconds() > 100 { startTime = time.Now() runtime.EventsEmit(b.ctx, processEvent, param) + // do some sleep to prevent blocking the Redis server + time.Sleep(10 * time.Millisecond) } key := strutil.DecodeRedisKey(k) @@ -2026,8 +2028,6 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo st canceled = true break } - // do some sleep to prevent blocking the Redis server - time.Sleep(100 * time.Microsecond) } return nil } @@ -2093,13 +2093,17 @@ func (b *browserService) ExportKey(server string, db int, ks []any, path string) total := len(ks) var exported, failed int64 var canceled bool + startTime := time.Now().Add(-10 * time.Second) for i, k := range ks { - param := map[string]any{ - "total": total, - "progress": i + 1, - "processing": k, + if i >= total-1 || time.Now().Sub(startTime).Milliseconds() > 100 { + startTime = time.Now() + param := map[string]any{ + "total": total, + "progress": i + 1, + "processing": k, + } + runtime.EventsEmit(b.ctx, processEvent, param) } - runtime.EventsEmit(b.ctx, processEvent, param) key := strutil.DecodeRedisKey(k) content, dumpErr := client.Dump(ctx, key).Bytes() @@ -2128,6 +2132,121 @@ func (b *browserService) ExportKey(server string, db int, ks []any, path string) return } +// ImportCSV import data from csv file +func (b *browserService) ImportCSV(server string, db int, path string, conflict int) (resp types.JSResp) { + // connect a new connection to export keys + conf := Connection().getConnection(server) + if conf == nil { + resp.Msg = fmt.Sprintf("no connection profile named: %s", server) + return + } + var client redis.UniversalClient + var err error + var connConfig = conf.ConnectionConfig + connConfig.LastDB = db + if client, err = b.createRedisClient(connConfig); err != nil { + resp.Msg = err.Error() + return + } + ctx, cancelFunc := context.WithCancel(b.ctx) + defer client.Close() + defer cancelFunc() + + file, err := os.Open(path) + if err != nil { + resp.Msg = err.Error() + return + } + defer file.Close() + + reader := csv.NewReader(file) + + cancelEvent := "import:stop:" + path + runtime.EventsOnce(ctx, cancelEvent, func(data ...any) { + cancelFunc() + }) + processEvent := "importing:" + path + var line []string + var readErr error + var key, value []byte + var ttl int64 + var imported, ignored int64 + var canceled bool + startTime := time.Now().Add(-10 * time.Second) + for { + readErr = nil + + ttl = redis.KeepTTL + line, readErr = reader.Read() + if readErr != nil { + break + } + + if len(line) < 1 { + continue + } + if key, readErr = hex.DecodeString(line[0]); readErr != nil { + continue + } + if value, readErr = hex.DecodeString(line[1]); readErr != nil { + continue + } + // get ttl + if len(line) > 2 { + if ttl, readErr = strconv.ParseInt(line[2], 10, 64); readErr != nil { + ttl = redis.KeepTTL + } + } + if conflict == 0 { + readErr = client.RestoreReplace(ctx, string(key), time.Duration(ttl), string(value)).Err() + } else { + keyStr := string(key) + // go-redis may crash when batch calling restore + // use "exists" to filter first + if n, _ := client.Exists(ctx, keyStr).Result(); n <= 0 { + readErr = client.Restore(ctx, keyStr, time.Duration(ttl), string(value)).Err() + } else { + readErr = errors.New("key existed") + } + } + if readErr != nil { + // restore fail + ignored += 1 + } else { + imported += 1 + } + if errors.Is(readErr, context.Canceled) || canceled { + canceled = true + break + } + + if time.Now().Sub(startTime).Milliseconds() > 100 { + startTime = time.Now() + param := map[string]any{ + "imported": imported, + "ignored": ignored, + //"processing": string(key), + } + runtime.EventsEmit(b.ctx, processEvent, param) + // do some sleep to prevent blocking the Redis server + time.Sleep(10 * time.Millisecond) + } + } + + runtime.EventsOff(ctx, cancelEvent) + resp.Success = true + resp.Data = struct { + Canceled bool `json:"canceled"` + Imported int64 `json:"imported"` + Ignored int64 `json:"ignored"` + }{ + Canceled: canceled, + Imported: imported, + Ignored: ignored, + } + return +} + // FlushDB flush database func (b *browserService) FlushDB(connName string, db int, async bool) (resp types.JSResp) { item, err := b.getRedisClient(connName, db) diff --git a/backend/services/system_service.go b/backend/services/system_service.go index 3f73d7b..aa33cc7 100644 --- a/backend/services/system_service.go +++ b/backend/services/system_service.go @@ -44,10 +44,16 @@ func (s *systemService) Start(ctx context.Context) { } // SelectFile open file dialog to select a file -func (s *systemService) SelectFile(title string) (resp types.JSResp) { +func (s *systemService) SelectFile(title string, extensions []string) (resp types.JSResp) { + filters := sliceutil.Map(extensions, func(i int) runtime.FileFilter { + return runtime.FileFilter{ + Pattern: "*." + extensions[i], + } + }) filepath, err := runtime.OpenFileDialog(s.ctx, runtime.OpenDialogOptions{ Title: title, ShowHiddenFiles: true, + Filters: filters, }) if err != nil { resp.Msg = err.Error() diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 040ad45..e02b1b2 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -19,6 +19,7 @@ import { darkThemeOverrides, themeOverrides } from '@/utils/theme.js' import AboutDialog from '@/components/dialogs/AboutDialog.vue' import FlushDbDialog from '@/components/dialogs/FlushDbDialog.vue' import ExportKeyDialog from '@/components/dialogs/ExportKeyDialog.vue' +import ImportKeyDialog from '@/components/dialogs/ImportKeyDialog.vue' const prefStore = usePreferencesStore() const connectionStore = useConnectionStore() @@ -69,6 +70,7 @@ watch( + diff --git a/frontend/src/components/common/FileOpenInput.vue b/frontend/src/components/common/FileOpenInput.vue index 1656d87..a05c4bc 100644 --- a/frontend/src/components/common/FileOpenInput.vue +++ b/frontend/src/components/common/FileOpenInput.vue @@ -1,17 +1,18 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + {{ $t('common.cancel') }} + + {{ $t('dialogue.export.export') }} + + + + + + + diff --git a/frontend/src/components/icons/Export.vue b/frontend/src/components/icons/Export.vue index 9d35eba..bac6d7a 100644 --- a/frontend/src/components/icons/Export.vue +++ b/frontend/src/components/icons/Export.vue @@ -9,36 +9,24 @@ const props = defineProps({ - - - - - - - - + + + diff --git a/frontend/src/components/icons/Import.vue b/frontend/src/components/icons/Import.vue index bac6d7a..88fb2c2 100644 --- a/frontend/src/components/icons/Import.vue +++ b/frontend/src/components/icons/Import.vue @@ -11,19 +11,19 @@ const props = defineProps({ diff --git a/frontend/src/components/new_value/AddHashValue.vue b/frontend/src/components/new_value/AddHashValue.vue index 5803039..3fb6492 100644 --- a/frontend/src/components/new_value/AddHashValue.vue +++ b/frontend/src/components/new_value/AddHashValue.vue @@ -45,7 +45,7 @@ const onUpdate = (val) => { - + emit('update:type', val)"> diff --git a/frontend/src/components/new_value/NewZSetValue.vue b/frontend/src/components/new_value/NewZSetValue.vue index 33813bd..6a0a769 100644 --- a/frontend/src/components/new_value/NewZSetValue.vue +++ b/frontend/src/components/new_value/NewZSetValue.vue @@ -41,7 +41,7 @@ defineExpose({ - + { const moreOptions = computed(() => { return [ + { key: 'import', label: i18n.t('interface.import_key'), icon: render.renderIcon(Import, { strokeWidth: 3.5 }) }, { key: 'flush', label: i18n.t('interface.flush_db'), icon: render.renderIcon(Delete, { strokeWidth: 3.5 }) }, { key: 'disconnect', @@ -162,6 +164,10 @@ const onExportChecked = () => { browserTreeRef.value?.exportCheckedItems() } +const onImportData = () => { + dialogStore.openImportKeyDialog(props.server, props.db) +} + const onFlush = () => { dialogStore.openFlushDBDialog(props.server, props.db) } @@ -215,6 +221,9 @@ const onMatchInput = (matchVal, filterVal) => { const onSelectOptions = (select) => { switch (select) { + case 'import': + onImportData() + break case 'flush': onFlush() break diff --git a/frontend/src/langs/en-us.json b/frontend/src/langs/en-us.json index 0a7932f..eb3770f 100644 --- a/frontend/src/langs/en-us.json +++ b/frontend/src/langs/en-us.json @@ -75,6 +75,7 @@ "rename_key": "Rename Key", "delete_key": "Delete Key", "batch_delete_key": "Batch Delete Keys", + "import_key": "Import Key", "flush_db": "Flush Database", "check_mode": "Check Mode", "quit_check_mode": "Quit Check Mode", @@ -225,8 +226,7 @@ }, "cluster": { "title": "Cluster", - "enable": "Serve as Cluster Node", - "readonly": "Enables read-only commands on slave nodes" + "enable": "Serve as Cluster Node" } }, "group": { @@ -252,6 +252,7 @@ "field": { "new": "Add New Field", "new_item": "Add New Item", + "conflict_handle": "When Field Conflict", "overwrite_field": "Overwrite Existing Field", "ignore_field": "Ignore Existing Field", "insert_type": "Insert", @@ -272,12 +273,23 @@ "filter_pattern_tip": "* : Matches zero or more characters. For example, 'key*' matches all keys starting with 'key'.\n? : Matches a single character. For example, 'key?' matches 'key1', 'key2'.\n[] : Matches a single character within the specified range. For example, 'key[1-3]' matches keys like 'key1', 'key2', 'key3'.\n\\ : Escape character. To match *, ?, [, or ], use the backslash '\\' for escaping." }, "export": { - "name": "Export Keys", + "name": "Export Data", "export": "Export", "save_file": "Export Path", - "save_file_tip": "Select the export file save path", - "exporting": "Exporting key({index}/{count}): {key}", - "export_completed": "Export process has been completed, {success} successed, {fail} failed" + "save_file_tip": "Select the path to save exported file", + "exporting": "Exporting keys({index}/{count})", + "export_completed": "Export completed, {success} successes, {fail} failed" + }, + "import": { + "name": "Import Data", + "import": "Import", + "open_csv_file": "Import File", + "open_csv_file_tip": "Select the file for import", + "conflict_handle": "Handle Key Conflict", + "conflict_overwrite": "Overwrite", + "conflict_ignore": "Ignore", + "importing": "Importing Keys imported/overwrite:{imported} conflict/fail:{conflict}", + "import_completed": "Import completed, {success} successes, {ignored} failed" }, "ttl": { "title": "Set Key TTL" diff --git a/frontend/src/langs/zh-cn.json b/frontend/src/langs/zh-cn.json index b9a0dfc..6f80d70 100644 --- a/frontend/src/langs/zh-cn.json +++ b/frontend/src/langs/zh-cn.json @@ -75,6 +75,7 @@ "rename_key": "重命名键", "delete_key": "删除键", "batch_delete_key": "批量删除键", + "import_key": "导入数据", "flush_db": "清空数据库", "check_mode": "勾选模式", "quit_check_mode": "退出勾选模式", @@ -251,8 +252,9 @@ "field": { "new": "添加新字段", "new_item": "添加新元素", - "overwrite_field": "覆盖同名字段", - "ignore_field": "忽略同名字段", + "conflict_handle": "字段冲突处理", + "overwrite_field": "覆盖", + "ignore_field": "忽略", "insert_type": "插入类型", "append_item": "尾部追加", "prepend_item": "插入头部", @@ -271,13 +273,24 @@ "filter_pattern_tip": "*:匹配零个或多个字符。例如:\"key*\"匹配到以\"key\"开头的所有键\n?:匹配单个字符。例如:\"key?\"匹配\"key1\"、\"key2\"\n[ ]:匹配指定范围内的单个字符。例如:\"key[1-3]\"可以匹配类似于 \"key1\"、\"key2\"、\"key3\" 的键\n\\:转义字符。如果想要匹配 *、?、[、或],需要使用反斜杠\"\\\"进行转义" }, "export": { - "name": "导出键", + "name": "导出数据", "export": "确认导出", "save_file": "导出路径", - "save_file_tip": "选择保存文件路径", - "exporting": "正在导出键({index}/{count}):{key}", + "save_file_tip": "选择导出文件保存路径", + "exporting": "正在导出键({index}/{count})", "export_completed": "已完成导出操作,成功{success}个,失败{fail}个" }, + "import": { + "name": "导入数据", + "import": "确认导入", + "open_csv_file": "导入文件路径", + "open_csv_file_tip": "选择需要导入的文件", + "conflict_handle": "键冲突处理", + "conflict_overwrite": "覆盖", + "conflict_ignore": "忽略", + "importing": "正在导入数据 已导入/覆盖:{imported} 冲突/失败:{conflict}", + "import_completed": "已完成导入操作,成功{success}个,忽略{ignored}个" + }, "ttl": { "title": "设置键存活时间" }, diff --git a/frontend/src/stores/browser.js b/frontend/src/stores/browser.js index 570a844..bce6288 100644 --- a/frontend/src/stores/browser.js +++ b/frontend/src/stores/browser.js @@ -17,6 +17,7 @@ import { GetKeySummary, GetKeyType, GetSlowLogs, + ImportCSV, LoadAllKeys, LoadNextAllKeys, LoadNextKeys, @@ -1527,7 +1528,7 @@ const useBrowserStore = defineStore('browser', { * @return {Promise} */ async deleteKeys(server, db, keys) { - const delMsgRef = $message.loading('', { duration: 0, closable: true }) + const msgRef = $message.loading('', { duration: 0, closable: true }) let deleted = [] let failCount = 0 let canceled = false @@ -1542,13 +1543,13 @@ const useBrowserStore = defineStore('browser', { maxProgress = progress } const k = decodeRedisKey(processing) - delMsgRef.content = i18nGlobal.t('dialogue.deleting_key', { + msgRef.content = i18nGlobal.t('dialogue.deleting_key', { key: k, index: maxProgress, count: total, }) }) - delMsgRef.onClose = () => { + msgRef.onClose = () => { EventsEmit(cancelEvent) } const { data, success, msg } = await DeleteKeys(server, db, keys, serialNo) @@ -1560,7 +1561,7 @@ const useBrowserStore = defineStore('browser', { $message.error(msg) } } finally { - delMsgRef.destroy() + msgRef.destroy() EventsOff(eventName) // clear checked keys const tab = useTabStore() @@ -1608,7 +1609,7 @@ const useBrowserStore = defineStore('browser', { * @returns {Promise} */ async exportKeys(server, db, keys, path) { - const delMsgRef = $message.loading('', { duration: 0, closable: true }) + const msgRef = $message.loading('', { duration: 0, closable: true }) let exported = 0 let failCount = 0 let canceled = false @@ -1616,13 +1617,13 @@ const useBrowserStore = defineStore('browser', { try { EventsOn(eventName, ({ total, progress, processing }) => { // update export progress - delMsgRef.content = i18nGlobal.t('dialogue.export.exporting', { - key: decodeRedisKey(processing), + msgRef.content = i18nGlobal.t('dialogue.export.exporting', { + // key: decodeRedisKey(processing), index: progress, count: total, }) }) - delMsgRef.onClose = () => { + msgRef.onClose = () => { EventsEmit('export:stop:' + path) } const { data, success, msg } = await ExportKey(server, db, keys, path) @@ -1634,7 +1635,7 @@ const useBrowserStore = defineStore('browser', { $message.error(msg) } } finally { - delMsgRef.destroy() + msgRef.destroy() EventsOff(eventName) } if (canceled) { @@ -1653,6 +1654,52 @@ const useBrowserStore = defineStore('browser', { } }, + /** + * import multiple keys from csv file + * @param {string} server + * @param {number} db + * @param {string} path + * @param {int} conflict + * @return {Promise} + */ + async importKeysFromCSVFile(server, db, path, conflict) { + const msgRef = $message.loading('', { duration: 0, closable: true }) + let imported = 0 + let ignored = 0 + let canceled = false + const eventName = 'importing:' + path + try { + EventsOn(eventName, ({ imported = 0, ignored = 0 }) => { + // update export progress + msgRef.content = i18nGlobal.t('dialogue.import.importing', { + // key: decodeRedisKey(processing), + imported, + conflict: ignored, + }) + }) + msgRef.onClose = () => { + EventsEmit('import:stop:' + path) + } + const { data, success, msg } = await ImportCSV(server, db, path, conflict) + if (success) { + canceled = get(data, 'canceled', false) + imported = get(data, 'imported', 0) + ignored = get(data, 'ignored', 0) + } else { + $message.error(msg) + } + } finally { + msgRef.destroy() + EventsOff(eventName) + } + if (canceled) { + $message.info(i18nGlobal.t('dialogue.handle_cancel')) + } else { + // no fail + $message.success(i18nGlobal.t('dialogue.import.import_completed', { success: imported, ignored })) + } + }, + /** * flush database * @param server diff --git a/frontend/src/stores/dialog.js b/frontend/src/stores/dialog.js index 9bf1e0a..5e38e9c 100644 --- a/frontend/src/stores/dialog.js +++ b/frontend/src/stores/dialog.js @@ -70,6 +70,12 @@ const useDialogStore = defineStore('dialog', { }, exportKeyDialogVisible: false, + importKeyParam: { + server: '', + db: 0, + }, + importKeyDialogVisible: false, + flushDBParam: { server: '', db: 0, @@ -199,6 +205,20 @@ const useDialogStore = defineStore('dialog', { this.exportKeyDialogVisible = false }, + /** + * + * @param {string} server + * @param {number} db + */ + openImportKeyDialog(server, db) { + this.importKeyParam.server = server + this.importKeyParam.db = db + this.importKeyDialogVisible = true + }, + closeImportKeyDialog() { + this.importKeyDialogVisible = false + }, + openFlushDBDialog(server, db) { this.flushDBParam.server = server this.flushDBParam.db = db