lingling
/
tiny-rdm
mirror of https://github.com/tiny-craft/tiny-rdm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

perf: significantly improve batch deletion performance with pipeline #123

This commit is contained in:
Lykin 2024-01-23 17:37:00 +08:00
parent 655cd539ca
commit cdac3c4496
5 changed files with 30 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
backend/services/browser_service.go
View File

@ -2073,7 +2073,7 @@ func (b *browserService) DeleteOneKey(server string, db int, k any) (resp types.
} }
// DeleteKeys delete keys sync with notification // DeleteKeys delete keys sync with notification
func (b *browserService) DeleteKeys(server string, db int, ks []any, notice bool, serialNo string) (resp types.JSResp) { func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo string) (resp types.JSResp) {
// connect a new connection to export keys // connect a new connection to export keys
conf := Connection().getConnection(server) conf := Connection().getConnection(server)
if conf == nil { if conf == nil {
@ -2096,47 +2096,28 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, notice bool
cancelStopEvent := runtime.EventsOnce(ctx, cancelEvent, func(data ...any) { cancelStopEvent := runtime.EventsOnce(ctx, cancelEvent, func(data ...any) {
cancelFunc() cancelFunc()
}) })
processEvent := "deleting:" + serialNo
total := len(ks) total := len(ks)
var failed atomic.Int64 var failed atomic.Int64
var canceled bool var canceled bool
var deletedKeys = make([]any, 0, total) var deletedKeys = make([]any, 0, total)
var mutex sync.Mutex var mutex sync.Mutex
del := func(ctx context.Context, cli redis.UniversalClient) error { del := func(ctx context.Context, cli redis.UniversalClient) error {
startTime := time.Now().Add(-10 * time.Second) const batchSize = 1000
supportUnlink := true for i := 0; i < total; i += batchSize {
for i, k := range ks { pipe := cli.Pipeline()
// emit progress per second for j := 0; j < batchSize; j++ {
if notice && (i >= total-1 || time.Now().Sub(startTime).Milliseconds() > 100) { if i+j < total {
startTime = time.Now() pipe.Del(ctx, strutil.DecodeRedisKey(ks[i+j]))
param := map[string]any{
"total": total,
"progress": i + 1,
"processing": k,
}
runtime.EventsEmit(ctx, processEvent, param)
// do some sleep to prevent blocking the Redis server
time.Sleep(10 * time.Millisecond)
}
key := strutil.DecodeRedisKey(k)
var delErr error
if supportUnlink {
if delErr = cli.Unlink(ctx, key).Err(); delErr != nil {
supportUnlink = false
delErr = nil
} }
} }
if !supportUnlink { cmders, delErr := pipe.Exec(ctx)
delErr = cli.Del(ctx, key).Err() for j, cmder := range cmders {
} if cmder.(*redis.IntCmd).Val() != 1 {
if notice {
if delErr != nil {
failed.Add(1) failed.Add(1)
} else { } else {
// save deleted key // save deleted key
mutex.Lock() mutex.Lock()
deletedKeys = append(deletedKeys, k) deletedKeys = append(deletedKeys, ks[i+j])
mutex.Unlock() mutex.Unlock()
} }
} }

8
frontend/src/components/dialogs/DeleteKeyDialog.vue
View File

@ -74,7 +74,7 @@ const onConfirmDelete = async () => {
deleting.value = true deleting.value = true
const { server, db, key, affectedKeys } = deleteForm const { server, db, key, affectedKeys } = deleteForm
await nextTick() await nextTick()
browserStore.deleteKeys(server, db, affectedKeys, !deleteForm.async).catch((e) => {}) browserStore.deleteKeys(server, db, affectedKeys).catch((e) => {})
} catch (e) { } catch (e) {
$message.error(e.message) $message.error(e.message)
return return
@ -115,9 +115,9 @@ const onClose = () => {
required> required>
<n-input v-model:value="deleteForm.key" placeholder="" @input="resetAffected" /> <n-input v-model:value="deleteForm.key" placeholder="" @input="resetAffected" />
</n-form-item> </n-form-item>
<n-checkbox v-model:checked="deleteForm.async"> <!-- <n-checkbox v-model:checked="deleteForm.async">-->
{{ $t('dialogue.key.silent') }} <!-- {{ $t('dialogue.key.silent') }}-->
</n-checkbox> <!-- </n-checkbox>-->
<n-card <n-card
v-if="deleteForm.showAffected" v-if="deleteForm.showAffected"
:title="$t('dialogue.key.affected_key') + `(${size(deleteForm.affectedKeys)})`" :title="$t('dialogue.key.affected_key') + `(${size(deleteForm.affectedKeys)})`"

4
frontend/src/langs/en-us.json
View File

@ -263,7 +263,6 @@
"confirm_delete_key": "Confirm Delete {num} Key(s)", "confirm_delete_key": "Confirm Delete {num} Key(s)",
"async_delete": "Asynchronously Execute", "async_delete": "Asynchronously Execute",
"async_delete_title": "Do not waiting for the operation's result", "async_delete_title": "Do not waiting for the operation's result",
"silent": "Do not display deletion status in real time",
"confirm_flush": "I know what I'm doing!", "confirm_flush": "I know what I'm doing!",
"confirm_flush_db": "Confirm Flush Database" "confirm_flush_db": "Confirm Flush Database"
}, },
@ -271,8 +270,7 @@
"success": "\"{key}\" has been deleted", "success": "\"{key}\" has been deleted",
"deleting": "Deleting", "deleting": "Deleting",
"doing": "Deleting key({index}/{count})", "doing": "Deleting key({index}/{count})",
"completed": "Deletion process has been completed", "completed": "Deletion process has been completed, {success} successed, {fail} failed"
"completed_status": "{success} successed, {fail} failed"
}, },
"field": { "field": {
"new": "Add New Field", "new": "Add New Field",

4
frontend/src/langs/zh-cn.json
View File

@ -263,7 +263,6 @@
"confirm_delete_key": "确认删除{num}个键", "confirm_delete_key": "确认删除{num}个键",
"async_delete": "异步执行", "async_delete": "异步执行",
"async_delete_title": "不等待操作结果", "async_delete_title": "不等待操作结果",
"silent": "不实时显示删除状态",
"confirm_flush": "我知道我正在执行的操作！", "confirm_flush": "我知道我正在执行的操作！",
"confirm_flush_db": "确认清空数据库" "confirm_flush_db": "确认清空数据库"
}, },
@ -271,8 +270,7 @@
"success": "{key} 已被删除", "success": "{key} 已被删除",
"deleting": "正在删除", "deleting": "正在删除",
"doing": "正在删除键({index}/{count})", "doing": "正在删除键({index}/{count})",
"completed": "已完成删除操作", "completed": "已完成删除操作，成功{success}个，失败{fail}个"
"completed_status": "成功{success}个，失败{fail}个"
}, },
"field": { "field": {
"new": "添加新字段", "new": "添加新字段",

63
frontend/src/stores/browser.js
View File

@ -36,7 +36,7 @@ import {
UpdateZSetValue, UpdateZSetValue,
} from 'wailsjs/go/services/browserService.js' } from 'wailsjs/go/services/browserService.js'
import useTabStore from 'stores/tab.js' import useTabStore from 'stores/tab.js'
import { decodeRedisKey, nativeRedisKey } from '@/utils/key_convert.js' import { nativeRedisKey } from '@/utils/key_convert.js'
import { BrowserTabType } from '@/consts/browser_tab_type.js' import { BrowserTabType } from '@/consts/browser_tab_type.js'
import { KeyViewType } from '@/consts/key_view_type.js' import { KeyViewType } from '@/consts/key_view_type.js'
import { ConnectionType } from '@/consts/connection_type.js' import { ConnectionType } from '@/consts/connection_type.js'
@ -1568,7 +1568,7 @@ const useBrowserStore = defineStore('browser', {
// $message.error(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount })) // $message.error(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
// } else { // } else {
// // some fail // // some fail
// $message.warn(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount })) // $message.warning(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
// } // }
}, },
@ -1614,33 +1614,19 @@ const useBrowserStore = defineStore('browser', {
* @param {string} server * @param {string} server
* @param {number} db * @param {number} db
* @param {string[]|number[][]} keys * @param {string[]|number[][]} keys
* @param {boolean} notice
* @return {Promise<void>} * @return {Promise<void>}
*/ */
async deleteKeys(server, db, keys, notice) { async deleteKeys(server, db, keys) {
const msgRef = $message.loading(i18nGlobal.t('dialogue.delete.deleting'), { duration: 0, closable: true }) const msgRef = $message.loading(i18nGlobal.t('dialogue.delete.deleting'), { duration: 0, closable: true })
let deleted = [] let deleted = []
let failCount = 0 let failCount = 0
let canceled = false let canceled = false
const serialNo = Date.now().valueOf().toString() const serialNo = Date.now().valueOf().toString()
let maxProgress = 0
const cancelEventFn = EventsOn('deleting:' + serialNo, ({ total, progress, processing }) => {
// update delete progress
if (progress > maxProgress) {
maxProgress = progress
}
const k = decodeRedisKey(processing)
msgRef.content = i18nGlobal.t('dialogue.delete.doing', {
key: k,
index: maxProgress,
count: total,
})
})
msgRef.onClose = () => { msgRef.onClose = () => {
EventsEmit('delete:stop:' + serialNo) EventsEmit('delete:stop:' + serialNo)
} }
try { try {
const { data, success, msg } = await DeleteKeys(server, db, keys, notice, serialNo) const { success, msg, data } = await DeleteKeys(server, db, keys, serialNo)
if (success) { if (success) {
canceled = get(data, 'canceled', false) canceled = get(data, 'canceled', false)
deleted = get(data, 'deleted', []) deleted = get(data, 'deleted', [])
@ -1649,7 +1635,6 @@ const useBrowserStore = defineStore('browser', {
$message.error(msg) $message.error(msg)
} }
} finally { } finally {
cancelEventFn()
msgRef.destroy() msgRef.destroy()
// clear checked keys // clear checked keys
const tab = useTabStore() const tab = useTabStore()
@ -1661,40 +1646,13 @@ const useBrowserStore = defineStore('browser', {
$message.info(i18nGlobal.t('dialogue.handle_cancel')) $message.info(i18nGlobal.t('dialogue.handle_cancel'))
} else if (failCount <= 0) { } else if (failCount <= 0) {
// no fail // no fail
let msg = i18nGlobal.t('dialogue.delete.completed') $message.success(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
if (notice) {
msg +=
'\n' +
i18nGlobal.t('dialogue.delete.completed_status', {
success: deletedCount,
fail: failCount,
})
}
$message.success(msg)
} else if (failCount >= deletedCount) { } else if (failCount >= deletedCount) {
// all fail // all fail
let msg = i18nGlobal.t('dialogue.delete.completed') $message.error(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
if (notice) {
msg +=
'\n' +
i18nGlobal.t('dialogue.delete.completed_status', {
success: deletedCount,
fail: failCount,
})
}
$message.error(msg)
} else { } else {
// some fail // some fail
let msg = i18nGlobal.t('dialogue.delete.completed') $message.warning(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
if (notice) {
msg +=
'\n' +
i18nGlobal.t('dialogue.delete.completed_status', {
success: deletedCount,
fail: failCount,
})
}
$message.warn(msg)
} }
// update ui // update ui
timeout(100).then(async () => { timeout(100).then(async () => {
@ -1765,7 +1723,12 @@ const useBrowserStore = defineStore('browser', {
$message.error(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount })) $message.error(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount }))
} else { } else {
// some fail // some fail
$message.warn(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount })) $message.warning(
i18nGlobal.t('dialogue.export.export_completed', {
success: exported,
fail: failCount,
}),
)
} }
}, },