From cdac3c4496dd152917cf9e1f4ade730d5ad9b1b0 Mon Sep 17 00:00:00 2001
From: Lykin <137850705+tiny-craft@users.noreply.github.com>
Date: Tue, 23 Jan 2024 17:37:00 +0800
Subject: [PATCH] perf: significantly improve batch deletion performance with
pipeline #123
---
backend/services/browser_service.go | 41 ++++--------
.../components/dialogs/DeleteKeyDialog.vue | 8 +--
frontend/src/langs/en-us.json | 4 +-
frontend/src/langs/zh-cn.json | 4 +-
frontend/src/stores/browser.js | 63 ++++---------------
5 files changed, 30 insertions(+), 90 deletions(-)
diff --git a/backend/services/browser_service.go b/backend/services/browser_service.go
index dfb4e86..0de4e34 100644
--- a/backend/services/browser_service.go
+++ b/backend/services/browser_service.go
@@ -2073,7 +2073,7 @@ func (b *browserService) DeleteOneKey(server string, db int, k any) (resp types.
}
// DeleteKeys delete keys sync with notification
-func (b *browserService) DeleteKeys(server string, db int, ks []any, notice bool, serialNo string) (resp types.JSResp) {
+func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo string) (resp types.JSResp) {
// connect a new connection to export keys
conf := Connection().getConnection(server)
if conf == nil {
@@ -2096,47 +2096,28 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, notice bool
cancelStopEvent := runtime.EventsOnce(ctx, cancelEvent, func(data ...any) {
cancelFunc()
})
- processEvent := "deleting:" + serialNo
total := len(ks)
var failed atomic.Int64
var canceled bool
var deletedKeys = make([]any, 0, total)
var mutex sync.Mutex
del := func(ctx context.Context, cli redis.UniversalClient) error {
- startTime := time.Now().Add(-10 * time.Second)
- supportUnlink := true
- for i, k := range ks {
- // emit progress per second
- if notice && (i >= total-1 || time.Now().Sub(startTime).Milliseconds() > 100) {
- startTime = time.Now()
- param := map[string]any{
- "total": total,
- "progress": i + 1,
- "processing": k,
- }
- runtime.EventsEmit(ctx, processEvent, param)
- // do some sleep to prevent blocking the Redis server
- time.Sleep(10 * time.Millisecond)
- }
-
- key := strutil.DecodeRedisKey(k)
- var delErr error
- if supportUnlink {
- if delErr = cli.Unlink(ctx, key).Err(); delErr != nil {
- supportUnlink = false
- delErr = nil
+ const batchSize = 1000
+ for i := 0; i < total; i += batchSize {
+ pipe := cli.Pipeline()
+ for j := 0; j < batchSize; j++ {
+ if i+j < total {
+ pipe.Del(ctx, strutil.DecodeRedisKey(ks[i+j]))
}
}
- if !supportUnlink {
- delErr = cli.Del(ctx, key).Err()
- }
- if notice {
- if delErr != nil {
+ cmders, delErr := pipe.Exec(ctx)
+ for j, cmder := range cmders {
+ if cmder.(*redis.IntCmd).Val() != 1 {
failed.Add(1)
} else {
// save deleted key
mutex.Lock()
- deletedKeys = append(deletedKeys, k)
+ deletedKeys = append(deletedKeys, ks[i+j])
mutex.Unlock()
}
}
diff --git a/frontend/src/components/dialogs/DeleteKeyDialog.vue b/frontend/src/components/dialogs/DeleteKeyDialog.vue
index 0bb83ec..b5e2c0a 100644
--- a/frontend/src/components/dialogs/DeleteKeyDialog.vue
+++ b/frontend/src/components/dialogs/DeleteKeyDialog.vue
@@ -74,7 +74,7 @@ const onConfirmDelete = async () => {
deleting.value = true
const { server, db, key, affectedKeys } = deleteForm
await nextTick()
- browserStore.deleteKeys(server, db, affectedKeys, !deleteForm.async).catch((e) => {})
+ browserStore.deleteKeys(server, db, affectedKeys).catch((e) => {})
} catch (e) {
$message.error(e.message)
return
@@ -115,9 +115,9 @@ const onClose = () => {
required>
-
- {{ $t('dialogue.key.silent') }}
-
+
+
+
}
*/
- async deleteKeys(server, db, keys, notice) {
+ async deleteKeys(server, db, keys) {
const msgRef = $message.loading(i18nGlobal.t('dialogue.delete.deleting'), { duration: 0, closable: true })
let deleted = []
let failCount = 0
let canceled = false
const serialNo = Date.now().valueOf().toString()
- let maxProgress = 0
- const cancelEventFn = EventsOn('deleting:' + serialNo, ({ total, progress, processing }) => {
- // update delete progress
- if (progress > maxProgress) {
- maxProgress = progress
- }
- const k = decodeRedisKey(processing)
- msgRef.content = i18nGlobal.t('dialogue.delete.doing', {
- key: k,
- index: maxProgress,
- count: total,
- })
- })
msgRef.onClose = () => {
EventsEmit('delete:stop:' + serialNo)
}
try {
- const { data, success, msg } = await DeleteKeys(server, db, keys, notice, serialNo)
+ const { success, msg, data } = await DeleteKeys(server, db, keys, serialNo)
if (success) {
canceled = get(data, 'canceled', false)
deleted = get(data, 'deleted', [])
@@ -1649,7 +1635,6 @@ const useBrowserStore = defineStore('browser', {
$message.error(msg)
}
} finally {
- cancelEventFn()
msgRef.destroy()
// clear checked keys
const tab = useTabStore()
@@ -1661,40 +1646,13 @@ const useBrowserStore = defineStore('browser', {
$message.info(i18nGlobal.t('dialogue.handle_cancel'))
} else if (failCount <= 0) {
// no fail
- let msg = i18nGlobal.t('dialogue.delete.completed')
- if (notice) {
- msg +=
- '\n' +
- i18nGlobal.t('dialogue.delete.completed_status', {
- success: deletedCount,
- fail: failCount,
- })
- }
- $message.success(msg)
+ $message.success(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
} else if (failCount >= deletedCount) {
// all fail
- let msg = i18nGlobal.t('dialogue.delete.completed')
- if (notice) {
- msg +=
- '\n' +
- i18nGlobal.t('dialogue.delete.completed_status', {
- success: deletedCount,
- fail: failCount,
- })
- }
- $message.error(msg)
+ $message.error(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
} else {
// some fail
- let msg = i18nGlobal.t('dialogue.delete.completed')
- if (notice) {
- msg +=
- '\n' +
- i18nGlobal.t('dialogue.delete.completed_status', {
- success: deletedCount,
- fail: failCount,
- })
- }
- $message.warn(msg)
+ $message.warning(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
}
// update ui
timeout(100).then(async () => {
@@ -1765,7 +1723,12 @@ const useBrowserStore = defineStore('browser', {
$message.error(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount }))
} else {
// some fail
- $message.warn(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount }))
+ $message.warning(
+ i18nGlobal.t('dialogue.export.export_completed', {
+ success: exported,
+ fail: failCount,
+ }),
+ )
}
},