perf: significantly improve batch deletion performance with pipeline #123
This commit is contained in:
parent
655cd539ca
commit
cdac3c4496
|
@ -2073,7 +2073,7 @@ func (b *browserService) DeleteOneKey(server string, db int, k any) (resp types.
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteKeys delete keys sync with notification
|
// DeleteKeys delete keys sync with notification
|
||||||
func (b *browserService) DeleteKeys(server string, db int, ks []any, notice bool, serialNo string) (resp types.JSResp) {
|
func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo string) (resp types.JSResp) {
|
||||||
// connect a new connection to export keys
|
// connect a new connection to export keys
|
||||||
conf := Connection().getConnection(server)
|
conf := Connection().getConnection(server)
|
||||||
if conf == nil {
|
if conf == nil {
|
||||||
|
@ -2096,47 +2096,28 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, notice bool
|
||||||
cancelStopEvent := runtime.EventsOnce(ctx, cancelEvent, func(data ...any) {
|
cancelStopEvent := runtime.EventsOnce(ctx, cancelEvent, func(data ...any) {
|
||||||
cancelFunc()
|
cancelFunc()
|
||||||
})
|
})
|
||||||
processEvent := "deleting:" + serialNo
|
|
||||||
total := len(ks)
|
total := len(ks)
|
||||||
var failed atomic.Int64
|
var failed atomic.Int64
|
||||||
var canceled bool
|
var canceled bool
|
||||||
var deletedKeys = make([]any, 0, total)
|
var deletedKeys = make([]any, 0, total)
|
||||||
var mutex sync.Mutex
|
var mutex sync.Mutex
|
||||||
del := func(ctx context.Context, cli redis.UniversalClient) error {
|
del := func(ctx context.Context, cli redis.UniversalClient) error {
|
||||||
startTime := time.Now().Add(-10 * time.Second)
|
const batchSize = 1000
|
||||||
supportUnlink := true
|
for i := 0; i < total; i += batchSize {
|
||||||
for i, k := range ks {
|
pipe := cli.Pipeline()
|
||||||
// emit progress per second
|
for j := 0; j < batchSize; j++ {
|
||||||
if notice && (i >= total-1 || time.Now().Sub(startTime).Milliseconds() > 100) {
|
if i+j < total {
|
||||||
startTime = time.Now()
|
pipe.Del(ctx, strutil.DecodeRedisKey(ks[i+j]))
|
||||||
param := map[string]any{
|
|
||||||
"total": total,
|
|
||||||
"progress": i + 1,
|
|
||||||
"processing": k,
|
|
||||||
}
|
|
||||||
runtime.EventsEmit(ctx, processEvent, param)
|
|
||||||
// do some sleep to prevent blocking the Redis server
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
key := strutil.DecodeRedisKey(k)
|
|
||||||
var delErr error
|
|
||||||
if supportUnlink {
|
|
||||||
if delErr = cli.Unlink(ctx, key).Err(); delErr != nil {
|
|
||||||
supportUnlink = false
|
|
||||||
delErr = nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !supportUnlink {
|
cmders, delErr := pipe.Exec(ctx)
|
||||||
delErr = cli.Del(ctx, key).Err()
|
for j, cmder := range cmders {
|
||||||
}
|
if cmder.(*redis.IntCmd).Val() != 1 {
|
||||||
if notice {
|
|
||||||
if delErr != nil {
|
|
||||||
failed.Add(1)
|
failed.Add(1)
|
||||||
} else {
|
} else {
|
||||||
// save deleted key
|
// save deleted key
|
||||||
mutex.Lock()
|
mutex.Lock()
|
||||||
deletedKeys = append(deletedKeys, k)
|
deletedKeys = append(deletedKeys, ks[i+j])
|
||||||
mutex.Unlock()
|
mutex.Unlock()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -74,7 +74,7 @@ const onConfirmDelete = async () => {
|
||||||
deleting.value = true
|
deleting.value = true
|
||||||
const { server, db, key, affectedKeys } = deleteForm
|
const { server, db, key, affectedKeys } = deleteForm
|
||||||
await nextTick()
|
await nextTick()
|
||||||
browserStore.deleteKeys(server, db, affectedKeys, !deleteForm.async).catch((e) => {})
|
browserStore.deleteKeys(server, db, affectedKeys).catch((e) => {})
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
$message.error(e.message)
|
$message.error(e.message)
|
||||||
return
|
return
|
||||||
|
@ -115,9 +115,9 @@ const onClose = () => {
|
||||||
required>
|
required>
|
||||||
<n-input v-model:value="deleteForm.key" placeholder="" @input="resetAffected" />
|
<n-input v-model:value="deleteForm.key" placeholder="" @input="resetAffected" />
|
||||||
</n-form-item>
|
</n-form-item>
|
||||||
<n-checkbox v-model:checked="deleteForm.async">
|
<!-- <n-checkbox v-model:checked="deleteForm.async">-->
|
||||||
{{ $t('dialogue.key.silent') }}
|
<!-- {{ $t('dialogue.key.silent') }}-->
|
||||||
</n-checkbox>
|
<!-- </n-checkbox>-->
|
||||||
<n-card
|
<n-card
|
||||||
v-if="deleteForm.showAffected"
|
v-if="deleteForm.showAffected"
|
||||||
:title="$t('dialogue.key.affected_key') + `(${size(deleteForm.affectedKeys)})`"
|
:title="$t('dialogue.key.affected_key') + `(${size(deleteForm.affectedKeys)})`"
|
||||||
|
|
|
@ -263,7 +263,6 @@
|
||||||
"confirm_delete_key": "Confirm Delete {num} Key(s)",
|
"confirm_delete_key": "Confirm Delete {num} Key(s)",
|
||||||
"async_delete": "Asynchronously Execute",
|
"async_delete": "Asynchronously Execute",
|
||||||
"async_delete_title": "Do not waiting for the operation's result",
|
"async_delete_title": "Do not waiting for the operation's result",
|
||||||
"silent": "Do not display deletion status in real time",
|
|
||||||
"confirm_flush": "I know what I'm doing!",
|
"confirm_flush": "I know what I'm doing!",
|
||||||
"confirm_flush_db": "Confirm Flush Database"
|
"confirm_flush_db": "Confirm Flush Database"
|
||||||
},
|
},
|
||||||
|
@ -271,8 +270,7 @@
|
||||||
"success": "\"{key}\" has been deleted",
|
"success": "\"{key}\" has been deleted",
|
||||||
"deleting": "Deleting",
|
"deleting": "Deleting",
|
||||||
"doing": "Deleting key({index}/{count})",
|
"doing": "Deleting key({index}/{count})",
|
||||||
"completed": "Deletion process has been completed",
|
"completed": "Deletion process has been completed, {success} successed, {fail} failed"
|
||||||
"completed_status": "{success} successed, {fail} failed"
|
|
||||||
},
|
},
|
||||||
"field": {
|
"field": {
|
||||||
"new": "Add New Field",
|
"new": "Add New Field",
|
||||||
|
|
|
@ -263,7 +263,6 @@
|
||||||
"confirm_delete_key": "确认删除{num}个键",
|
"confirm_delete_key": "确认删除{num}个键",
|
||||||
"async_delete": "异步执行",
|
"async_delete": "异步执行",
|
||||||
"async_delete_title": "不等待操作结果",
|
"async_delete_title": "不等待操作结果",
|
||||||
"silent": "不实时显示删除状态",
|
|
||||||
"confirm_flush": "我知道我正在执行的操作!",
|
"confirm_flush": "我知道我正在执行的操作!",
|
||||||
"confirm_flush_db": "确认清空数据库"
|
"confirm_flush_db": "确认清空数据库"
|
||||||
},
|
},
|
||||||
|
@ -271,8 +270,7 @@
|
||||||
"success": "{key} 已被删除",
|
"success": "{key} 已被删除",
|
||||||
"deleting": "正在删除",
|
"deleting": "正在删除",
|
||||||
"doing": "正在删除键({index}/{count})",
|
"doing": "正在删除键({index}/{count})",
|
||||||
"completed": "已完成删除操作",
|
"completed": "已完成删除操作,成功{success}个,失败{fail}个"
|
||||||
"completed_status": "成功{success}个,失败{fail}个"
|
|
||||||
},
|
},
|
||||||
"field": {
|
"field": {
|
||||||
"new": "添加新字段",
|
"new": "添加新字段",
|
||||||
|
|
|
@ -36,7 +36,7 @@ import {
|
||||||
UpdateZSetValue,
|
UpdateZSetValue,
|
||||||
} from 'wailsjs/go/services/browserService.js'
|
} from 'wailsjs/go/services/browserService.js'
|
||||||
import useTabStore from 'stores/tab.js'
|
import useTabStore from 'stores/tab.js'
|
||||||
import { decodeRedisKey, nativeRedisKey } from '@/utils/key_convert.js'
|
import { nativeRedisKey } from '@/utils/key_convert.js'
|
||||||
import { BrowserTabType } from '@/consts/browser_tab_type.js'
|
import { BrowserTabType } from '@/consts/browser_tab_type.js'
|
||||||
import { KeyViewType } from '@/consts/key_view_type.js'
|
import { KeyViewType } from '@/consts/key_view_type.js'
|
||||||
import { ConnectionType } from '@/consts/connection_type.js'
|
import { ConnectionType } from '@/consts/connection_type.js'
|
||||||
|
@ -1568,7 +1568,7 @@ const useBrowserStore = defineStore('browser', {
|
||||||
// $message.error(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
|
// $message.error(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
|
||||||
// } else {
|
// } else {
|
||||||
// // some fail
|
// // some fail
|
||||||
// $message.warn(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
|
// $message.warning(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
|
||||||
// }
|
// }
|
||||||
},
|
},
|
||||||
|
|
||||||
|
@ -1614,33 +1614,19 @@ const useBrowserStore = defineStore('browser', {
|
||||||
* @param {string} server
|
* @param {string} server
|
||||||
* @param {number} db
|
* @param {number} db
|
||||||
* @param {string[]|number[][]} keys
|
* @param {string[]|number[][]} keys
|
||||||
* @param {boolean} notice
|
|
||||||
* @return {Promise<void>}
|
* @return {Promise<void>}
|
||||||
*/
|
*/
|
||||||
async deleteKeys(server, db, keys, notice) {
|
async deleteKeys(server, db, keys) {
|
||||||
const msgRef = $message.loading(i18nGlobal.t('dialogue.delete.deleting'), { duration: 0, closable: true })
|
const msgRef = $message.loading(i18nGlobal.t('dialogue.delete.deleting'), { duration: 0, closable: true })
|
||||||
let deleted = []
|
let deleted = []
|
||||||
let failCount = 0
|
let failCount = 0
|
||||||
let canceled = false
|
let canceled = false
|
||||||
const serialNo = Date.now().valueOf().toString()
|
const serialNo = Date.now().valueOf().toString()
|
||||||
let maxProgress = 0
|
|
||||||
const cancelEventFn = EventsOn('deleting:' + serialNo, ({ total, progress, processing }) => {
|
|
||||||
// update delete progress
|
|
||||||
if (progress > maxProgress) {
|
|
||||||
maxProgress = progress
|
|
||||||
}
|
|
||||||
const k = decodeRedisKey(processing)
|
|
||||||
msgRef.content = i18nGlobal.t('dialogue.delete.doing', {
|
|
||||||
key: k,
|
|
||||||
index: maxProgress,
|
|
||||||
count: total,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
msgRef.onClose = () => {
|
msgRef.onClose = () => {
|
||||||
EventsEmit('delete:stop:' + serialNo)
|
EventsEmit('delete:stop:' + serialNo)
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const { data, success, msg } = await DeleteKeys(server, db, keys, notice, serialNo)
|
const { success, msg, data } = await DeleteKeys(server, db, keys, serialNo)
|
||||||
if (success) {
|
if (success) {
|
||||||
canceled = get(data, 'canceled', false)
|
canceled = get(data, 'canceled', false)
|
||||||
deleted = get(data, 'deleted', [])
|
deleted = get(data, 'deleted', [])
|
||||||
|
@ -1649,7 +1635,6 @@ const useBrowserStore = defineStore('browser', {
|
||||||
$message.error(msg)
|
$message.error(msg)
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
cancelEventFn()
|
|
||||||
msgRef.destroy()
|
msgRef.destroy()
|
||||||
// clear checked keys
|
// clear checked keys
|
||||||
const tab = useTabStore()
|
const tab = useTabStore()
|
||||||
|
@ -1661,40 +1646,13 @@ const useBrowserStore = defineStore('browser', {
|
||||||
$message.info(i18nGlobal.t('dialogue.handle_cancel'))
|
$message.info(i18nGlobal.t('dialogue.handle_cancel'))
|
||||||
} else if (failCount <= 0) {
|
} else if (failCount <= 0) {
|
||||||
// no fail
|
// no fail
|
||||||
let msg = i18nGlobal.t('dialogue.delete.completed')
|
$message.success(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
|
||||||
if (notice) {
|
|
||||||
msg +=
|
|
||||||
'\n' +
|
|
||||||
i18nGlobal.t('dialogue.delete.completed_status', {
|
|
||||||
success: deletedCount,
|
|
||||||
fail: failCount,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
$message.success(msg)
|
|
||||||
} else if (failCount >= deletedCount) {
|
} else if (failCount >= deletedCount) {
|
||||||
// all fail
|
// all fail
|
||||||
let msg = i18nGlobal.t('dialogue.delete.completed')
|
$message.error(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
|
||||||
if (notice) {
|
|
||||||
msg +=
|
|
||||||
'\n' +
|
|
||||||
i18nGlobal.t('dialogue.delete.completed_status', {
|
|
||||||
success: deletedCount,
|
|
||||||
fail: failCount,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
$message.error(msg)
|
|
||||||
} else {
|
} else {
|
||||||
// some fail
|
// some fail
|
||||||
let msg = i18nGlobal.t('dialogue.delete.completed')
|
$message.warning(i18nGlobal.t('dialogue.delete.completed', { success: deletedCount, fail: failCount }))
|
||||||
if (notice) {
|
|
||||||
msg +=
|
|
||||||
'\n' +
|
|
||||||
i18nGlobal.t('dialogue.delete.completed_status', {
|
|
||||||
success: deletedCount,
|
|
||||||
fail: failCount,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
$message.warn(msg)
|
|
||||||
}
|
}
|
||||||
// update ui
|
// update ui
|
||||||
timeout(100).then(async () => {
|
timeout(100).then(async () => {
|
||||||
|
@ -1765,7 +1723,12 @@ const useBrowserStore = defineStore('browser', {
|
||||||
$message.error(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount }))
|
$message.error(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount }))
|
||||||
} else {
|
} else {
|
||||||
// some fail
|
// some fail
|
||||||
$message.warn(i18nGlobal.t('dialogue.export.export_completed', { success: exported, fail: failCount }))
|
$message.warning(
|
||||||
|
i18nGlobal.t('dialogue.export.export_completed', {
|
||||||
|
success: exported,
|
||||||
|
fail: failCount,
|
||||||
|
}),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue