perf: optimize the logic for bulk deletion of selected keys

This commit is contained in:
Lykin 2023-12-26 12:04:46 +08:00
parent e28bb57a25
commit 21a569d9bb
7 changed files with 53 additions and 49 deletions

View File

@ -1999,6 +1999,7 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo st
var deletedKeys = make([]any, 0, total) var deletedKeys = make([]any, 0, total)
var mutex sync.Mutex var mutex sync.Mutex
del := func(ctx context.Context, cli redis.UniversalClient) error { del := func(ctx context.Context, cli redis.UniversalClient) error {
startTime := time.Now()
for i, k := range ks { for i, k := range ks {
// emit progress per second // emit progress per second
param := map[string]any{ param := map[string]any{
@ -2006,12 +2007,13 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo st
"progress": i + 1, "progress": i + 1,
"processing": k, "processing": k,
} }
if i >= total-1 || time.Now().Sub(startTime).Milliseconds() > 100 {
startTime = time.Now()
runtime.EventsEmit(b.ctx, processEvent, param) runtime.EventsEmit(b.ctx, processEvent, param)
}
key := strutil.DecodeRedisKey(k) key := strutil.DecodeRedisKey(k)
delErr := cli.Del(ctx, key).Err() delErr := cli.Del(ctx, key).Err()
// do some sleep to prevent blocking the Redis server
time.Sleep(100 * time.Microsecond)
if err != nil { if err != nil {
failed.Add(1) failed.Add(1)
} else { } else {
@ -2024,6 +2026,8 @@ func (b *browserService) DeleteKeys(server string, db int, ks []any, serialNo st
canceled = true canceled = true
break break
} }
// do some sleep to prevent blocking the Redis server
time.Sleep(100 * time.Microsecond)
} }
return nil return nil
} }

View File

@ -143,7 +143,7 @@
"remove_tip": "{type} \"{name}\" will be deleted", "remove_tip": "{type} \"{name}\" will be deleted",
"remove_group_tip": "Group \"{name}\" and all connections in it will be deleted", "remove_group_tip": "Group \"{name}\" and all connections in it will be deleted",
"delete_key_succ": "\"{key}\" has been deleted", "delete_key_succ": "\"{key}\" has been deleted",
"deleting_key": "Deleting key({index}/{count}): {key}", "deleting_key": "Deleting key({index}/{count})",
"delete_completed": "Deletion process has been completed, {success} successed, {fail} failed", "delete_completed": "Deletion process has been completed, {success} successed, {fail} failed",
"rename_binary_key_fail": "Rename binary key name is unsupported", "rename_binary_key_fail": "Rename binary key name is unsupported",
"handle_succ": "Success!", "handle_succ": "Success!",

View File

@ -143,7 +143,7 @@
"remove_tip": "{type} \"{name}\" 将会被删除", "remove_tip": "{type} \"{name}\" 将会被删除",
"remove_group_tip": "分组 \"{name}\"及其所有连接将会被删除", "remove_group_tip": "分组 \"{name}\"及其所有连接将会被删除",
"delete_key_succ": "{key} 已被删除", "delete_key_succ": "{key} 已被删除",
"deleting_key": "正在删除键({index}/{count}){key}", "deleting_key": "正在删除键({index}/{count})",
"delete_completed": "已完成删除操作,成功{success}个,失败{fail}个", "delete_completed": "已完成删除操作,成功{success}个,失败{fail}个",
"rename_binary_key_fail": "不支持重命名二进制键名", "rename_binary_key_fail": "不支持重命名二进制键名",
"handle_succ": "操作成功", "handle_succ": "操作成功",

View File

@ -1,4 +1,4 @@
import { isEmpty, remove, sortBy, sortedIndexBy, sumBy } from 'lodash' import { isEmpty, remove, size, sortBy, sortedIndexBy, sumBy } from 'lodash'
import { ConnectionType } from '@/consts/connection_type.js' import { ConnectionType } from '@/consts/connection_type.js'
/** /**
@ -95,30 +95,6 @@ export class RedisNodeItem {
} }
} }
return false return false
// let count = 0
// if (!isEmpty(this.children)) {
// if (skipSort !== true) {
// this.sortChildren()
// }
//
// for (const elem of this.children) {
// elem.tidy(skipSort)
// count += elem.keyCount
// }
// } else {
// if (this.type === ConnectionType.RedisValue) {
// count += 1
// } else {
// // no children in db node or layer node, set count to 0
// count = 0
// }
// }
// if (this.keyCount !== count) {
// this.keyCount = count
// return true
// }
// return false
} }
sortChildren() { sortChildren() {
@ -142,15 +118,14 @@ export class RedisNodeItem {
/** /**
* *
* @param {{}} predicate * @param {{}} predicate
* @return {number}
*/ */
removeChild(predicate) { removeChild(predicate) {
if (this.type !== ConnectionType.RedisKey) { if (this.type !== ConnectionType.RedisKey && this.type !== ConnectionType.RedisDB) {
return return 0
} }
const removed = remove(this.children, predicate) const removed = remove(this.children, predicate)
if (this.children.length <= 0) { return size(removed)
// TODO: remove from parent if no children
}
} }
getChildren() { getChildren() {

View File

@ -25,7 +25,7 @@ export class RedisServerState {
* @param {string|null} typeFilter redis type filter * @param {string|null} typeFilter redis type filter
* @param {LoadingState} loadingState all loading state in opened connections map by server and LoadingState * @param {LoadingState} loadingState all loading state in opened connections map by server and LoadingState
* @param {KeyViewType} viewType view type selection for all opened connections group by 'server' * @param {KeyViewType} viewType view type selection for all opened connections group by 'server'
* @param {Map<string, RedisNodeItem>} nodeMap map nodes by "key#type" * @param {Map<string, RedisNodeItem>} nodeMap map nodes by "type#key"
*/ */
constructor({ constructor({
name, name,
@ -341,7 +341,7 @@ export class RedisServerState {
* @return * @return
*/ */
tidyNode(key, skipResort) { tidyNode(key, skipResort) {
const dbNode = this.getRoot() const rootNode = this.getRoot()
const keyParts = split(key, this.separator) const keyParts = split(key, this.separator)
const totalParts = size(keyParts) const totalParts = size(keyParts)
let node let node
@ -355,7 +355,7 @@ export class RedisServerState {
} }
} }
if (node == null) { if (node == null) {
node = dbNode node = rootNode
} }
const keyCountUpdated = node.tidy(skipResort) const keyCountUpdated = node.tidy(skipResort)
if (keyCountUpdated) { if (keyCountUpdated) {
@ -366,12 +366,23 @@ export class RedisServerState {
if (parentNode == null) { if (parentNode == null) {
break break
} }
parentNode.reCalcKeyCount() const count = parentNode.reCalcKeyCount()
if (count <= 0) {
let anceKeyNode = rootNode
// remove from ancestor node
if (i > 1) {
const anceKey = join(slice(keyParts, 0, i - 1), this.separator)
anceKeyNode = this.nodeMap.get(`${ConnectionType.RedisKey}/${anceKey}`)
}
if (anceKeyNode != null) {
anceKeyNode.removeChild({ type: ConnectionType.RedisKey, redisKey: parentKey })
}
}
} }
// update key count of db // update key count of db
const dbInst = this.databases[this.db] const dbInst = this.databases[this.db]
if (dbInst != null) { if (dbInst != null) {
dbInst.keyCount = dbNode.reCalcKeyCount() dbInst.keyCount = rootNode.reCalcKeyCount()
} }
} }
} }

View File

@ -1,5 +1,5 @@
import { defineStore } from 'pinia' import { defineStore } from 'pinia'
import { endsWith, get, isEmpty, map, size } from 'lodash' import { endsWith, get, isEmpty, map, now, size } from 'lodash'
import { import {
AddHashField, AddHashField,
AddListItem, AddListItem,
@ -46,6 +46,7 @@ import { EventsEmit, EventsOff, EventsOn } from 'wailsjs/runtime/runtime.js'
import { RedisNodeItem } from '@/objects/redisNodeItem.js' import { RedisNodeItem } from '@/objects/redisNodeItem.js'
import { RedisServerState } from '@/objects/redisServerState.js' import { RedisServerState } from '@/objects/redisServerState.js'
import { RedisDatabaseItem } from '@/objects/redisDatabaseItem.js' import { RedisDatabaseItem } from '@/objects/redisDatabaseItem.js'
import { timeout } from '@/utils/promise.js'
const useBrowserStore = defineStore('browser', { const useBrowserStore = defineStore('browser', {
/** /**
@ -1537,7 +1538,7 @@ const useBrowserStore = defineStore('browser', {
let canceled = false let canceled = false
const serialNo = Date.now().valueOf().toString() const serialNo = Date.now().valueOf().toString()
const eventName = 'deleting:' + serialNo const eventName = 'deleting:' + serialNo
const cancelEvent = 'deleting:' + serialNo const cancelEvent = 'delete:stop:' + serialNo
try { try {
let maxProgress = 0 let maxProgress = 0
EventsOn(eventName, ({ total, progress, processing }) => { EventsOn(eventName, ({ total, progress, processing }) => {
@ -1551,7 +1552,6 @@ const useBrowserStore = defineStore('browser', {
index: maxProgress, index: maxProgress,
count: total, count: total,
}) })
// this._deleteKeyNode(server, db, k, false)
}) })
delMsgRef.onClose = () => { delMsgRef.onClose = () => {
EventsEmit(cancelEvent) EventsEmit(cancelEvent)
@ -1573,12 +1573,6 @@ const useBrowserStore = defineStore('browser', {
} }
// refresh model data // refresh model data
const deletedCount = size(deleted) const deletedCount = size(deleted)
/** @type RedisServerState **/
const serverInst = this.servers[server]
if (serverInst != null) {
serverInst.tidyNode('', true)
serverInst.updateDBKeyCount(db, -deletedCount)
}
if (canceled) { if (canceled) {
$message.info(i18nGlobal.t('dialogue.handle_cancel')) $message.info(i18nGlobal.t('dialogue.handle_cancel'))
} else if (failCount <= 0) { } else if (failCount <= 0) {
@ -1591,6 +1585,23 @@ const useBrowserStore = defineStore('browser', {
// some fail // some fail
$message.warn(i18nGlobal.t('dialogue.delete_completed', { success: deletedCount, fail: failCount })) $message.warn(i18nGlobal.t('dialogue.delete_completed', { success: deletedCount, fail: failCount }))
} }
// update ui
timeout(100).then(async () => {
/** @type RedisServerState **/
const serverInst = this.servers[server]
if (serverInst != null) {
let start = now()
for (let i = 0; i < deleted.length; i++) {
serverInst.removeKeyNode(deleted[i], false)
if (now() - start > 300) {
await timeout(100)
start = now()
}
}
serverInst.tidyNode('', true)
serverInst.updateDBKeyCount(db, -deletedCount)
}
})
}, },
/** /**

View File

@ -0,0 +1,3 @@
export const timeout = (ms) => {
return new Promise((resolve) => setTimeout(resolve, ms))
}