perf: evenly divide the number of scans for each cluster node

This commit is contained in:
Lykin 2024-06-04 10:34:42 +08:00
parent eefa7b1346
commit 028a240f49
1 changed files with 9 additions and 3 deletions

View File

@ -445,7 +445,7 @@ func (b *browserService) scanKeys(ctx context.Context, client redis.UniversalCli
filterType := len(keyType) > 0 filterType := len(keyType) > 0
scanSize := int64(Preferences().GetScanSize()) scanSize := int64(Preferences().GetScanSize())
// define sub scan function // define sub scan function
scan := func(ctx context.Context, cli redis.UniversalClient, appendFunc func(k []any)) error { scan := func(ctx context.Context, cli redis.UniversalClient, count int64, appendFunc func(k []any)) error {
var loadedKey []string var loadedKey []string
var scanCount int64 var scanCount int64
for { for {
@ -475,16 +475,22 @@ func (b *browserService) scanKeys(ctx context.Context, client redis.UniversalCli
if cluster, ok := client.(*redis.ClusterClient); ok { if cluster, ok := client.(*redis.ClusterClient); ok {
// cluster mode // cluster mode
var mutex sync.Mutex var mutex sync.Mutex
var totalMaster int64
cluster.ForEachMaster(ctx, func(ctx context.Context, cli *redis.Client) error {
totalMaster += 1
return nil
})
partCount := count / max(totalMaster, 1)
err = cluster.ForEachMaster(ctx, func(ctx context.Context, cli *redis.Client) error { err = cluster.ForEachMaster(ctx, func(ctx context.Context, cli *redis.Client) error {
// FIXME: BUG? can not fully load in cluster mode? maybe remove the shared "cursor" // FIXME: BUG? can not fully load in cluster mode? maybe remove the shared "cursor"
return scan(ctx, cli, func(k []any) { return scan(ctx, cli, partCount, func(k []any) {
mutex.Lock() mutex.Lock()
keys = append(keys, k...) keys = append(keys, k...)
mutex.Unlock() mutex.Unlock()
}) })
}) })
} else { } else {
err = scan(ctx, client, func(k []any) { err = scan(ctx, client, count, func(k []any) {
keys = append(keys, k...) keys = append(keys, k...)
}) })
} }