From 3e04b4d7e7a44dab884400f1ca43fbf25108ecd4 Mon Sep 17 00:00:00 2001 From: cauchywei Date: Sun, 19 Apr 2026 20:29:58 +0800 Subject: [PATCH 1/2] add Valkey compatibility regression coverage --- cases/hash_as_listpack_with_hfe.aof | 16 +++--- cases/hash_with_hfe.aof | 72 ++++++++++++------------- core/valkey_test.go | 84 +++++++++++++++++++++++++++++ helper/converter_test.go | 77 ++++++++++++++++++++++++++ helper/resp.go | 38 ++++++++----- 5 files changed, 231 insertions(+), 56 deletions(-) create mode 100644 core/valkey_test.go diff --git a/cases/hash_as_listpack_with_hfe.aof b/cases/hash_as_listpack_with_hfe.aof index ee4cfad..8fe6258 100644 --- a/cases/hash_as_listpack_with_hfe.aof +++ b/cases/hash_as_listpack_with_hfe.aof @@ -8,26 +8,26 @@ F1 $2 V1 $2 -F3 -$2 -V3 -$2 F2 $2 V2 +$2 +F3 +$2 +V3 *6 $10 HPEXPIREAT $12 listpack-hfe $13 -2755484483878 +2755482478325 $6 FIELDS $1 1 $2 -F3 +F1 *5 $8 HPERSIST @@ -45,10 +45,10 @@ HPEXPIREAT $12 listpack-hfe $13 -2755482478325 +2755484483878 $6 FIELDS $1 1 $2 -F1 +F3 diff --git a/cases/hash_with_hfe.aof b/cases/hash_with_hfe.aof index 8983988..e86c13d 100644 --- a/cases/hash_with_hfe.aof +++ b/cases/hash_with_hfe.aof @@ -4,48 +4,50 @@ HMSET $8 hash-hfe $2 -F4 +F1 $2 -V4 +V1 $2 -F7 +F2 $2 -V7 +V2 $2 -F8 +F3 $2 -V8 +V3 $2 -F2 +F4 $2 -V2 +V4 $2 F5 $2 V5 $2 -F3 +F6 $2 -V3 +V6 $2 -F1 +F7 $2 -V1 +V7 $2 -F6 +F8 $2 -V6 -*5 -$8 -HPERSIST +V8 +*6 +$10 +HPEXPIREAT $8 hash-hfe +$13 +2755482424661 $6 FIELDS $1 1 $2 -F8 +F1 *6 $10 HPEXPIREAT @@ -59,43 +61,41 @@ $1 1 $2 F2 -*5 -$8 -HPERSIST +*6 +$10 +HPEXPIREAT $8 hash-hfe +$13 +2755484433842 $6 FIELDS $1 1 $2 -F5 -*6 -$10 -HPEXPIREAT +F3 +*5 +$8 +HPERSIST $8 hash-hfe -$13 -2755484433842 $6 FIELDS $1 1 $2 -F3 -*6 -$10 -HPEXPIREAT +F4 +*5 +$8 +HPERSIST $8 hash-hfe -$13 -2755482424661 $6 FIELDS $1 1 $2 -F1 +F5 *5 $8 HPERSIST @@ -117,7 +117,7 @@ FIELDS $1 1 $2 -F4 +F7 *5 $8 HPERSIST @@ -128,4 +128,4 @@ FIELDS $1 1 $2 -F7 +F8 diff --git a/core/valkey_test.go b/core/valkey_test.go new file mode 100644 index 0000000..2034c7c --- /dev/null +++ b/core/valkey_test.go @@ -0,0 +1,84 @@ +package core + +import ( + "bytes" + "testing" + + "github.com/hdt3213/rdb/model" +) + +func TestValkeyClusterMetadataOpcodes(t *testing.T) { + buf := bytes.NewBuffer(nil) + enc := NewEncoderValkey(buf) + if err := enc.WriteHeader(); err != nil { + t.Fatalf("write header: %v", err) + } + if err := enc.WriteDBHeader(0, 1, 0); err != nil { + t.Fatalf("write db header: %v", err) + } + + if err := enc.write([]byte{opCodeSlotInfo}); err != nil { + t.Fatalf("write slot info opcode: %v", err) + } + if err := enc.writeLength(42); err != nil { + t.Fatalf("write slot id: %v", err) + } + if err := enc.writeLength(3); err != nil { + t.Fatalf("write slot size: %v", err) + } + if err := enc.writeLength(1); err != nil { + t.Fatalf("write expires slot size: %v", err) + } + + if err := enc.write([]byte{opCodeSlotImport}); err != nil { + t.Fatalf("write slot import opcode: %v", err) + } + if err := enc.writeString("job-1"); err != nil { + t.Fatalf("write slot import job: %v", err) + } + if err := enc.writeLength(2); err != nil { + t.Fatalf("write slot range count: %v", err) + } + if err := enc.writeLength(10); err != nil { + t.Fatalf("write slot range start: %v", err) + } + if err := enc.writeLength(11); err != nil { + t.Fatalf("write slot range end: %v", err) + } + if err := enc.writeLength(20); err != nil { + t.Fatalf("write slot range start: %v", err) + } + if err := enc.writeLength(25); err != nil { + t.Fatalf("write slot range end: %v", err) + } + + if err := enc.WriteStringObject("cluster-key", []byte("value")); err != nil { + t.Fatalf("write string object: %v", err) + } + if err := enc.WriteEnd(); err != nil { + t.Fatalf("write end: %v", err) + } + + var objects []model.RedisObject + dec := NewDecoder(bytes.NewReader(buf.Bytes())) + if err := dec.Parse(func(object model.RedisObject) bool { + objects = append(objects, object) + return true + }); err != nil { + t.Fatalf("parse valkey cluster rdb: %v", err) + } + + if len(objects) != 1 { + t.Fatalf("expected 1 object, got %d", len(objects)) + } + strObj, ok := objects[0].(*model.StringObject) + if !ok { + t.Fatalf("expected string object, got %T", objects[0]) + } + if strObj.Key != "cluster-key" { + t.Fatalf("unexpected key: %s", strObj.Key) + } + if string(strObj.Value) != "value" { + t.Fatalf("unexpected value: %s", strObj.Value) + } +} diff --git a/helper/converter_test.go b/helper/converter_test.go index 2afda93..e3ad532 100644 --- a/helper/converter_test.go +++ b/helper/converter_test.go @@ -133,6 +133,46 @@ func TestToJson(t *testing.T) { } } +func TestToJsonWithHashFieldExpiration(t *testing.T) { + jsonEncoder = sonic.ConfigStd + var cstZone = time.FixedZone("CST", 8*3600) + time.Local = cstZone + + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "hash_with_hfe", + "hash_as_listpack_with_hfe", + "valkey_hash2_with_hfe", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualJSON := filepath.Join("tmp", filename+".json") + expectJSON := filepath.Join("../cases", filename+".json") + err = ToJsons(srcRdb, actualJSON, WithConcurrent(1)) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualJSON, expectJSON) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} func TestToJsonWithGlobalMeta(t *testing.T) { // SortMapKeys will cause performance losses, only enabled during test @@ -254,6 +294,43 @@ func TestToAof(t *testing.T) { } } +func TestToAofWithHashFieldExpiration(t *testing.T) { + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "hash_with_hfe", + "hash_as_listpack_with_hfe", + "valkey_hash2_with_hfe", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualFile := filepath.Join("tmp", filename+".aof") + expectFile := filepath.Join("../cases", filename+".aof") + err = ToAOF(srcRdb, actualFile, lexOrder{}) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualFile, expectFile) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} + func TestToAofWithRegex(t *testing.T) { err := os.MkdirAll("tmp", os.ModePerm) if err != nil { diff --git a/helper/resp.go b/helper/resp.go index 3829017..dd126d3 100644 --- a/helper/resp.go +++ b/helper/resp.go @@ -100,26 +100,40 @@ func hashToCmd(obj *model.HashObject, useLexOrder bool) []CmdLine { cmds := []CmdLine{cmdLine} if len(obj.FieldExpirations) == len(obj.Hash) { - for field, expire := range obj.FieldExpirations { + appendFieldExpiration := func(field string, expire int64) { if expire == 0 { hpexp := make([][]byte, 5) - // HPEXPIRE key seconds FIELDS num FIELD... + // HPERSIST key FIELDS num FIELD... hpexp[0] = hPersistCmd hpexp[1] = []byte(obj.Key) hpexp[2] = []byte("FIELDS") hpexp[3] = []byte("1") hpexp[4] = []byte(field) cmds = append(cmds, hpexp) - } else { - hpexp := make([][]byte, 6) - // HPEXPIRE key seconds FIELDS num FIELD... - hpexp[0] = hPExpireAtCmd - hpexp[1] = []byte(obj.Key) - hpexp[2] = []byte(fmt.Sprintf("%d", expire)) - hpexp[3] = []byte("FIELDS") - hpexp[4] = []byte("1") - hpexp[5] = []byte(field) - cmds = append(cmds, hpexp) + return + } + hpexp := make([][]byte, 6) + // HPEXPIREAT key milliseconds FIELDS num FIELD... + hpexp[0] = hPExpireAtCmd + hpexp[1] = []byte(obj.Key) + hpexp[2] = []byte(fmt.Sprintf("%d", expire)) + hpexp[3] = []byte("FIELDS") + hpexp[4] = []byte("1") + hpexp[5] = []byte(field) + cmds = append(cmds, hpexp) + } + if useLexOrder { + fields := make([]string, 0, len(obj.FieldExpirations)) + for field := range obj.FieldExpirations { + fields = append(fields, field) + } + sort.Strings(fields) + for _, field := range fields { + appendFieldExpiration(field, obj.FieldExpirations[field]) + } + } else { + for field, expire := range obj.FieldExpirations { + appendFieldExpiration(field, expire) } } } From 3222062e2cf864bf18915bf464b606b1229484cd Mon Sep 17 00:00:00 2001 From: cauchywei Date: Sun, 19 Apr 2026 20:54:20 +0800 Subject: [PATCH 2/2] use real Valkey cluster fixtures in tests --- cases/valkey_cluster_slot_import.aof | 0 cases/valkey_cluster_slot_import.json | 3 + cases/valkey_cluster_slot_import.rdb | Bin 0 -> 220 bytes cases/valkey_cluster_slot_info.aof | 7 +++ cases/valkey_cluster_slot_info.json | 3 + cases/valkey_cluster_slot_info.rdb | Bin 0 -> 222 bytes core/valkey_test.go | 84 -------------------------- helper/converter_test.go | 78 +++++++++++++++++++++++- 8 files changed, 90 insertions(+), 85 deletions(-) create mode 100644 cases/valkey_cluster_slot_import.aof create mode 100644 cases/valkey_cluster_slot_import.json create mode 100644 cases/valkey_cluster_slot_import.rdb create mode 100644 cases/valkey_cluster_slot_info.aof create mode 100644 cases/valkey_cluster_slot_info.json create mode 100644 cases/valkey_cluster_slot_info.rdb delete mode 100644 core/valkey_test.go diff --git a/cases/valkey_cluster_slot_import.aof b/cases/valkey_cluster_slot_import.aof new file mode 100644 index 0000000..e69de29 diff --git a/cases/valkey_cluster_slot_import.json b/cases/valkey_cluster_slot_import.json new file mode 100644 index 0000000..41b42e6 --- /dev/null +++ b/cases/valkey_cluster_slot_import.json @@ -0,0 +1,3 @@ +[ + +] diff --git a/cases/valkey_cluster_slot_import.rdb b/cases/valkey_cluster_slot_import.rdb new file mode 100644 index 0000000000000000000000000000000000000000..d9794877263cb5e45a3b8ebb2947e33fba890011 GIT binary patch literal 220 zcmW-YJxT*X7(h2gNQjU#q>RFZ`OWSWf`V3B3n{;u?;9ABC1Ga`*qD=ucmVIP<^uNC z-oS2}HKcm*c<<@<@!@{OE1oBN*sSS=?Wq~s1z)HG@ThqZx;*6yHAo>(sJt{-NI9m-4{h{xF?yl0? H<@@J9$_Y>) literal 0 HcmV?d00001 diff --git a/cases/valkey_cluster_slot_info.aof b/cases/valkey_cluster_slot_info.aof new file mode 100644 index 0000000..d22f365 --- /dev/null +++ b/cases/valkey_cluster_slot_info.aof @@ -0,0 +1,7 @@ +*3 +$3 +SET +$13 +{6ZJ}:fixture +$5 +value diff --git a/cases/valkey_cluster_slot_info.json b/cases/valkey_cluster_slot_info.json new file mode 100644 index 0000000..2cd2b08 --- /dev/null +++ b/cases/valkey_cluster_slot_info.json @@ -0,0 +1,3 @@ +[ +{"db":0,"key":"{6ZJ}:fixture","size":64,"type":"string","encoding":"string","value":"value"} +] diff --git a/cases/valkey_cluster_slot_info.rdb b/cases/valkey_cluster_slot_info.rdb new file mode 100644 index 0000000000000000000000000000000000000000..84c409feeb25e33f3be7cede5d00f359ca810928 GIT binary patch literal 222 zcmWG@^zn9$G_WxE#Z{J=lbu?rTb5eHYN=u zO-d|IJ;3mf;Ws10FV5ne{1V;FytI67Lo;IwV;w^s104ef-fFWbuUf0L%!-oIqEuF( RCrVTQAFWj0kvDPH8vu*BQGEaa literal 0 HcmV?d00001 diff --git a/core/valkey_test.go b/core/valkey_test.go deleted file mode 100644 index 2034c7c..0000000 --- a/core/valkey_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package core - -import ( - "bytes" - "testing" - - "github.com/hdt3213/rdb/model" -) - -func TestValkeyClusterMetadataOpcodes(t *testing.T) { - buf := bytes.NewBuffer(nil) - enc := NewEncoderValkey(buf) - if err := enc.WriteHeader(); err != nil { - t.Fatalf("write header: %v", err) - } - if err := enc.WriteDBHeader(0, 1, 0); err != nil { - t.Fatalf("write db header: %v", err) - } - - if err := enc.write([]byte{opCodeSlotInfo}); err != nil { - t.Fatalf("write slot info opcode: %v", err) - } - if err := enc.writeLength(42); err != nil { - t.Fatalf("write slot id: %v", err) - } - if err := enc.writeLength(3); err != nil { - t.Fatalf("write slot size: %v", err) - } - if err := enc.writeLength(1); err != nil { - t.Fatalf("write expires slot size: %v", err) - } - - if err := enc.write([]byte{opCodeSlotImport}); err != nil { - t.Fatalf("write slot import opcode: %v", err) - } - if err := enc.writeString("job-1"); err != nil { - t.Fatalf("write slot import job: %v", err) - } - if err := enc.writeLength(2); err != nil { - t.Fatalf("write slot range count: %v", err) - } - if err := enc.writeLength(10); err != nil { - t.Fatalf("write slot range start: %v", err) - } - if err := enc.writeLength(11); err != nil { - t.Fatalf("write slot range end: %v", err) - } - if err := enc.writeLength(20); err != nil { - t.Fatalf("write slot range start: %v", err) - } - if err := enc.writeLength(25); err != nil { - t.Fatalf("write slot range end: %v", err) - } - - if err := enc.WriteStringObject("cluster-key", []byte("value")); err != nil { - t.Fatalf("write string object: %v", err) - } - if err := enc.WriteEnd(); err != nil { - t.Fatalf("write end: %v", err) - } - - var objects []model.RedisObject - dec := NewDecoder(bytes.NewReader(buf.Bytes())) - if err := dec.Parse(func(object model.RedisObject) bool { - objects = append(objects, object) - return true - }); err != nil { - t.Fatalf("parse valkey cluster rdb: %v", err) - } - - if len(objects) != 1 { - t.Fatalf("expected 1 object, got %d", len(objects)) - } - strObj, ok := objects[0].(*model.StringObject) - if !ok { - t.Fatalf("expected string object, got %T", objects[0]) - } - if strObj.Key != "cluster-key" { - t.Fatalf("unexpected key: %s", strObj.Key) - } - if string(strObj.Value) != "value" { - t.Fatalf("unexpected value: %s", strObj.Value) - } -} diff --git a/helper/converter_test.go b/helper/converter_test.go index e3ad532..5d19319 100644 --- a/helper/converter_test.go +++ b/helper/converter_test.go @@ -174,6 +174,46 @@ func TestToJsonWithHashFieldExpiration(t *testing.T) { } } +func TestToJsonWithValkeyClusterMetadata(t *testing.T) { + jsonEncoder = sonic.ConfigStd + var cstZone = time.FixedZone("CST", 8*3600) + time.Local = cstZone + + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "valkey_cluster_slot_info", + "valkey_cluster_slot_import", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualJSON := filepath.Join("tmp", filename+".json") + expectJSON := filepath.Join("../cases", filename+".json") + err = ToJsons(srcRdb, actualJSON, WithConcurrent(1)) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualJSON, expectJSON) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} + func TestToJsonWithGlobalMeta(t *testing.T) { // SortMapKeys will cause performance losses, only enabled during test jsonEncoder = sonic.ConfigStd @@ -235,7 +275,7 @@ func TestToJsonWithRegex(t *testing.T) { srcRdb := filepath.Join("../cases", "memory.rdb") actualJSON := filepath.Join("tmp", "memory_regex.json") expectJSON := filepath.Join("../cases", "memory_regex.json") - err = ToJsons(srcRdb, actualJSON, WithRegexOption("^l.*")) + err = ToJsons(srcRdb, actualJSON, WithRegexOption("^l.*"), WithConcurrent(1)) if err != nil { t.Errorf("error occurs during parse, err: %v", err) return @@ -331,6 +371,42 @@ func TestToAofWithHashFieldExpiration(t *testing.T) { } } +func TestToAofWithValkeyClusterMetadata(t *testing.T) { + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "valkey_cluster_slot_info", + "valkey_cluster_slot_import", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualFile := filepath.Join("tmp", filename+".aof") + expectFile := filepath.Join("../cases", filename+".aof") + err = ToAOF(srcRdb, actualFile, lexOrder{}) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualFile, expectFile) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} + func TestToAofWithRegex(t *testing.T) { err := os.MkdirAll("tmp", os.ModePerm) if err != nil {