diff --git a/cases/hash_as_listpack_with_hfe.aof b/cases/hash_as_listpack_with_hfe.aof index ee4cfad..8fe6258 100644 --- a/cases/hash_as_listpack_with_hfe.aof +++ b/cases/hash_as_listpack_with_hfe.aof @@ -8,26 +8,26 @@ F1 $2 V1 $2 -F3 -$2 -V3 -$2 F2 $2 V2 +$2 +F3 +$2 +V3 *6 $10 HPEXPIREAT $12 listpack-hfe $13 -2755484483878 +2755482478325 $6 FIELDS $1 1 $2 -F3 +F1 *5 $8 HPERSIST @@ -45,10 +45,10 @@ HPEXPIREAT $12 listpack-hfe $13 -2755482478325 +2755484483878 $6 FIELDS $1 1 $2 -F1 +F3 diff --git a/cases/hash_with_hfe.aof b/cases/hash_with_hfe.aof index 8983988..e86c13d 100644 --- a/cases/hash_with_hfe.aof +++ b/cases/hash_with_hfe.aof @@ -4,48 +4,50 @@ HMSET $8 hash-hfe $2 -F4 +F1 $2 -V4 +V1 $2 -F7 +F2 $2 -V7 +V2 $2 -F8 +F3 $2 -V8 +V3 $2 -F2 +F4 $2 -V2 +V4 $2 F5 $2 V5 $2 -F3 +F6 $2 -V3 +V6 $2 -F1 +F7 $2 -V1 +V7 $2 -F6 +F8 $2 -V6 -*5 -$8 -HPERSIST +V8 +*6 +$10 +HPEXPIREAT $8 hash-hfe +$13 +2755482424661 $6 FIELDS $1 1 $2 -F8 +F1 *6 $10 HPEXPIREAT @@ -59,43 +61,41 @@ $1 1 $2 F2 -*5 -$8 -HPERSIST +*6 +$10 +HPEXPIREAT $8 hash-hfe +$13 +2755484433842 $6 FIELDS $1 1 $2 -F5 -*6 -$10 -HPEXPIREAT +F3 +*5 +$8 +HPERSIST $8 hash-hfe -$13 -2755484433842 $6 FIELDS $1 1 $2 -F3 -*6 -$10 -HPEXPIREAT +F4 +*5 +$8 +HPERSIST $8 hash-hfe -$13 -2755482424661 $6 FIELDS $1 1 $2 -F1 +F5 *5 $8 HPERSIST @@ -117,7 +117,7 @@ FIELDS $1 1 $2 -F4 +F7 *5 $8 HPERSIST @@ -128,4 +128,4 @@ FIELDS $1 1 $2 -F7 +F8 diff --git a/cases/valkey_cluster_slot_import.aof b/cases/valkey_cluster_slot_import.aof new file mode 100644 index 0000000..e69de29 diff --git a/cases/valkey_cluster_slot_import.json b/cases/valkey_cluster_slot_import.json new file mode 100644 index 0000000..41b42e6 --- /dev/null +++ b/cases/valkey_cluster_slot_import.json @@ -0,0 +1,3 @@ +[ + +] diff --git a/cases/valkey_cluster_slot_import.rdb b/cases/valkey_cluster_slot_import.rdb new file mode 100644 index 0000000..d979487 Binary files /dev/null and b/cases/valkey_cluster_slot_import.rdb differ diff --git a/cases/valkey_cluster_slot_info.aof b/cases/valkey_cluster_slot_info.aof new file mode 100644 index 0000000..d22f365 --- /dev/null +++ b/cases/valkey_cluster_slot_info.aof @@ -0,0 +1,7 @@ +*3 +$3 +SET +$13 +{6ZJ}:fixture +$5 +value diff --git a/cases/valkey_cluster_slot_info.json b/cases/valkey_cluster_slot_info.json new file mode 100644 index 0000000..2cd2b08 --- /dev/null +++ b/cases/valkey_cluster_slot_info.json @@ -0,0 +1,3 @@ +[ +{"db":0,"key":"{6ZJ}:fixture","size":64,"type":"string","encoding":"string","value":"value"} +] diff --git a/cases/valkey_cluster_slot_info.rdb b/cases/valkey_cluster_slot_info.rdb new file mode 100644 index 0000000..84c409f Binary files /dev/null and b/cases/valkey_cluster_slot_info.rdb differ diff --git a/helper/converter_test.go b/helper/converter_test.go index 2afda93..5d19319 100644 --- a/helper/converter_test.go +++ b/helper/converter_test.go @@ -133,6 +133,86 @@ func TestToJson(t *testing.T) { } } +func TestToJsonWithHashFieldExpiration(t *testing.T) { + jsonEncoder = sonic.ConfigStd + var cstZone = time.FixedZone("CST", 8*3600) + time.Local = cstZone + + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "hash_with_hfe", + "hash_as_listpack_with_hfe", + "valkey_hash2_with_hfe", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualJSON := filepath.Join("tmp", filename+".json") + expectJSON := filepath.Join("../cases", filename+".json") + err = ToJsons(srcRdb, actualJSON, WithConcurrent(1)) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualJSON, expectJSON) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} + +func TestToJsonWithValkeyClusterMetadata(t *testing.T) { + jsonEncoder = sonic.ConfigStd + var cstZone = time.FixedZone("CST", 8*3600) + time.Local = cstZone + + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "valkey_cluster_slot_info", + "valkey_cluster_slot_import", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualJSON := filepath.Join("tmp", filename+".json") + expectJSON := filepath.Join("../cases", filename+".json") + err = ToJsons(srcRdb, actualJSON, WithConcurrent(1)) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualJSON, expectJSON) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} func TestToJsonWithGlobalMeta(t *testing.T) { // SortMapKeys will cause performance losses, only enabled during test @@ -195,7 +275,7 @@ func TestToJsonWithRegex(t *testing.T) { srcRdb := filepath.Join("../cases", "memory.rdb") actualJSON := filepath.Join("tmp", "memory_regex.json") expectJSON := filepath.Join("../cases", "memory_regex.json") - err = ToJsons(srcRdb, actualJSON, WithRegexOption("^l.*")) + err = ToJsons(srcRdb, actualJSON, WithRegexOption("^l.*"), WithConcurrent(1)) if err != nil { t.Errorf("error occurs during parse, err: %v", err) return @@ -254,6 +334,79 @@ func TestToAof(t *testing.T) { } } +func TestToAofWithHashFieldExpiration(t *testing.T) { + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "hash_with_hfe", + "hash_as_listpack_with_hfe", + "valkey_hash2_with_hfe", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualFile := filepath.Join("tmp", filename+".aof") + expectFile := filepath.Join("../cases", filename+".aof") + err = ToAOF(srcRdb, actualFile, lexOrder{}) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualFile, expectFile) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} + +func TestToAofWithValkeyClusterMetadata(t *testing.T) { + err := os.MkdirAll("tmp", os.ModePerm) + if err != nil { + return + } + defer func() { + err := os.RemoveAll("tmp") + if err != nil { + t.Logf("remove tmp directory failed: %v", err) + } + }() + + testCases := []string{ + "valkey_cluster_slot_info", + "valkey_cluster_slot_import", + } + for _, filename := range testCases { + srcRdb := filepath.Join("../cases", filename+".rdb") + actualFile := filepath.Join("tmp", filename+".aof") + expectFile := filepath.Join("../cases", filename+".aof") + err = ToAOF(srcRdb, actualFile, lexOrder{}) + if err != nil { + t.Errorf("error occurs during parse %s, err: %v", filename, err) + continue + } + equals, err := compareFileByLine(t, actualFile, expectFile) + if err != nil { + t.Errorf("error occurs during compare %s, err: %v", filename, err) + continue + } + if !equals { + t.Errorf("result is not equal of %s", filename) + } + } +} + func TestToAofWithRegex(t *testing.T) { err := os.MkdirAll("tmp", os.ModePerm) if err != nil { diff --git a/helper/resp.go b/helper/resp.go index 3829017..dd126d3 100644 --- a/helper/resp.go +++ b/helper/resp.go @@ -100,26 +100,40 @@ func hashToCmd(obj *model.HashObject, useLexOrder bool) []CmdLine { cmds := []CmdLine{cmdLine} if len(obj.FieldExpirations) == len(obj.Hash) { - for field, expire := range obj.FieldExpirations { + appendFieldExpiration := func(field string, expire int64) { if expire == 0 { hpexp := make([][]byte, 5) - // HPEXPIRE key seconds FIELDS num FIELD... + // HPERSIST key FIELDS num FIELD... hpexp[0] = hPersistCmd hpexp[1] = []byte(obj.Key) hpexp[2] = []byte("FIELDS") hpexp[3] = []byte("1") hpexp[4] = []byte(field) cmds = append(cmds, hpexp) - } else { - hpexp := make([][]byte, 6) - // HPEXPIRE key seconds FIELDS num FIELD... - hpexp[0] = hPExpireAtCmd - hpexp[1] = []byte(obj.Key) - hpexp[2] = []byte(fmt.Sprintf("%d", expire)) - hpexp[3] = []byte("FIELDS") - hpexp[4] = []byte("1") - hpexp[5] = []byte(field) - cmds = append(cmds, hpexp) + return + } + hpexp := make([][]byte, 6) + // HPEXPIREAT key milliseconds FIELDS num FIELD... + hpexp[0] = hPExpireAtCmd + hpexp[1] = []byte(obj.Key) + hpexp[2] = []byte(fmt.Sprintf("%d", expire)) + hpexp[3] = []byte("FIELDS") + hpexp[4] = []byte("1") + hpexp[5] = []byte(field) + cmds = append(cmds, hpexp) + } + if useLexOrder { + fields := make([]string, 0, len(obj.FieldExpirations)) + for field := range obj.FieldExpirations { + fields = append(fields, field) + } + sort.Strings(fields) + for _, field := range fields { + appendFieldExpiration(field, obj.FieldExpirations[field]) + } + } else { + for field, expire := range obj.FieldExpirations { + appendFieldExpiration(field, expire) } } }