From c7f339ff3ded0f762e7630eeb8a442de9f250c2c Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Tue, 24 Mar 2026 18:58:54 +0000 Subject: [PATCH 1/2] Introduce `storage_optimize_copy` GUC --- config/storage.go | 3 +++ pkg/proc/interaction.go | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/config/storage.go b/config/storage.go index 2f29e37..0a2cf3b 100644 --- a/config/storage.go +++ b/config/storage.go @@ -32,6 +32,9 @@ type Storage struct { StorageRegion string `json:"storage_region" toml:"storage_region" yaml:"storage_region"` + // copy behaviour control option + StorageOptimizeCopy bool `json:"storage_optimize_copy" toml:"storage_optimize_copy" yaml:"storage_optimize_copy"` + // File storage default s3. Available: s3, fs StorageType string `json:"storage_type" toml:"storage_type" yaml:"storage_type"` diff --git a/pkg/proc/interaction.go b/pkg/proc/interaction.go index aa01198..f664ee8 100644 --- a/pkg/proc/interaction.go +++ b/pkg/proc/interaction.go @@ -910,10 +910,12 @@ func ListFilesToCopy(prefix string, port uint64, cfg config.Storage, src storage for i := range len(objectMetas) { path := strings.TrimPrefix(objectMetas[i].Path, cfg.StoragePrefix) reworked := path - if _, ok := vi[reworked]; !ok { - ylogger.Zero.Info().Int("index", i).Str("object path", objectMetas[i].Path).Msg("not in virtual index, skipping...") - skipped = append(skipped, objectMetas[i]) - continue + if config.InstanceConfig().StorageCnf.StorageOptimizeCopy { + if _, ok := vi[reworked]; !ok { + ylogger.Zero.Info().Int("index", i).Str("object path", objectMetas[i].Path).Msg("not in virtual index, skipping...") + skipped = append(skipped, objectMetas[i]) + continue + } } if sz, ok := copiedSizes[objectMetas[i].Path]; ok { ylogger.Zero.Info(). From dbcdefdb48c28af04078f0fe6e314ed4cab2272e Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Wed, 25 Mar 2026 08:53:10 +0000 Subject: [PATCH 2/2] toggle logic to log everything --- pkg/proc/interaction.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/proc/interaction.go b/pkg/proc/interaction.go index f664ee8..6764a06 100644 --- a/pkg/proc/interaction.go +++ b/pkg/proc/interaction.go @@ -910,10 +910,13 @@ func ListFilesToCopy(prefix string, port uint64, cfg config.Storage, src storage for i := range len(objectMetas) { path := strings.TrimPrefix(objectMetas[i].Path, cfg.StoragePrefix) reworked := path - if config.InstanceConfig().StorageCnf.StorageOptimizeCopy { - if _, ok := vi[reworked]; !ok { - ylogger.Zero.Info().Int("index", i).Str("object path", objectMetas[i].Path).Msg("not in virtual index, skipping...") - skipped = append(skipped, objectMetas[i]) + if _, ok := vi[reworked]; !ok { + skipCopy := config.InstanceConfig().StorageCnf.StorageOptimizeCopy + skipped = append(skipped, objectMetas[i]) + + ylogger.Zero.Info().Int("index", i).Str("reworked name", reworked).Str("object path", objectMetas[i].Path).Bool("skipping", skipCopy).Msg("not in virtual index") + + if skipCopy { continue } }