From cade3922ec99643e7b1599e3361226e66af619bd Mon Sep 17 00:00:00 2001 From: JicLotus Date: Sun, 15 Feb 2026 13:00:45 +0100 Subject: [PATCH 1/4] adding recursive implementation + ut --- cmd/cmd.go | 12 +++ cmd/scan.go | 6 +- go.mod | 1 + go.sum | 2 + utils/file_utils.go | 53 ++++++++--- utils/file_utils_test.go | 201 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 260 insertions(+), 15 deletions(-) create mode 100644 utils/file_utils_test.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 5e140c8..9f52a56 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -74,6 +74,18 @@ func addThreadsFlag(flags *pflag.FlagSet) { "number of threads working in parallel") } +func addRecursive(flags *pflag.FlagSet) { + flags.BoolP( + "recursive", "r", false, + "enable recursive traversal of subdirectories") +} + +func addMaxDepth(flags *pflag.FlagSet) { + flags.IntP( + "maxDepth", "d", 1, + "maximum recursion depth for directory traversal") +} + func addIDOnlyFlag(flags *pflag.FlagSet) { flags.BoolP( "identifiers-only", "I", false, diff --git a/cmd/scan.go b/cmd/scan.go index 2d181bd..255df7a 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -164,7 +164,9 @@ func NewScanFileCmd() *cobra.Command { if len(args) == 1 && args[0] == "-" { argReader = utils.NewStringIOReader(os.Stdin) } else if len(args) == 1 && utils.IsDir(args[0]) { - argReader, _ = utils.NewFileDirReader(args[0]) + recursive := viper.GetBool("recursive") + maxDepth := viper.GetInt("maxDepth") + argReader, _ = utils.NewFileDirReader(args[0], recursive, maxDepth) } else { argReader = utils.NewStringArrayReader(args) } @@ -188,6 +190,8 @@ func NewScanFileCmd() *cobra.Command { }, } + addRecursive(cmd.Flags()) + addMaxDepth(cmd.Flags()) addThreadsFlag(cmd.Flags()) addOpenInVTFlag(cmd.Flags()) addPasswordFlag(cmd.Flags()) diff --git a/go.mod b/go.mod index e98d51f..67240a5 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/go.sum b/go.sum index 8b25c1f..49f65ac 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= diff --git a/utils/file_utils.go b/utils/file_utils.go index a3fd82c..ce6fc4f 100644 --- a/utils/file_utils.go +++ b/utils/file_utils.go @@ -14,26 +14,51 @@ package utils import ( + "io/fs" "os" - "path" + "path/filepath" + "strings" ) -// FileDirReader returns all files inside a given directory -// as a StringArrayReader -func NewFileDirReader(fileDir string) (*StringArrayReader, error) { - files, err := os.ReadDir(fileDir) +// NewFileDirReader reads all files from the given directory `fileDir`. +// It can optionally traverse subdirectories if `recursive` is true, +// and will limit recursion to `maxDepth` levels if specified. +// +// Uses the standard library's `filepath.WalkDir` to traverse directories efficiently, +// and `fs.SkipDir` to skip directories when recursion is disabled or maxDepth is reached. +func NewFileDirReader(fileDir string, recursive bool, maxDepth int) (*StringArrayReader, error) { + var filePaths []string + rootDepth := pathDepth(fileDir) + + err := filepath.WalkDir(fileDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if !d.IsDir() { + filePaths = append(filePaths, path) + return nil + } + + currentDepth := pathDepth(path) - rootDepth + // we skip directory if recursive is disabled or + // if we reached configured maxDepth + if !recursive && path != fileDir || + currentDepth >= maxDepth { + return fs.SkipDir + } + + return nil + }) + if err != nil { return nil, err } - fileNames := []string{} - for _, f := range files { - // Skip subdirectories - if f.IsDir() { - continue - } - fileNames = append(fileNames, path.Join(fileDir, f.Name())) - } - return &StringArrayReader{strings: fileNames}, nil + return &StringArrayReader{strings: filePaths}, nil +} + +func pathDepth(path string) int { + return len(strings.Split(filepath.Clean(path), string(os.PathSeparator))) } // IsDir function returns whether a file is a directory or not diff --git a/utils/file_utils_test.go b/utils/file_utils_test.go new file mode 100644 index 0000000..b259a41 --- /dev/null +++ b/utils/file_utils_test.go @@ -0,0 +1,201 @@ +// Copyright © 2017 The VirusTotal CLI authors. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func Test_NewFileDirReader(t *testing.T) { + t.Parallel() + + useCases := []struct { + name string + + directories []string + files []string + recursive bool + maxDepth int + + want func(string) *StringArrayReader + }{ + + { + name: "want get back empty files", + directories: []string{}, + files: []string{}, + recursive: true, + maxDepth: 2, + want: func(d string) *StringArrayReader { + return &StringArrayReader{ + strings: nil, + } + }, + }, + { + name: "want to read single file", + directories: []string{}, + files: []string{"z.txt"}, + recursive: true, + maxDepth: 2, + want: func(d string) *StringArrayReader { + return &StringArrayReader{ + strings: []string{ + filepath.Join(d, "z.txt"), + }, + } + }, + }, + { + name: "want read all files within all subdirectories", + directories: []string{"sub", "sub/sub", ".hidden"}, + files: []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt", ".hidden/config"}, + recursive: true, + maxDepth: 3, + want: func(d string) *StringArrayReader { + return &StringArrayReader{ + strings: []string{ + filepath.Join(d, ".hidden/config"), + filepath.Join(d, "a.txt"), + filepath.Join(d, "b.txt"), + filepath.Join(d, "sub/c.txt"), + filepath.Join(d, "sub/sub/d.txt"), + }, + } + }, + }, + { + name: "want to ignore all subdirectories", + directories: []string{"sub", "sub/sub"}, + files: []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt"}, + recursive: false, + maxDepth: 10, + want: func(d string) *StringArrayReader { + return &StringArrayReader{ + strings: []string{ + filepath.Join(d, "a.txt"), + filepath.Join(d, "b.txt"), + }, + } + }, + }, + { + name: "want to read until first depth", + directories: []string{"sub", "sub/sub"}, + files: []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt"}, + recursive: true, + maxDepth: 1, + want: func(d string) *StringArrayReader { + return &StringArrayReader{ + strings: []string{ + filepath.Join(d, "a.txt"), + filepath.Join(d, "b.txt"), + }, + } + }, + }, + { + name: "want to read until second depth", + directories: []string{"sub", "sub/sub"}, + files: []string{"a.txt", "b.txt", "sub/c.txt", "sub/sub/d.txt"}, + recursive: true, + maxDepth: 2, + want: func(d string) *StringArrayReader { + return &StringArrayReader{ + strings: []string{ + filepath.Join(d, "a.txt"), + filepath.Join(d, "b.txt"), + filepath.Join(d, "sub/c.txt"), + }, + } + }, + }, + } + + for _, uc := range useCases { + t.Run(uc.name, func(t *testing.T) { + + // create a temp directory, and will clean up after test ends + rootDir := t.TempDir() + + for _, d := range uc.directories { + path := filepath.Join(rootDir, d) + if err := os.Mkdir(path, 0755); err != nil { + t.Fatalf("unexpected error while Mkdir %v", err) + } + } + + for _, f := range uc.files { + path := filepath.Join(rootDir, f) + if err := os.WriteFile(path, []byte("hello world!"), 0644); err != nil { + t.Fatalf("unexpected error while WriteFile %v", err) + } + } + + got, err := NewFileDirReader(rootDir, uc.recursive, uc.maxDepth) + if err != nil { + t.Errorf("unexpected error while NewFileDirReader err:%v", err) + } + if diff := cmp.Diff( + uc.want(rootDir), + got, + cmp.AllowUnexported(StringArrayReader{}), + ); diff != "" { + t.Errorf("unexpected StringArrayReader mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func Test_pathDepth(t *testing.T) { + t.Parallel() + + useCases := []struct { + name string + dir string + want int + }{ + { + name: "want one depth when empty directory", + want: 1, + }, + { + name: "want one depth when simple directory", + dir: "a", + want: 1, + }, + { + name: "want 2 depth when sub directory", + dir: "a/a", + want: 2, + }, + { + name: "want 3 depth when sub directory", + dir: "a-a/b_bb__/cccc/", + want: 3, + }, + } + + for _, uc := range useCases { + t.Run(uc.name, func(t *testing.T) { + if got, want := pathDepth(uc.dir), uc.want; got != want { + t.Errorf("unexpected pathDepth, got:%v, want:%v", got, want) + } + }) + } +} From c876a935cc3ccfb99a3dc57d7be1c9ec779c8ef1 Mon Sep 17 00:00:00 2001 From: JicLotus Date: Sun, 15 Feb 2026 13:39:32 +0100 Subject: [PATCH 2/4] adding comments --- utils/file_utils.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/file_utils.go b/utils/file_utils.go index ce6fc4f..bbad8db 100644 --- a/utils/file_utils.go +++ b/utils/file_utils.go @@ -30,6 +30,8 @@ func NewFileDirReader(fileDir string, recursive bool, maxDepth int) (*StringArra var filePaths []string rootDepth := pathDepth(fileDir) + // filePaths is safely appended within WalkDir because WalkDir executes the callback sequentially. + // No race conditions occur in this implementation, even with slice reallocation. err := filepath.WalkDir(fileDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -57,8 +59,11 @@ func NewFileDirReader(fileDir string, recursive bool, maxDepth int) (*StringArra return &StringArrayReader{strings: filePaths}, nil } +// pathDepth returns the depth of a given path by counting its components. +// It uses filepath.Separator, which ensures correct behavior across all platforms +// (Windows, macOS, Linux), regardless of the underlying path separator. func pathDepth(path string) int { - return len(strings.Split(filepath.Clean(path), string(os.PathSeparator))) + return len(strings.Split(filepath.Clean(path), string(filepath.Separator))) } // IsDir function returns whether a file is a directory or not From 906b2e38eeadd0b3944ca0d665dcd004a245ea29 Mon Sep 17 00:00:00 2001 From: JicLotus Date: Sun, 15 Feb 2026 16:39:06 +0100 Subject: [PATCH 3/4] adding error ut --- utils/file_utils_test.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/utils/file_utils_test.go b/utils/file_utils_test.go index b259a41..caf1d28 100644 --- a/utils/file_utils_test.go +++ b/utils/file_utils_test.go @@ -16,6 +16,7 @@ package utils import ( "os" "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -135,14 +136,16 @@ func Test_NewFileDirReader(t *testing.T) { for _, d := range uc.directories { path := filepath.Join(rootDir, d) - if err := os.Mkdir(path, 0755); err != nil { + rwxPerm := os.FileMode(0755) + if err := os.Mkdir(path, rwxPerm); err != nil { t.Fatalf("unexpected error while Mkdir %v", err) } } for _, f := range uc.files { path := filepath.Join(rootDir, f) - if err := os.WriteFile(path, []byte("hello world!"), 0644); err != nil { + rwPerm := os.FileMode(0644) + if err := os.WriteFile(path, []byte("hello world!"), rwPerm); err != nil { t.Fatalf("unexpected error while WriteFile %v", err) } } @@ -162,6 +165,26 @@ func Test_NewFileDirReader(t *testing.T) { } } +func Test_NewFileDirReader_Error(t *testing.T) { + rootDir := t.TempDir() + noPerm := os.FileMode(0000) + if err := os.WriteFile(filepath.Join(rootDir, "a.txt"), []byte("hello world!"), noPerm); err != nil { + t.Fatalf("unexpected error while WriteFile %v", err) + } + path := filepath.Join(rootDir, "sub") + if err := os.Mkdir(path, noPerm); err != nil { + t.Fatalf("unexpected error while Mkdir %v", err) + } + _, err := NewFileDirReader(rootDir, false, 10) + if err != nil { + t.Errorf("unexpected error while NewFileDirReader err:%v", err) + } + _, err = NewFileDirReader(rootDir, true, 10) + if !strings.Contains(err.Error(), "permission denied") { + t.Errorf("unexpected error permissions denied message got:%v", err.Error()) + } +} + func Test_pathDepth(t *testing.T) { t.Parallel() From 9a2eb0767de87996a99b1d6c78616ee95d8885a4 Mon Sep 17 00:00:00 2001 From: JicLotus Date: Sun, 15 Feb 2026 16:40:31 +0100 Subject: [PATCH 4/4] adding parallel to ut --- utils/file_utils_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/file_utils_test.go b/utils/file_utils_test.go index caf1d28..7ea8826 100644 --- a/utils/file_utils_test.go +++ b/utils/file_utils_test.go @@ -166,6 +166,8 @@ func Test_NewFileDirReader(t *testing.T) { } func Test_NewFileDirReader_Error(t *testing.T) { + t.Parallel() + rootDir := t.TempDir() noPerm := os.FileMode(0000) if err := os.WriteFile(filepath.Join(rootDir, "a.txt"), []byte("hello world!"), noPerm); err != nil {