From bc88328578709fe4a98399c5c1c7ebff6573a788 Mon Sep 17 00:00:00 2001 From: Michael Gibson Date: Sat, 3 Sep 2016 14:22:58 -0600 Subject: [PATCH 1/3] initial checkin for issue_1985 --- logstreamer/filehandling.go | 30 ++++++++++++++++++++++-- logstreamer/filehandling_test.go | 14 +++++++++++ plugins/logstreamer/logstreamer_input.go | 4 ++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/logstreamer/filehandling.go b/logstreamer/filehandling.go index 40887ba62..94f9fde4f 100644 --- a/logstreamer/filehandling.go +++ b/logstreamer/filehandling.go @@ -289,6 +289,23 @@ func ScanDirectoryForLogfiles(directoryPath string, fileMatch *regexp.Regexp) Lo return files } +// Globs through log_directory filtering out files that match the fileMatch regexp +func GlobForLogfiles(directoryPath string, fileMatch *regexp.Regexp, globPattern string) Logfiles { + files := make(Logfiles, 0) + glob, _ := filepath.Glob(globPattern) + + for _, path := range glob { + fi, err := os.Stat(path) + if err != nil || fi.Mode().IsDir() { + return nil + } + if fileMatch.MatchString(path) { + files = append(files, &Logfile{FileName: path}) + } + } + return files +} + // A single logstream // Implements io.Reader interface for reading from the logstream type Logstream struct { @@ -446,12 +463,18 @@ func (ls *LogstreamSet) ScanForLogstreams() (result []string, errors *MultipleEr var ( logstream *Logstream ok bool + logfiles Logfiles ) result = make([]string, 0, 0) errors = NewMultipleError() - // Scan for all our logfiles - logfiles := ScanDirectoryForLogfiles(ls.logRoot, ls.fileMatch) + if ls.sortPattern.GlobPattern != "" { + // Scan for all our logfiles using filepath.Glob() + logfiles = GlobForLogfiles(ls.logRoot, ls.fileMatch, ls.sortPattern.GlobPattern) + } else { + // Scan for all our logfiles using filepath.Walk() + logfiles = ScanDirectoryForLogfiles(ls.logRoot, ls.fileMatch) + } // Filter out old logfiles if ls.oldestDuration != time.Duration(0) { @@ -583,6 +606,9 @@ type SortPattern struct { // These names will be translated from short/long month/day names to the appropriate // integer value. FileMatch string + // Glob pattern to be used as an alternative method for scanning log_directory path for + // file_match. + GlobPattern string // Translation is used for custom ordering lookups where a custom value needs to be // translated to a value for sorting. ie. a different tool using weekdays with values // causing the wrong day of the week to be parsed first diff --git a/logstreamer/filehandling_test.go b/logstreamer/filehandling_test.go index 7dff607ab..c566c2681 100644 --- a/logstreamer/filehandling_test.go +++ b/logstreamer/filehandling_test.go @@ -31,6 +31,7 @@ func FilehandlingSpec(c gs.Context) { if runtime.GOOS == "windows" { regex1, regex2 = `\\subdir\\.*\.log(\..*)?`, `\\subdir\\.*.logg(.*)?` } + globPattern, globPattern2 := filepath.Join(dirPath, "subdir/*.log*"), filepath.Join(dirPath, "subdir/*.logg*") c.Specify("The directory scanner", func() { @@ -45,6 +46,19 @@ func FilehandlingSpec(c gs.Context) { }) }) + c.Specify("The glob scanner", func() { + + c.Specify("globs a directory properly", func() { + results := GlobForLogfiles(dirPath, fileMatchRegexp(dirPath, regex1), globPattern) + c.Expect(len(results), gs.Equals, 3) + }) + + c.Specify("globs a directory with a bad pattern", func() { + results := GlobForLogfiles(dirPath, fileMatchRegexp(dirPath, regex2), globPattern2) + c.Expect(len(results), gs.Equals, 0) + }) + }) + c.Specify("Populating logfile with match parts", func() { logfile := Logfile{} diff --git a/plugins/logstreamer/logstreamer_input.go b/plugins/logstreamer/logstreamer_input.go index e2d342fdb..e3588c937 100644 --- a/plugins/logstreamer/logstreamer_input.go +++ b/plugins/logstreamer/logstreamer_input.go @@ -38,6 +38,9 @@ type LogstreamerInputConfig struct { JournalDirectory string `toml:"journal_directory"` // File match for regular expression FileMatch string `toml:"file_match"` + // Glob pattern to be used as an alternative method for scanning log_directory path for + // file_match. + GlobPattern string `toml:"glob_pattern"` // Priority to sort in Priority []string // Differentiator for splitting out logstreams if applicable @@ -152,6 +155,7 @@ func (li *LogstreamerInput) Init(config interface{}) (err error) { Translation: conf.Translation, Priority: conf.Priority, Differentiator: conf.Differentiator, + GlobPattern: conf.GlobPattern, } // Create the main logstream set From 1a9657fc3cc25dc4293601ae193ddc627a883fa8 Mon Sep 17 00:00:00 2001 From: Michael Gibson Date: Sat, 3 Sep 2016 14:59:42 -0600 Subject: [PATCH 2/3] updating documentation --- docs/source/config/inputs/logstreamer.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/config/inputs/logstreamer.rst b/docs/source/config/inputs/logstreamer.rst index e0267862c..88b1a3fc3 100644 --- a/docs/source/config/inputs/logstreamer.rst +++ b/docs/source/config/inputs/logstreamer.rst @@ -46,6 +46,15 @@ Config: all backslashes to be escaped. For example, `'access\\.log'` will work as expected, but `"access\\.log"` will not, you would need `"access\\\\.log"` to achieve the same result. +- glob_pattern (string, optional): + By default, the method for scanning the filesystem uses the filepath.Walk() + method. That will travers every directory/file down "log_dir" and check + to see if it matches "file_match". This can be slower depending on the + complexity of the filesystem. "glob_pattern" provides an optional + alternative method for scanning the filesystem by quickly ruling out + locations that would otherwise be scanned. This is still used in conjunction + with "log_dir" and "file_match. Uses `format + `_. (e.x. `"/var/log/*/*.log*"`). - priority (list of strings): When using sequential logstreams, the priority is how to sort the logfiles in order from oldest to newest. From 35e250a78ac0122dad12a95038fad9cadcc6e814 Mon Sep 17 00:00:00 2001 From: Michael Gibson Date: Sat, 3 Sep 2016 15:01:41 -0600 Subject: [PATCH 3/3] fixing typo in docs --- docs/source/config/inputs/logstreamer.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config/inputs/logstreamer.rst b/docs/source/config/inputs/logstreamer.rst index 88b1a3fc3..313425dcc 100644 --- a/docs/source/config/inputs/logstreamer.rst +++ b/docs/source/config/inputs/logstreamer.rst @@ -48,7 +48,7 @@ Config: to achieve the same result. - glob_pattern (string, optional): By default, the method for scanning the filesystem uses the filepath.Walk() - method. That will travers every directory/file down "log_dir" and check + method. That will traverse every directory/file down "log_dir" and check to see if it matches "file_match". This can be slower depending on the complexity of the filesystem. "glob_pattern" provides an optional alternative method for scanning the filesystem by quickly ruling out