Features/issue 1985 - Add glob_pattern to LogstreamerInput

docs/source/config/inputs/logstreamer.rst

@@ -46,6 +46,15 @@ Config:
     all backslashes to be escaped. For example, `'access\\.log'` will work as
     expected, but `"access\\.log"` will not, you would need `"access\\\\.log"`
     to achieve the same result.
+- glob_pattern (string, optional):
+    By default, the method for scanning the filesystem uses the filepath.Walk()
+    method. That will traverse every directory/file down "log_dir" and check
+    to see if it matches "file_match". This can be slower depending on the
+    complexity of the filesystem. "glob_pattern" provides an optional
+    alternative method for scanning the filesystem by quickly ruling out
+    locations that would otherwise be scanned. This is still used in conjunction
+    with "log_dir" and "file_match. Uses `format
+    <https://golang.org/pkg/path/filepath/#Match>`_. (e.x. `"/var/log/*/*.log*"`).
 - priority (list of strings):
     When using sequential logstreams, the priority is how to sort the logfiles
     in order from oldest to newest.

logstreamer/filehandling.go

@@ -289,6 +289,23 @@ func ScanDirectoryForLogfiles(directoryPath string, fileMatch *regexp.Regexp) Lo
 	return files
 }
 
+// Globs through log_directory filtering out files that match the fileMatch regexp
+func GlobForLogfiles(directoryPath string, fileMatch *regexp.Regexp, globPattern string) Logfiles {
+	files := make(Logfiles, 0)
+	glob, _ := filepath.Glob(globPattern)
+
+	for _, path := range glob {
+		fi, err := os.Stat(path)
+		if err != nil || fi.Mode().IsDir() {
+			return nil
+		}
+		if fileMatch.MatchString(path) {
+			files = append(files, &Logfile{FileName: path})
+		}
+	}
+	return files
+}
+
 // A single logstream
 // Implements io.Reader interface for reading from the logstream
 type Logstream struct {
@@ -446,12 +463,18 @@ func (ls *LogstreamSet) ScanForLogstreams() (result []string, errors *MultipleEr
 	var (
 		logstream *Logstream
 		ok        bool
+		logfiles  Logfiles
 	)
 	result = make([]string, 0, 0)
 	errors = NewMultipleError()
 
-	// Scan for all our logfiles
-	logfiles := ScanDirectoryForLogfiles(ls.logRoot, ls.fileMatch)
+	if ls.sortPattern.GlobPattern != "" {
+		// Scan for all our logfiles using filepath.Glob()
+		logfiles = GlobForLogfiles(ls.logRoot, ls.fileMatch, ls.sortPattern.GlobPattern)
+	} else {
+		// Scan for all our logfiles using filepath.Walk()
+		logfiles = ScanDirectoryForLogfiles(ls.logRoot, ls.fileMatch)
+	}
 
 	// Filter out old logfiles
 	if ls.oldestDuration != time.Duration(0) {
@@ -583,6 +606,9 @@ type SortPattern struct {
 	// These names will be translated from short/long month/day names to the appropriate
 	// integer value.
 	FileMatch string
+	// Glob pattern to be used as an alternative method for scanning log_directory path for
+	// file_match.
+	GlobPattern string
 	// Translation is used for custom ordering lookups where a custom value needs to be
 	// translated to a value for sorting. ie. a different tool using weekdays with values
 	// causing the wrong day of the week to be parsed first

logstreamer/filehandling_test.go

@@ -31,6 +31,7 @@ func FilehandlingSpec(c gs.Context) {
 	if runtime.GOOS == "windows" {
 		regex1, regex2 = `\\subdir\\.*\.log(\..*)?`, `\\subdir\\.*.logg(.*)?`
 	}
+	globPattern, globPattern2 := filepath.Join(dirPath, "subdir/*.log*"), filepath.Join(dirPath, "subdir/*.logg*")
 
 	c.Specify("The directory scanner", func() {
 
@@ -45,6 +46,19 @@ func FilehandlingSpec(c gs.Context) {
 		})
 	})
 
+	c.Specify("The glob scanner", func() {
+
+		c.Specify("globs a directory properly", func() {
+			results := GlobForLogfiles(dirPath, fileMatchRegexp(dirPath, regex1), globPattern)
+			c.Expect(len(results), gs.Equals, 3)
+		})
+
+		c.Specify("globs a directory with a bad pattern", func() {
+			results := GlobForLogfiles(dirPath, fileMatchRegexp(dirPath, regex2), globPattern2)
+			c.Expect(len(results), gs.Equals, 0)
+		})
+	})
+
 	c.Specify("Populating logfile with match parts", func() {
 		logfile := Logfile{}
 

plugins/logstreamer/logstreamer_input.go

@@ -38,6 +38,9 @@ type LogstreamerInputConfig struct {
 	JournalDirectory string `toml:"journal_directory"`
 	// File match for regular expression
 	FileMatch string `toml:"file_match"`
+	// Glob pattern to be used as an alternative method for scanning log_directory path for
+	// file_match.
+	GlobPattern string `toml:"glob_pattern"`
 	// Priority to sort in
 	Priority []string
 	// Differentiator for splitting out logstreams if applicable
@@ -152,6 +155,7 @@ func (li *LogstreamerInput) Init(config interface{}) (err error) {
 		Translation:    conf.Translation,
 		Priority:       conf.Priority,
 		Differentiator: conf.Differentiator,
+		GlobPattern:    conf.GlobPattern,
 	}
 
 	// Create the main logstream set