Skip to content

refactor(generators): replace embedded data pools with datagen imports across apache_combined, apache_error, nginx, paloalto, winevt #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Dylan-M wants to merge 1 commit into
base: 04-06-docs_datagen_add_documentation_for_identity_hierarchy_and_seed_configuration
Choose a base branch
from
Open

refactor(generators): replace embedded data pools with datagen imports across apache_combined, apache_error, nginx, paloalto, winevt #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 14 additions & 74 deletions generator/apache_combined/apache_combined.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/observiq/blitz/generator"
"github.com/observiq/blitz/generator/count"
"github.com/observiq/blitz/internal/datagen"
"github.com/observiq/blitz/internal/useragent"
"github.com/observiq/blitz/output"
"go.opentelemetry.io/otel/attribute"
Expand Down Expand Up @@ -191,7 +192,7 @@ func (g *ApacheCombinedLogGenerator) generateApacheCombinedLogData() (*apacheCom
}

// Generate remote host IP address
data.remoteHost = generateRandomIP(r)
data.remoteHost = datagen.RandomIPv4(r)

// Identity is typically "-" in CLF
data.identity = "-"
Expand All @@ -217,65 +218,24 @@ func (g *ApacheCombinedLogGenerator) generateApacheCombinedLogData() (*apacheCom
return data, nil
}

// generateRandomIP generates a random IP address
func generateRandomIP(r *rand.Rand) string {
return fmt.Sprintf("%d.%d.%d.%d",
r.Intn(256), // #nosec G404
r.Intn(256), // #nosec G404
r.Intn(256), // #nosec G404
r.Intn(256)) // #nosec G404
}

// generateRequest generates a random HTTP request string
func generateRequest(r *rand.Rand) string {
methods := []string{"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}
method := methods[r.Intn(len(methods))] // #nosec G404

paths := []string{
"/api/v1/users",
"/api/v1/orders",
"/health",
"/status",
"/api/v2/data",
"/index.html",
"/api/v1/auth",
"/api/v1/payments",
"/api/v1/transactions",
"/api/v1/accounts",
"/api/v1/products",
"/api/v1/inventory",
"/api/v1/customers",
"/api/v1/loans",
"/api/v1/transfers",
"/api/v1/verification",
}

path := paths[r.Intn(len(paths))] // #nosec G404

protocols := []string{"HTTP/1.0", "HTTP/1.1", "HTTP/2.0"}
protocol := protocols[r.Intn(len(protocols))] // #nosec G404

method := datagen.Methods.Random(r)
path := datagen.APIPaths.Random(r)
protocol := datagen.Protocols.Random(r)
return fmt.Sprintf("%s %s %s", method, path, protocol)
}

// generateStatusAndSeverity generates a random HTTP status code and corresponding severity
func generateStatusAndSeverity(r *rand.Rand) (int, string) {
// Weight status codes to be more realistic (mostly 2xx, some 4xx, few 5xx)
roll := r.Float64() // #nosec G404

status := datagen.RandomStatusCode(r)
switch {
case roll < 0.85: // 85% success
statusCodes := []int{200, 201, 204}
status := statusCodes[r.Intn(len(statusCodes))] // #nosec G404
return status, "INFO"
case roll < 0.95: // 10% client errors
statusCodes := []int{400, 401, 403, 404, 429}
status := statusCodes[r.Intn(len(statusCodes))] // #nosec G404
return status, "WARN"
default: // 5% server errors
statusCodes := []int{500, 502, 503, 504}
status := statusCodes[r.Intn(len(statusCodes))] // #nosec G404
case status >= 500:
return status, "ERROR"
case status >= 400:
return status, "WARN"
default:
return status, "INFO"
}
}

Expand All @@ -286,29 +246,9 @@ func generateReferer(r *rand.Rand) string {
return "-"
}

domains := []string{
"https://www.example.com",
"https://search.example.com",
"https://www.google.com",
"https://www.bing.com",
"https://github.com",
"https://stackoverflow.com",
}

pages := []string{
"/",
"/search",
"/page1",
"/page2",
"/index.html",
"/products",
"/about",
}

domain := domains[r.Intn(len(domains))] // #nosec G404
page := pages[r.Intn(len(pages))] // #nosec G404

return fmt.Sprintf("%s%s", domain, page)
domain := datagen.RefererDomains.Random(r)
path := datagen.APIPaths.Random(r)
return fmt.Sprintf("https://%s%s", domain, path)
}

// formatAsApacheCombined converts apacheCombinedLogData to Apache Combined Log Format
Expand Down
12 changes: 2 additions & 10 deletions generator/apache_error/apache_error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/observiq/blitz/generator"
"github.com/observiq/blitz/generator/count"
"github.com/observiq/blitz/internal/datagen"
"github.com/observiq/blitz/output"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
Expand Down Expand Up @@ -220,7 +221,7 @@ func (g *ApacheErrorLogGenerator) generateApacheErrorLogData() (*apacheErrorLogD

// Generate client IP (sometimes empty for non-client errors)
if r.Float64() < 0.8 { // #nosec G404
data.client = generateRandomIP(r)
data.client = datagen.RandomIPv4(r)
} else {
data.client = ""
}
Expand All @@ -231,15 +232,6 @@ func (g *ApacheErrorLogGenerator) generateApacheErrorLogData() (*apacheErrorLogD
return data, nil
}

// generateRandomIP generates a random IP address
func generateRandomIP(r *rand.Rand) string {
return fmt.Sprintf("%d.%d.%d.%d",
r.Intn(256), // #nosec G404
r.Intn(256), // #nosec G404
r.Intn(256), // #nosec G404
r.Intn(256)) // #nosec G404
}

// generateErrorMessage generates a random error message based on log level
func generateErrorMessage(r *rand.Rand, level string) string {
switch level {
Expand Down
102 changes: 14 additions & 88 deletions generator/nginx/nginx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/observiq/blitz/generator"
"github.com/observiq/blitz/generator/count"
"github.com/observiq/blitz/internal/datagen"
"github.com/observiq/blitz/internal/generator/security"
"github.com/observiq/blitz/internal/useragent"
"github.com/observiq/blitz/output"
Expand Down Expand Up @@ -59,67 +60,7 @@ type Generator struct {
}

// Predefined lists for fast random generation
var (
httpMethods = []string{"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}

httpPaths = []string{
"/",
"/index.html",
"/api/v1/users",
"/api/v1/orders",
"/api/v1/products",
"/api/v1/inventory",
"/api/v1/customers",
"/api/v1/payments",
"/api/v1/transactions",
"/api/v1/accounts",
"/api/v1/auth",
"/api/v1/loans",
"/api/v1/transfers",
"/api/v1/verification",
"/api/v2/data",
"/health",
"/status",
"/about",
"/contact",
"/search",
"/login",
"/logout",
"/dashboard",
"/profile",
"/settings",
}

httpProtocols = []string{"HTTP/1.0", "HTTP/1.1", "HTTP/2.0"}

statusCodes2xx = []int{200, 201, 204}
statusCodes4xx = []int{400, 401, 403, 404, 429}
statusCodes5xx = []int{500, 502, 503, 504}

refererDomains = []string{
"https://www.example.com",
"https://search.example.com",
"https://www.google.com",
"https://www.bing.com",
"https://github.com",
"https://stackoverflow.com",
"https://www.reddit.com",
"https://www.linkedin.com",
}

refererPages = []string{
"/",
"/search",
"/page1",
"/page2",
"/index.html",
"/products",
"/about",
"/contact",
}

remoteUsers = []string{"-", "admin", "user1", "user2", "guest"}
)
var remoteUsers = []string{"-", "admin", "user1", "user2", "guest"}

// New creates a new NGINX log generator
func New(logger *zap.Logger, workers int, rate time.Duration) (*Generator, error) {
Expand Down Expand Up @@ -261,7 +202,7 @@ func (g *Generator) generateNginxLogData() (*nginxLogData, error) {
timestamp: time.Now(),
}

data.remoteAddr = generateRandomIP(r)
data.remoteAddr = datagen.RandomIPv4(r)
data.remoteUser = remoteUsers[r.Intn(len(remoteUsers))] // #nosec G404
data.request = generateRequest(r)
data.statusCode, data.severity = generateStatusAndSeverity(r)
Expand All @@ -272,46 +213,32 @@ func (g *Generator) generateNginxLogData() (*nginxLogData, error) {
return data, nil
}

// generateRandomIP generates a random IP address
func generateRandomIP(r *rand.Rand) string {
return fmt.Sprintf("%d.%d.%d.%d",
r.Intn(256), // #nosec G404
r.Intn(256), // #nosec G404
r.Intn(256), // #nosec G404
r.Intn(256)) // #nosec G404
}

// generateRequest generates a random HTTP request string
func generateRequest(r *rand.Rand) string {
method := httpMethods[r.Intn(len(httpMethods))] // #nosec G404
method := datagen.Methods.Random(r)

// 20% chance of generating a security-focused path
var path string
if r.Float64() < 0.20 { // #nosec G404
path = security.RandomAttackPath(r)
} else {
path = httpPaths[r.Intn(len(httpPaths))] // #nosec G404
path = datagen.APIPaths.Random(r)
}

protocol := httpProtocols[r.Intn(len(httpProtocols))] // #nosec G404

protocol := datagen.Protocols.Random(r)
return fmt.Sprintf("%s %s %s", method, path, protocol)
}

// generateStatusAndSeverity generates a random HTTP status code and corresponding severity
func generateStatusAndSeverity(r *rand.Rand) (int, string) {
roll := r.Float64() // #nosec G404

status := datagen.RandomStatusCode(r)
switch {
case roll < 0.85:
status := statusCodes2xx[r.Intn(len(statusCodes2xx))] // #nosec G404
return status, severityInfo
case roll < 0.95:
status := statusCodes4xx[r.Intn(len(statusCodes4xx))] // #nosec G404
case status >= 500:
return status, severityError
case status >= 400:
return status, severityWarn
default:
status := statusCodes5xx[r.Intn(len(statusCodes5xx))] // #nosec G404
return status, severityError
return status, severityInfo
}
}

Expand All @@ -321,10 +248,9 @@ func generateReferer(r *rand.Rand) string {
return emptyValue
}

domain := refererDomains[r.Intn(len(refererDomains))] // #nosec G404
page := refererPages[r.Intn(len(refererPages))] // #nosec G404

return fmt.Sprintf("%s%s", domain, page)
domain := datagen.RefererDomains.Random(r)
path := datagen.APIPaths.Random(r)
return fmt.Sprintf("https://%s%s", domain, path)
}

// formatAsNginxCombined converts nginxLogData to NGINX Combined Log Format
Expand Down
5 changes: 3 additions & 2 deletions generator/paloalto/paloalto.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/observiq/blitz/generator"
"github.com/observiq/blitz/generator/count"
"github.com/observiq/blitz/internal/datagen"
"github.com/observiq/blitz/output"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
Expand Down Expand Up @@ -299,9 +300,9 @@ func generateRandomIP() string {
}

func generateRandomPort() string {
common := []int{80, 443, 8080, 8088, 22, 21, 25, 53, 110, 143, 993, 995, 3306, 5432, 6379, 27017}
commonPorts := datagen.CommonPorts.All()
if randInt(0, 10) < 7 {
return strconv.Itoa(common[randInt(0, len(common)-1)])
return strconv.Itoa(commonPorts[randInt(0, len(commonPorts)-1)])
}
return strconv.Itoa(randInt(1024, 65535))
}
Expand Down
34 changes: 13 additions & 21 deletions internal/generators/winevt/templates/templates.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"math/rand"
"strings"
"time"

"github.com/observiq/blitz/internal/datagen"
)

// RenderOptions controls rendering of a Windows Event template.
Expand All @@ -18,28 +20,18 @@ type RenderOptions struct {
Hostnames []string
}

// DefaultIPs provides fallback IP candidates if none are configured.
var DefaultIPs = []string{
"103.165.114.4",
"192.0.2.10",
"198.51.100.23",
"203.0.113.77",
"10.0.0.5",
}
// DefaultIPs provides fallback IP candidates generated from a fixed seed.
var DefaultIPs = func() []string {
r := rand.New(rand.NewSource(42)) // #nosec G404 - deterministic default
ips := make([]string, 5)
for i := range ips {
ips[i] = datagen.RandomPrivateIPv4(r)
}
return ips
}()

// DefaultHostnames provides fallback hostname candidates if none are configured.
var DefaultHostnames = []string{
"iis-east1-prd-0",
"web-west2-stg-1",
"db-north1-prod-2",
"app-south1-dev-3",
"cache-central1-prod-4",
"api-east2-stg-5",
"worker-west1-prod-6",
"monitor-north2-prod-7",
"gateway-south2-stg-8",
"loadbalancer-central2-prod-9",
}
// DefaultHostnames provides fallback hostname candidates using mythology names.
var DefaultHostnames = datagen.GenerateHostnames(42, 10, datagen.StyleWindows, datagen.RomanNames)

// templateNames is a pre-computed list of all available template names for efficient random selection.
var templateNames = []string{
Expand Down
Loading