From 85f9e848e320011a8203050e3f6db2e14e097a5f Mon Sep 17 00:00:00 2001 From: David Flagg Date: Sun, 15 Mar 2026 10:11:13 -0400 Subject: [PATCH 1/2] feat: add automatic wildcard detection for all input domains Adds --auto-wildcard / -aw flag that automatically detects and filters wildcard DNS domains across all input, similar to PureDNS. How it works: 1. Before resolution, extracts unique root domains from all inputs 2. Probes each domain with a random subdomain (xid-generated) 3. Compares response IPs against root domain IPs 4. Domains returning the same IPs for random subdomains are marked as wildcard and their results are filtered from output This eliminates the need to manually specify -wd for each domain, making wildcard filtering practical for large multi-domain scans. Changes: - internal/runner/options.go: Add AutoWildcard bool field and -aw flag, block in stream mode (consistent with existing -wd behavior) - internal/runner/wildcard.go: Add auto-detection logic with thread-safe domain tracking (RWMutex), root domain extraction, and per-domain wildcard probing - internal/runner/runner.go: Integrate auto-detection before workers start, add post-processing filter for detected wildcard domains Fixes #924 --- internal/runner/options.go | 5 ++ internal/runner/runner.go | 45 +++++++++++++++++ internal/runner/wildcard.go | 97 +++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..324a35e8 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,7 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + AutoWildcard bool ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +190,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVarP(&options.AutoWildcard, "auto-wildcard", "aw", false, "automatically detect and filter wildcards for all input domains"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) @@ -307,6 +309,9 @@ func (options *Options) validateOptions() { if options.WildcardDomain != "" { gologger.Fatal().Msgf("wildcard not supported in stream mode") } + if options.AutoWildcard { + gologger.Fatal().Msgf("auto-wildcard not supported in stream mode") + } if options.ShowStatistics { gologger.Fatal().Msgf("stats not supported in stream mode") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..52680f8b 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -454,6 +454,14 @@ func (r *Runner) run() error { gologger.Debug().Msgf("Resuming scan using file %s. Restarting at position %d: %s\n", DefaultResumeFile, r.options.resumeCfg.Index, r.options.resumeCfg.ResumeFrom) } + // Auto wildcard detection + if r.options.AutoWildcard { + err = r.AutoDetectWildcards() + if err != nil { + return err + } + } + r.startWorkers() r.wgresolveworkers.Wait() @@ -548,6 +556,43 @@ func (r *Runner) run() error { gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) } + // Auto wildcard filtering - filter results from detected wildcard domains + if r.options.AutoWildcard && len(autoWildcardDomains) > 0 { + gologger.Print().Msgf("Starting to filter auto-detected wildcard domains\n") + + // we need to restart output + r.startOutputWorker() + + seen := make(map[string]struct{}) + numRemovedSubdomains := 0 + + r.hm.Scan(func(k, v []byte) error { + host := string(k) + rootDomain := getRootDomain(host) + + // Skip if this domain was detected as wildcard + if IsAutoWildcardDomain(rootDomain) { + if _, ok := seen[host]; !ok { + numRemovedSubdomains++ + seen[host] = struct{}{} + } + return nil + } + + // Output non-wildcard results + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + _ = r.lookupAndOutput(host) + } + return nil + }) + + close(r.outputchan) + // waiting output worker + r.wgoutputworker.Wait() + gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) + } + return nil } diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 5fdfc6a4..bf85af45 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -2,10 +2,16 @@ package runner import ( "strings" + "sync" + "github.com/projectdiscovery/gologger" "github.com/rs/xid" ) +// autoWildcardDomains stores domains that have been detected as wildcard +var autoWildcardDomains = make(map[string]struct{}) +var autoWildcardDomainsMutex sync.RWMutex + // IsWildcard checks if a host is wildcard func (r *Runner) IsWildcard(host string) bool { orig := make(map[string]struct{}) @@ -69,3 +75,94 @@ func (r *Runner) IsWildcard(host string) bool { return false } + +// getRootDomain extracts the root domain from a full domain. +// Note: this is a simple heuristic that takes the last two labels, +// which works for most TLDs (e.g. example.com) but not for multi-part +// TLDs like co.uk or com.au. A public suffix list could be used for +// more accurate extraction if needed. +func getRootDomain(host string) string { + parts := strings.Split(host, ".") + if len(parts) >= 2 { + return strings.Join(parts[len(parts)-2:], ".") + } + return host +} + +// detectWildcardForDomain detects if a domain has wildcard DNS +func (r *Runner) detectWildcardForDomain(domain string) bool { + // Query a random subdomain to see if we get a response + randomID := xid.New().String() + testHost := randomID + "." + domain + + in, err := r.dnsx.QueryOne(testHost) + if err != nil || in == nil || len(in.A) == 0 { + return false + } + + // If we got a response, query the root domain + rootResult, err := r.dnsx.QueryOne(domain) + if err != nil || rootResult == nil { + // Root domain doesn't resolve but subdomain does - likely wildcard + return true + } + + // Check if the same IPs are returned (indicating wildcard) + rootIPs := make(map[string]struct{}) + for _, a := range rootResult.A { + rootIPs[a] = struct{}{} + } + + for _, a := range in.A { + if _, ok := rootIPs[a]; !ok { + // Different IP for random subdomain - not a wildcard at root level + return false + } + } + + // Same IP returned - likely a wildcard + return true +} + +// AutoDetectWildcards automatically detects wildcard domains from the input +// and populates the wildcard detection data +func (r *Runner) AutoDetectWildcards() error { + if !r.options.AutoWildcard { + return nil + } + + gologger.Info().Msgf("Starting automatic wildcard detection\n") + + // Collect all unique root domains from input + domains := make(map[string]struct{}) + r.hm.Scan(func(k, v []byte) error { + host := string(k) + rootDomain := getRootDomain(host) + domains[rootDomain] = struct{}{} + return nil + }) + + gologger.Info().Msgf("Detected %d unique domains for wildcard check\n", len(domains)) + + // Test each domain for wildcard + for domain := range domains { + if r.detectWildcardForDomain(domain) { + autoWildcardDomainsMutex.Lock() + autoWildcardDomains[domain] = struct{}{} + autoWildcardDomainsMutex.Unlock() + gologger.Info().Msgf("Wildcard detected for domain: %s\n", domain) + } + } + + gologger.Info().Msgf("Automatic wildcard detection complete. Found %d wildcard domains\n", len(autoWildcardDomains)) + + return nil +} + +// IsAutoWildcardDomain checks if a domain was detected as having wildcards +func IsAutoWildcardDomain(domain string) bool { + autoWildcardDomainsMutex.RLock() + defer autoWildcardDomainsMutex.RUnlock() + _, ok := autoWildcardDomains[domain] + return ok +} From cce4de811db85300d123e1e432f74356c0ec6cdf Mon Sep 17 00:00:00 2001 From: David Flagg Date: Fri, 20 Mar 2026 10:29:37 -0400 Subject: [PATCH 2/2] fix: address review feedback for auto-wildcard feature - Add mutual exclusion validation for --auto-wildcard and --wildcard-domain - Buffer results when auto-wildcard is active (store DNS data during worker phase, output only in post-processing) to prevent double output - Use Lookup() for wildcard detection to always check A records regardless of configured query types (fixes detection when using -aaaa, -cname, etc.) - Scope wildcard registry to Runner struct instead of package-global state to prevent cross-run leakage Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/runner/options.go | 4 +++ internal/runner/runner.go | 69 +++++++++++++++++++------------------ internal/runner/wildcard.go | 56 +++++++++++++++--------------- 3 files changed, 67 insertions(+), 62 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index 324a35e8..e7e7eed6 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -269,6 +269,10 @@ func (options *Options) validateOptions() { gologger.Fatal().Msgf("resp and resp-only can't be used at the same time") } + if options.AutoWildcard && options.WildcardDomain != "" { + gologger.Fatal().Msgf("auto-wildcard and wildcard-domain are mutually exclusive") + } + if options.Retries == 0 { gologger.Fatal().Msgf("retries must be at least 1") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 52680f8b..4036d12f 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -32,22 +32,24 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - dnsx *dnsx.DNSX - wgoutputworker *sync.WaitGroup - wgresolveworkers *sync.WaitGroup - wgwildcardworker *sync.WaitGroup - workerchan chan string - outputchan chan string - wildcardworkerchan chan string - wildcards *mapsutil.SyncLockMap[string, struct{}] - wildcardscache map[string][]string - wildcardscachemutex sync.Mutex - limiter *ratelimit.Limiter - hm *hybrid.HybridMap - stats clistats.StatisticsClient - tmpStdinFile string - aurora aurora.Aurora + options *Options + dnsx *dnsx.DNSX + wgoutputworker *sync.WaitGroup + wgresolveworkers *sync.WaitGroup + wgwildcardworker *sync.WaitGroup + workerchan chan string + outputchan chan string + wildcardworkerchan chan string + wildcards *mapsutil.SyncLockMap[string, struct{}] + wildcardscache map[string][]string + wildcardscachemutex sync.Mutex + autoWildcardDomains map[string]struct{} + autoWildcardDomainsMutex sync.RWMutex + limiter *ratelimit.Limiter + hm *hybrid.HybridMap + stats clistats.StatisticsClient + tmpStdinFile string + aurora aurora.Aurora } func New(options *Options) (*Runner, error) { @@ -150,19 +152,20 @@ func New(options *Options) (*Runner, error) { } r := Runner{ - options: options, - dnsx: dnsX, - wgoutputworker: &sync.WaitGroup{}, - wgresolveworkers: &sync.WaitGroup{}, - wgwildcardworker: &sync.WaitGroup{}, - workerchan: make(chan string), - wildcardworkerchan: make(chan string), - wildcards: mapsutil.NewSyncLockMap[string, struct{}](), - wildcardscache: make(map[string][]string), - limiter: limiter, - hm: hm, - stats: stats, - aurora: aurora.NewAurora(!options.NoColor), + options: options, + dnsx: dnsX, + wgoutputworker: &sync.WaitGroup{}, + wgresolveworkers: &sync.WaitGroup{}, + wgwildcardworker: &sync.WaitGroup{}, + workerchan: make(chan string), + wildcardworkerchan: make(chan string), + wildcards: mapsutil.NewSyncLockMap[string, struct{}](), + wildcardscache: make(map[string][]string), + autoWildcardDomains: make(map[string]struct{}), + limiter: limiter, + hm: hm, + stats: stats, + aurora: aurora.NewAurora(!options.NoColor), } return &r, nil @@ -556,8 +559,8 @@ func (r *Runner) run() error { gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) } - // Auto wildcard filtering - filter results from detected wildcard domains - if r.options.AutoWildcard && len(autoWildcardDomains) > 0 { + // Auto wildcard filtering - output results, filtering detected wildcard domains + if r.options.AutoWildcard { gologger.Print().Msgf("Starting to filter auto-detected wildcard domains\n") // we need to restart output @@ -571,7 +574,7 @@ func (r *Runner) run() error { rootDomain := getRootDomain(host) // Skip if this domain was detected as wildcard - if IsAutoWildcardDomain(rootDomain) { + if r.isAutoWildcardDomain(rootDomain) { if _, ok := seen[host]; !ok { numRemovedSubdomains++ seen[host] = struct{}{} @@ -776,7 +779,7 @@ func (r *Runner) worker() { } } // if wildcard filtering just store the data - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { if err := r.storeDNSData(dnsData.DNSData); err != nil { gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err) } diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index bf85af45..5b3a83b6 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -2,16 +2,11 @@ package runner import ( "strings" - "sync" "github.com/projectdiscovery/gologger" "github.com/rs/xid" ) -// autoWildcardDomains stores domains that have been detected as wildcard -var autoWildcardDomains = make(map[string]struct{}) -var autoWildcardDomainsMutex sync.RWMutex - // IsWildcard checks if a host is wildcard func (r *Runner) IsWildcard(host string) bool { orig := make(map[string]struct{}) @@ -89,43 +84,45 @@ func getRootDomain(host string) string { return host } -// detectWildcardForDomain detects if a domain has wildcard DNS +// detectWildcardForDomain detects if a domain has wildcard DNS. +// Uses A-record lookups regardless of configured query types to ensure +// reliable detection even when the user queries non-A record types. func (r *Runner) detectWildcardForDomain(domain string) bool { - // Query a random subdomain to see if we get a response + // Query a random subdomain using A-record lookup randomID := xid.New().String() testHost := randomID + "." + domain - in, err := r.dnsx.QueryOne(testHost) - if err != nil || in == nil || len(in.A) == 0 { + testIPs, err := r.dnsx.Lookup(testHost) + if err != nil || len(testIPs) == 0 { return false } // If we got a response, query the root domain - rootResult, err := r.dnsx.QueryOne(domain) - if err != nil || rootResult == nil { - // Root domain doesn't resolve but subdomain does - likely wildcard + rootIPs, err := r.dnsx.Lookup(domain) + if err != nil || len(rootIPs) == 0 { + // Root domain doesn't resolve but random subdomain does - likely wildcard return true } // Check if the same IPs are returned (indicating wildcard) - rootIPs := make(map[string]struct{}) - for _, a := range rootResult.A { - rootIPs[a] = struct{}{} + rootIPSet := make(map[string]struct{}) + for _, ip := range rootIPs { + rootIPSet[ip] = struct{}{} } - for _, a := range in.A { - if _, ok := rootIPs[a]; !ok { - // Different IP for random subdomain - not a wildcard at root level + for _, ip := range testIPs { + if _, ok := rootIPSet[ip]; !ok { + // Different IP for random subdomain - not a wildcard return false } } - // Same IP returned - likely a wildcard + // Same IPs returned - likely a wildcard return true } // AutoDetectWildcards automatically detects wildcard domains from the input -// and populates the wildcard detection data +// and populates the runner's wildcard detection data func (r *Runner) AutoDetectWildcards() error { if !r.options.AutoWildcard { return nil @@ -147,22 +144,23 @@ func (r *Runner) AutoDetectWildcards() error { // Test each domain for wildcard for domain := range domains { if r.detectWildcardForDomain(domain) { - autoWildcardDomainsMutex.Lock() - autoWildcardDomains[domain] = struct{}{} - autoWildcardDomainsMutex.Unlock() + r.autoWildcardDomainsMutex.Lock() + r.autoWildcardDomains[domain] = struct{}{} + r.autoWildcardDomainsMutex.Unlock() gologger.Info().Msgf("Wildcard detected for domain: %s\n", domain) } } - gologger.Info().Msgf("Automatic wildcard detection complete. Found %d wildcard domains\n", len(autoWildcardDomains)) + gologger.Info().Msgf("Automatic wildcard detection complete. Found %d wildcard domains\n", len(r.autoWildcardDomains)) return nil } -// IsAutoWildcardDomain checks if a domain was detected as having wildcards -func IsAutoWildcardDomain(domain string) bool { - autoWildcardDomainsMutex.RLock() - defer autoWildcardDomainsMutex.RUnlock() - _, ok := autoWildcardDomains[domain] +// isAutoWildcardDomain checks if a domain was detected as having wildcards +func (r *Runner) isAutoWildcardDomain(domain string) bool { + r.autoWildcardDomainsMutex.RLock() + defer r.autoWildcardDomainsMutex.RUnlock() + _, ok := r.autoWildcardDomains[domain] return ok } +