feat: add automatic wildcard detection (--auto-wildcard)

internal/runner/options.go

@@ -59,6 +59,7 @@ type Options struct {
 	TraceMaxRecursion     int
 	WildcardThreshold     int
 	WildcardDomain        string
+	AutoWildcard          bool
 	ShowStatistics        bool
 	rcodes                map[int]struct{}
 	RCode                 string
@@ -189,6 +190,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"),
 		flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"),
 		flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"),
+		flagSet.BoolVarP(&options.AutoWildcard, "auto-wildcard", "aw", false, "automatically detect and filter wildcards for all input domains"),
 		flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"),
 	)
 
@@ -267,6 +269,10 @@ func (options *Options) validateOptions() {
 		gologger.Fatal().Msgf("resp and resp-only can't be used at the same time")
 	}
 
+	if options.AutoWildcard && options.WildcardDomain != "" {
+		gologger.Fatal().Msgf("auto-wildcard and wildcard-domain are mutually exclusive")
+	}
+
 	if options.Retries == 0 {
 		gologger.Fatal().Msgf("retries must be at least 1")
 	}
@@ -307,6 +313,9 @@ func (options *Options) validateOptions() {
 		if options.WildcardDomain != "" {
 			gologger.Fatal().Msgf("wildcard not supported in stream mode")
 		}
+		if options.AutoWildcard {
+			gologger.Fatal().Msgf("auto-wildcard not supported in stream mode")
+		}
 		if options.ShowStatistics {
 			gologger.Fatal().Msgf("stats not supported in stream mode")
 		}

internal/runner/runner.go

@@ -32,22 +32,24 @@ import (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	options             *Options
-	dnsx                *dnsx.DNSX
-	wgoutputworker      *sync.WaitGroup
-	wgresolveworkers    *sync.WaitGroup
-	wgwildcardworker    *sync.WaitGroup
-	workerchan          chan string
-	outputchan          chan string
-	wildcardworkerchan  chan string
-	wildcards           *mapsutil.SyncLockMap[string, struct{}]
-	wildcardscache      map[string][]string
-	wildcardscachemutex sync.Mutex
-	limiter             *ratelimit.Limiter
-	hm                  *hybrid.HybridMap
-	stats               clistats.StatisticsClient
-	tmpStdinFile        string
-	aurora              aurora.Aurora
+	options                  *Options
+	dnsx                     *dnsx.DNSX
+	wgoutputworker           *sync.WaitGroup
+	wgresolveworkers         *sync.WaitGroup
+	wgwildcardworker         *sync.WaitGroup
+	workerchan               chan string
+	outputchan               chan string
+	wildcardworkerchan       chan string
+	wildcards                *mapsutil.SyncLockMap[string, struct{}]
+	wildcardscache           map[string][]string
+	wildcardscachemutex      sync.Mutex
+	autoWildcardDomains      map[string]struct{}
+	autoWildcardDomainsMutex sync.RWMutex
+	limiter                  *ratelimit.Limiter
+	hm                       *hybrid.HybridMap
+	stats                    clistats.StatisticsClient
+	tmpStdinFile             string
+	aurora                   aurora.Aurora
 }
 
 func New(options *Options) (*Runner, error) {
@@ -150,19 +152,20 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	r := Runner{
-		options:            options,
-		dnsx:               dnsX,
-		wgoutputworker:     &sync.WaitGroup{},
-		wgresolveworkers:   &sync.WaitGroup{},
-		wgwildcardworker:   &sync.WaitGroup{},
-		workerchan:         make(chan string),
-		wildcardworkerchan: make(chan string),
-		wildcards:          mapsutil.NewSyncLockMap[string, struct{}](),
-		wildcardscache:     make(map[string][]string),
-		limiter:            limiter,
-		hm:                 hm,
-		stats:              stats,
-		aurora:             aurora.NewAurora(!options.NoColor),
+		options:             options,
+		dnsx:                dnsX,
+		wgoutputworker:      &sync.WaitGroup{},
+		wgresolveworkers:    &sync.WaitGroup{},
+		wgwildcardworker:    &sync.WaitGroup{},
+		workerchan:          make(chan string),
+		wildcardworkerchan:  make(chan string),
+		wildcards:           mapsutil.NewSyncLockMap[string, struct{}](),
+		wildcardscache:      make(map[string][]string),
+		autoWildcardDomains: make(map[string]struct{}),
+		limiter:             limiter,
+		hm:                  hm,
+		stats:               stats,
+		aurora:              aurora.NewAurora(!options.NoColor),
 	}
 
 	return &r, nil
@@ -454,6 +457,14 @@ func (r *Runner) run() error {
 		gologger.Debug().Msgf("Resuming scan using file %s. Restarting at position %d: %s\n", DefaultResumeFile, r.options.resumeCfg.Index, r.options.resumeCfg.ResumeFrom)
 	}
 
+	// Auto wildcard detection
+	if r.options.AutoWildcard {
+		err = r.AutoDetectWildcards()
+		if err != nil {
+			return err
+		}
+	}
+
 	r.startWorkers()
 
 	r.wgresolveworkers.Wait()
@@ -548,6 +559,43 @@ func (r *Runner) run() error {
 		gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains)
 	}
 
+	// Auto wildcard filtering - output results, filtering detected wildcard domains
+	if r.options.AutoWildcard {
+		gologger.Print().Msgf("Starting to filter auto-detected wildcard domains\n")
+
+		// we need to restart output
+		r.startOutputWorker()
+
+		seen := make(map[string]struct{})
+		numRemovedSubdomains := 0
+
+		r.hm.Scan(func(k, v []byte) error {
+			host := string(k)
+			rootDomain := getRootDomain(host)
+
+			// Skip if this domain was detected as wildcard
+			if r.isAutoWildcardDomain(rootDomain) {
+				if _, ok := seen[host]; !ok {
+					numRemovedSubdomains++
+					seen[host] = struct{}{}
+				}
+				return nil
+			}
+
+			// Output non-wildcard results
+			if _, ok := seen[host]; !ok {
+				seen[host] = struct{}{}
+				_ = r.lookupAndOutput(host)
+			}
+			return nil
+		})
+
+		close(r.outputchan)
+		// waiting output worker
+		r.wgoutputworker.Wait()
+		gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains)
+	}
+
 	return nil
 }
 
@@ -731,7 +779,7 @@ func (r *Runner) worker() {
 			}
 		}
 		// if wildcard filtering just store the data
-		if r.options.WildcardDomain != "" {
+		if r.options.WildcardDomain != "" || r.options.AutoWildcard {
 			if err := r.storeDNSData(dnsData.DNSData); err != nil {
 				gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err)
 			}

internal/runner/wildcard.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"strings"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/rs/xid"
 )
 
@@ -69,3 +70,97 @@ func (r *Runner) IsWildcard(host string) bool {
 
 	return false
 }
+
+// getRootDomain extracts the root domain from a full domain.
+// Note: this is a simple heuristic that takes the last two labels,
+// which works for most TLDs (e.g. example.com) but not for multi-part
+// TLDs like co.uk or com.au. A public suffix list could be used for
+// more accurate extraction if needed.
+func getRootDomain(host string) string {
+	parts := strings.Split(host, ".")
+	if len(parts) >= 2 {
+		return strings.Join(parts[len(parts)-2:], ".")
+	}
+	return host
+}
+
+// detectWildcardForDomain detects if a domain has wildcard DNS.
+// Uses A-record lookups regardless of configured query types to ensure
+// reliable detection even when the user queries non-A record types.
+func (r *Runner) detectWildcardForDomain(domain string) bool {
+	// Query a random subdomain using A-record lookup
+	randomID := xid.New().String()
+	testHost := randomID + "." + domain
+
+	testIPs, err := r.dnsx.Lookup(testHost)
+	if err != nil || len(testIPs) == 0 {
+		return false
+	}
+
+	// If we got a response, query the root domain
+	rootIPs, err := r.dnsx.Lookup(domain)
+	if err != nil || len(rootIPs) == 0 {
+		// Root domain doesn't resolve but random subdomain does - likely wildcard
+		return true
+	}
+
+	// Check if the same IPs are returned (indicating wildcard)
+	rootIPSet := make(map[string]struct{})
+	for _, ip := range rootIPs {
+		rootIPSet[ip] = struct{}{}
+	}
+
+	for _, ip := range testIPs {
+		if _, ok := rootIPSet[ip]; !ok {
+			// Different IP for random subdomain - not a wildcard
+			return false
+		}
+	}
+
+	// Same IPs returned - likely a wildcard
+	return true
+}
+
+// AutoDetectWildcards automatically detects wildcard domains from the input
+// and populates the runner's wildcard detection data
+func (r *Runner) AutoDetectWildcards() error {
+	if !r.options.AutoWildcard {
+		return nil
+	}
+
+	gologger.Info().Msgf("Starting automatic wildcard detection\n")
+
+	// Collect all unique root domains from input
+	domains := make(map[string]struct{})
+	r.hm.Scan(func(k, v []byte) error {
+		host := string(k)
+		rootDomain := getRootDomain(host)
+		domains[rootDomain] = struct{}{}
+		return nil
+	})
+
+	gologger.Info().Msgf("Detected %d unique domains for wildcard check\n", len(domains))
+
+	// Test each domain for wildcard
+	for domain := range domains {
+		if r.detectWildcardForDomain(domain) {
+			r.autoWildcardDomainsMutex.Lock()
+			r.autoWildcardDomains[domain] = struct{}{}
+			r.autoWildcardDomainsMutex.Unlock()
+			gologger.Info().Msgf("Wildcard detected for domain: %s\n", domain)
+		}
+	}
+
+	gologger.Info().Msgf("Automatic wildcard detection complete. Found %d wildcard domains\n", len(r.autoWildcardDomains))
+
+	return nil
+}
+
+// isAutoWildcardDomain checks if a domain was detected as having wildcards
+func (r *Runner) isAutoWildcardDomain(domain string) bool {
+	r.autoWildcardDomainsMutex.RLock()
+	defer r.autoWildcardDomainsMutex.RUnlock()
+	_, ok := r.autoWildcardDomains[domain]
+	return ok
+}
+