diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..e7e7eed6 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,7 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + AutoWildcard bool ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +190,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVarP(&options.AutoWildcard, "auto-wildcard", "aw", false, "automatically detect and filter wildcards for all input domains"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) @@ -267,6 +269,10 @@ func (options *Options) validateOptions() { gologger.Fatal().Msgf("resp and resp-only can't be used at the same time") } + if options.AutoWildcard && options.WildcardDomain != "" { + gologger.Fatal().Msgf("auto-wildcard and wildcard-domain are mutually exclusive") + } + if options.Retries == 0 { gologger.Fatal().Msgf("retries must be at least 1") } @@ -307,6 +313,9 @@ func (options *Options) validateOptions() { if options.WildcardDomain != "" { gologger.Fatal().Msgf("wildcard not supported in stream mode") } + if options.AutoWildcard { + gologger.Fatal().Msgf("auto-wildcard not supported in stream mode") + } if options.ShowStatistics { gologger.Fatal().Msgf("stats not supported in stream mode") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..4036d12f 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -32,22 +32,24 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - dnsx *dnsx.DNSX - wgoutputworker *sync.WaitGroup - wgresolveworkers *sync.WaitGroup - wgwildcardworker *sync.WaitGroup - workerchan chan string - outputchan chan string - wildcardworkerchan chan string - wildcards *mapsutil.SyncLockMap[string, struct{}] - wildcardscache map[string][]string - wildcardscachemutex sync.Mutex - limiter *ratelimit.Limiter - hm *hybrid.HybridMap - stats clistats.StatisticsClient - tmpStdinFile string - aurora aurora.Aurora + options *Options + dnsx *dnsx.DNSX + wgoutputworker *sync.WaitGroup + wgresolveworkers *sync.WaitGroup + wgwildcardworker *sync.WaitGroup + workerchan chan string + outputchan chan string + wildcardworkerchan chan string + wildcards *mapsutil.SyncLockMap[string, struct{}] + wildcardscache map[string][]string + wildcardscachemutex sync.Mutex + autoWildcardDomains map[string]struct{} + autoWildcardDomainsMutex sync.RWMutex + limiter *ratelimit.Limiter + hm *hybrid.HybridMap + stats clistats.StatisticsClient + tmpStdinFile string + aurora aurora.Aurora } func New(options *Options) (*Runner, error) { @@ -150,19 +152,20 @@ func New(options *Options) (*Runner, error) { } r := Runner{ - options: options, - dnsx: dnsX, - wgoutputworker: &sync.WaitGroup{}, - wgresolveworkers: &sync.WaitGroup{}, - wgwildcardworker: &sync.WaitGroup{}, - workerchan: make(chan string), - wildcardworkerchan: make(chan string), - wildcards: mapsutil.NewSyncLockMap[string, struct{}](), - wildcardscache: make(map[string][]string), - limiter: limiter, - hm: hm, - stats: stats, - aurora: aurora.NewAurora(!options.NoColor), + options: options, + dnsx: dnsX, + wgoutputworker: &sync.WaitGroup{}, + wgresolveworkers: &sync.WaitGroup{}, + wgwildcardworker: &sync.WaitGroup{}, + workerchan: make(chan string), + wildcardworkerchan: make(chan string), + wildcards: mapsutil.NewSyncLockMap[string, struct{}](), + wildcardscache: make(map[string][]string), + autoWildcardDomains: make(map[string]struct{}), + limiter: limiter, + hm: hm, + stats: stats, + aurora: aurora.NewAurora(!options.NoColor), } return &r, nil @@ -454,6 +457,14 @@ func (r *Runner) run() error { gologger.Debug().Msgf("Resuming scan using file %s. Restarting at position %d: %s\n", DefaultResumeFile, r.options.resumeCfg.Index, r.options.resumeCfg.ResumeFrom) } + // Auto wildcard detection + if r.options.AutoWildcard { + err = r.AutoDetectWildcards() + if err != nil { + return err + } + } + r.startWorkers() r.wgresolveworkers.Wait() @@ -548,6 +559,43 @@ func (r *Runner) run() error { gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) } + // Auto wildcard filtering - output results, filtering detected wildcard domains + if r.options.AutoWildcard { + gologger.Print().Msgf("Starting to filter auto-detected wildcard domains\n") + + // we need to restart output + r.startOutputWorker() + + seen := make(map[string]struct{}) + numRemovedSubdomains := 0 + + r.hm.Scan(func(k, v []byte) error { + host := string(k) + rootDomain := getRootDomain(host) + + // Skip if this domain was detected as wildcard + if r.isAutoWildcardDomain(rootDomain) { + if _, ok := seen[host]; !ok { + numRemovedSubdomains++ + seen[host] = struct{}{} + } + return nil + } + + // Output non-wildcard results + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + _ = r.lookupAndOutput(host) + } + return nil + }) + + close(r.outputchan) + // waiting output worker + r.wgoutputworker.Wait() + gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) + } + return nil } @@ -731,7 +779,7 @@ func (r *Runner) worker() { } } // if wildcard filtering just store the data - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { if err := r.storeDNSData(dnsData.DNSData); err != nil { gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err) } diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 5fdfc6a4..5b3a83b6 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -3,6 +3,7 @@ package runner import ( "strings" + "github.com/projectdiscovery/gologger" "github.com/rs/xid" ) @@ -69,3 +70,97 @@ func (r *Runner) IsWildcard(host string) bool { return false } + +// getRootDomain extracts the root domain from a full domain. +// Note: this is a simple heuristic that takes the last two labels, +// which works for most TLDs (e.g. example.com) but not for multi-part +// TLDs like co.uk or com.au. A public suffix list could be used for +// more accurate extraction if needed. +func getRootDomain(host string) string { + parts := strings.Split(host, ".") + if len(parts) >= 2 { + return strings.Join(parts[len(parts)-2:], ".") + } + return host +} + +// detectWildcardForDomain detects if a domain has wildcard DNS. +// Uses A-record lookups regardless of configured query types to ensure +// reliable detection even when the user queries non-A record types. +func (r *Runner) detectWildcardForDomain(domain string) bool { + // Query a random subdomain using A-record lookup + randomID := xid.New().String() + testHost := randomID + "." + domain + + testIPs, err := r.dnsx.Lookup(testHost) + if err != nil || len(testIPs) == 0 { + return false + } + + // If we got a response, query the root domain + rootIPs, err := r.dnsx.Lookup(domain) + if err != nil || len(rootIPs) == 0 { + // Root domain doesn't resolve but random subdomain does - likely wildcard + return true + } + + // Check if the same IPs are returned (indicating wildcard) + rootIPSet := make(map[string]struct{}) + for _, ip := range rootIPs { + rootIPSet[ip] = struct{}{} + } + + for _, ip := range testIPs { + if _, ok := rootIPSet[ip]; !ok { + // Different IP for random subdomain - not a wildcard + return false + } + } + + // Same IPs returned - likely a wildcard + return true +} + +// AutoDetectWildcards automatically detects wildcard domains from the input +// and populates the runner's wildcard detection data +func (r *Runner) AutoDetectWildcards() error { + if !r.options.AutoWildcard { + return nil + } + + gologger.Info().Msgf("Starting automatic wildcard detection\n") + + // Collect all unique root domains from input + domains := make(map[string]struct{}) + r.hm.Scan(func(k, v []byte) error { + host := string(k) + rootDomain := getRootDomain(host) + domains[rootDomain] = struct{}{} + return nil + }) + + gologger.Info().Msgf("Detected %d unique domains for wildcard check\n", len(domains)) + + // Test each domain for wildcard + for domain := range domains { + if r.detectWildcardForDomain(domain) { + r.autoWildcardDomainsMutex.Lock() + r.autoWildcardDomains[domain] = struct{}{} + r.autoWildcardDomainsMutex.Unlock() + gologger.Info().Msgf("Wildcard detected for domain: %s\n", domain) + } + } + + gologger.Info().Msgf("Automatic wildcard detection complete. Found %d wildcard domains\n", len(r.autoWildcardDomains)) + + return nil +} + +// isAutoWildcardDomain checks if a domain was detected as having wildcards +func (r *Runner) isAutoWildcardDomain(domain string) bool { + r.autoWildcardDomainsMutex.RLock() + defer r.autoWildcardDomainsMutex.RUnlock() + _, ok := r.autoWildcardDomains[domain] + return ok +} +