feat: add auto wildcard detection and filtering

internal/runner/options.go

@@ -59,6 +59,7 @@ type Options struct {
 	TraceMaxRecursion     int
 	WildcardThreshold     int
 	WildcardDomain        string
+	AutoWildcard          bool
 	ShowStatistics        bool
 	rcodes                map[int]struct{}
 	RCode                 string
@@ -189,6 +190,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"),
 		flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"),
 		flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"),
+		flagSet.BoolVarP(&options.AutoWildcard, "auto-wildcard", "aw", false, "automatically detect and filter wildcard DNS records across multiple domains"),
 		flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"),
 	)
 
@@ -271,6 +273,10 @@ func (options *Options) validateOptions() {
 		gologger.Fatal().Msgf("retries must be at least 1")
 	}
 
+	if options.WildcardDomain != "" && options.AutoWildcard {
+		gologger.Fatal().Msgf("wildcard-domain and auto-wildcard can't be used at the same time")
+	}
+
 	wordListPresent := options.WordList != ""
 	domainsPresent := options.Domains != ""
 	hostsPresent := options.Hosts != ""
@@ -307,6 +313,9 @@ func (options *Options) validateOptions() {
 		if options.WildcardDomain != "" {
 			gologger.Fatal().Msgf("wildcard not supported in stream mode")
 		}
+		if options.AutoWildcard {
+			gologger.Fatal().Msgf("auto-wildcard not supported in stream mode")
+		}
 		if options.ShowStatistics {
 			gologger.Fatal().Msgf("stats not supported in stream mode")
 		}

internal/runner/runner.go

@@ -43,6 +43,7 @@ type Runner struct {
 	wildcards           *mapsutil.SyncLockMap[string, struct{}]
 	wildcardscache      map[string][]string
 	wildcardscachemutex sync.Mutex
+	autoWildcardDomains *mapsutil.SyncLockMap[string, bool]
 	limiter             *ratelimit.Limiter
 	hm                  *hybrid.HybridMap
 	stats               clistats.StatisticsClient
@@ -159,6 +160,7 @@ func New(options *Options) (*Runner, error) {
 		wildcardworkerchan: make(chan string),
 		wildcards:          mapsutil.NewSyncLockMap[string, struct{}](),
 		wildcardscache:     make(map[string][]string),
+		autoWildcardDomains: mapsutil.NewSyncLockMap[string, bool](),
 		limiter:            limiter,
 		hm:                 hm,
 		stats:              stats,
@@ -548,6 +550,11 @@ func (r *Runner) run() error {
 		gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains)
 	}
 
+	if r.options.AutoWildcard {
+		gologger.Print().Msgf("Starting auto wildcard detection and filtering\n")
+		r.runAutoWildcardFiltering()
+	}
+
 	return nil
 }
 
@@ -572,6 +579,91 @@ func (r *Runner) lookupAndOutput(host string) error {
 	return nil
 }
 
+func (r *Runner) runAutoWildcardFiltering() {
+	ipDomain := make(map[string]map[string]struct{})
+	domainHosts := make(map[string][]string)
+	listIPs := []string{}
+
+	// Scan all stored DNS data and group by IP and domain
+	r.hm.Scan(func(k, v []byte) error {
+		var dnsdata retryabledns.DNSData
+		if err := json.Unmarshal(v, &dnsdata); err != nil {
+			return nil
+		}
+
+		host := string(k)
+		baseDomain := r.GetBaseDomain(host)
+
+		// Track hosts per base domain
+		if _, ok := domainHosts[baseDomain]; !ok {
+			domainHosts[baseDomain] = make([]string, 0)
+		}
+		domainHosts[baseDomain] = append(domainHosts[baseDomain], host)
+
+		for _, a := range dnsdata.A {
+			_, ok := ipDomain[a]
+			if !ok {
+				ipDomain[a] = make(map[string]struct{})
+				listIPs = append(listIPs, a)
+			}
+			ipDomain[a][host] = struct{}{}
+		}
+
+		return nil
+	})
+
+	gologger.Debug().Msgf("Found %d unique IPs across %d domains\n", len(listIPs), len(domainHosts))
+
+	// Start wildcard workers for auto detection
+	numThreads := r.options.Threads
+	if numThreads > len(listIPs) {
+		numThreads = len(listIPs)
+	}
+	for i := 0; i < numThreads; i++ {
+		r.wgwildcardworker.Add(1)
+		go r.wildcardWorkerAuto()
+	}
+
+	// Send hosts to wildcard worker for checking
+	seen := make(map[string]struct{})
+	for _, hosts := range domainHosts {
+		for _, host := range hosts {
+			if _, ok := seen[host]; !ok {
+				seen[host] = struct{}{}
+				r.wildcardworkerchan <- host
+			}
+		}
+	}
+	close(r.wildcardworkerchan)
+	r.wgwildcardworker.Wait()
+
+	// Restart output worker
+	r.startOutputWorker()
+
+	// Output non-wildcard results
+	seen = make(map[string]struct{})
+	seenRemovedSubdomains := make(map[string]struct{})
+	numRemovedSubdomains := 0
+
+	for _, hosts := range domainHosts {
+		for _, host := range hosts {
+			if _, ok := seen[host]; !ok {
+				seen[host] = struct{}{}
+				if !r.wildcards.Has(host) {
+					_ = r.lookupAndOutput(host)
+				} else {
+					numRemovedSubdomains++
+					seenRemovedSubdomains[host] = struct{}{}
+				}
+			}
+		}
+	}
+
+	close(r.outputchan)
+	r.wgoutputworker.Wait()
+	gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains)
+}
+
 func (r *Runner) runStream() error {
 	r.startWorkers()
 
@@ -731,7 +823,7 @@ func (r *Runner) worker() {
 			}
 		}
 		// if wildcard filtering just store the data
-		if r.options.WildcardDomain != "" {
+		if r.options.WildcardDomain != "" || r.options.AutoWildcard {
 			if err := r.storeDNSData(dnsData.DNSData); err != nil {
 				gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err)
 			}
@@ -945,3 +1037,18 @@ func (r *Runner) wildcardWorker() {
 		}
 	}
 }
+
+func (r *Runner) wildcardWorkerAuto() {
+	defer r.wgwildcardworker.Done()
+
+	for {
+		host, more := <-r.wildcardworkerchan
+		if !more {
+			break
+		}
+		if r.IsWildcardAuto(host) {
+			// mark this host as a wildcard subdomain
+			_ = r.wildcards.Set(host, struct{}{})
+		}
+	}
+}

internal/runner/wildcard.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/rs/xid"
+	"github.com/weppos/publicsuffix-go/publicsuffix"
 )
 
 // IsWildcard checks if a host is wildcard
@@ -69,3 +70,120 @@ func (r *Runner) IsWildcard(host string) bool {
 
 	return false
 }
+
+// IsWildcardDomain checks if a domain has wildcard DNS records by querying a random subdomain
+func (r *Runner) IsWildcardDomain(domain string) bool {
+	// Check cache first
+	if hasWildcard, ok := r.autoWildcardDomains.Get(domain); ok {
+		return hasWildcard
+	}
+
+	// Generate a random subdomain to test for wildcards
+	randomSubdomain := xid.New().String() + "." + domain
+
+	in, err := r.dnsx.QueryOne(randomSubdomain)
+	if err != nil || in == nil {
+		// Random subdomain doesn't resolve, not a wildcard domain
+		_ = r.autoWildcardDomains.Set(domain, false)
+		return false
+	}
+
+	// Random subdomain resolved, this is a wildcard domain
+	_ = r.autoWildcardDomains.Set(domain, true)
+	return true
+}
+
+// GetBaseDomain extracts the base domain from a subdomain
+func (r *Runner) GetBaseDomain(host string) string {
+	// Trim trailing dot if present
+	host = strings.TrimSuffix(host, ".")
+
+	// Use publicsuffix library for accurate base domain extraction
+	if domain, err := publicsuffix.Parse(host); err == nil {
+		// domain has fields: TLD, SLD, TRD
+		// For co.uk, TLD is "co.uk", SLD is "example", TRD is "sub"
+		// We want eTLD+1: SLD + "." + TLD
+		if domain.TLD != "" && domain.SLD != "" {
+			return domain.SLD + "." + domain.TLD
+		}
+	}
+
+	// Fallback to original logic if publicsuffix parsing fails
+	parts := strings.Split(host, ".")
+	if len(parts) <= 2 {
+		return host
+	}
+	// Return the last two parts as the base domain
+	return strings.Join(parts[len(parts)-2:], ".")
+}
+
+// IsWildcardAuto checks if a host is wildcard using auto detection
+func (r *Runner) IsWildcardAuto(host string) bool {
+	baseDomain := r.GetBaseDomain(host)
+
+	// First check if this domain has wildcards
+	if !r.IsWildcardDomain(baseDomain) {
+		return false
+	}
+
+	// Domain has wildcards, now check if this specific host is a wildcard
+	orig := make(map[string]struct{})
+	wildcards := make(map[string]struct{})
+
+	in, err := r.dnsx.QueryOne(host)
+	if err != nil || in == nil {
+		return false
+	}
+	for _, A := range in.A {
+		orig[A] = struct{}{}
+	}
+
+	// Get wildcard IPs by querying random subdomains at each level
+	subdomainPart := strings.TrimSuffix(host, "."+baseDomain)
+	subdomainTokens := strings.Split(subdomainPart, ".")
+
+	var hosts []string
+	hosts = append(hosts, baseDomain)
+
+	if len(subdomainTokens) > 0 && subdomainTokens[0] != "" {
+		for i := 1; i <= len(subdomainTokens); i++ {
+			newhost := strings.Join(subdomainTokens[i:], ".") + "." + baseDomain
+			if newhost != "" {
+				hosts = append(hosts, newhost)
+			}
+		}
+	}
+
+	// Iterate over all the hosts generated for rand.
+	for _, h := range hosts {
+		r.wildcardscachemutex.Lock()
+		listip, ok := r.wildcardscache[h]
+		r.wildcardscachemutex.Unlock()
+		if !ok {
+			in, err := r.dnsx.QueryOne(xid.New().String() + "." + h)
+			if err != nil || in == nil {
+				continue
+			}
+			listip = in.A
+			r.wildcardscachemutex.Lock()
+			r.wildcardscache[h] = in.A
+			r.wildcardscachemutex.Unlock()
+		}
+
+		// Get all the records and add them to the wildcard map
+		for _, A := range listip {
+			if _, ok := wildcards[A]; !ok {
+				wildcards[A] = struct{}{}
+			}
+		}
+	}
+
+	// check if original ip are among wildcards
+	for a := range orig {
+		if _, ok := wildcards[a]; ok {
+			return true
+		}
+	}
+
+	return false
+}