From 7ef3fa1694944d281c0a4c932ad85664dd076f19 Mon Sep 17 00:00:00 2001 From: Pitrat-wav Date: Tue, 24 Feb 2026 04:44:27 +0500 Subject: [PATCH 1/2] feat: add auto wildcard detection and filtering - Add --auto-wildcard (-aw) flag to automatically detect and filter wildcard DNS records - Implement IsWildcardDomain() to detect wildcard domains by querying random subdomains - Implement IsWildcardAuto() to check if a specific host is a wildcard - Add GetBaseDomain() helper to extract base domain from subdomain - Add runAutoWildcardFiltering() to process all domains with auto wildcard detection - Cache wildcard detection results per domain for efficiency - Add validation to prevent using --wildcard-domain and --auto-wildcard together - Auto wildcard mode not supported in stream mode Fixes: #924 Co-authored-by: Qwen-Coder --- internal/runner/options.go | 9 +++ internal/runner/runner.go | 109 +++++++++++++++++++++++++++++++++++- internal/runner/wildcard.go | 103 ++++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 1 deletion(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..6b41083c 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,7 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + AutoWildcard bool ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +190,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVarP(&options.AutoWildcard, "auto-wildcard", "aw", false, "automatically detect and filter wildcard DNS records across multiple domains"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) @@ -271,6 +273,10 @@ func (options *Options) validateOptions() { gologger.Fatal().Msgf("retries must be at least 1") } + if options.WildcardDomain != "" && options.AutoWildcard { + gologger.Fatal().Msgf("wildcard-domain and auto-wildcard can't be used at the same time") + } + wordListPresent := options.WordList != "" domainsPresent := options.Domains != "" hostsPresent := options.Hosts != "" @@ -307,6 +313,9 @@ func (options *Options) validateOptions() { if options.WildcardDomain != "" { gologger.Fatal().Msgf("wildcard not supported in stream mode") } + if options.AutoWildcard { + gologger.Fatal().Msgf("auto-wildcard not supported in stream mode") + } if options.ShowStatistics { gologger.Fatal().Msgf("stats not supported in stream mode") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..05511915 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -43,6 +43,7 @@ type Runner struct { wildcards *mapsutil.SyncLockMap[string, struct{}] wildcardscache map[string][]string wildcardscachemutex sync.Mutex + autoWildcardDomains *mapsutil.SyncLockMap[string, bool] limiter *ratelimit.Limiter hm *hybrid.HybridMap stats clistats.StatisticsClient @@ -159,6 +160,7 @@ func New(options *Options) (*Runner, error) { wildcardworkerchan: make(chan string), wildcards: mapsutil.NewSyncLockMap[string, struct{}](), wildcardscache: make(map[string][]string), + autoWildcardDomains: mapsutil.NewSyncLockMap[string, bool](), limiter: limiter, hm: hm, stats: stats, @@ -548,6 +550,11 @@ func (r *Runner) run() error { gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) } + if r.options.AutoWildcard { + gologger.Print().Msgf("Starting auto wildcard detection and filtering\n") + r.runAutoWildcardFiltering() + } + return nil } @@ -572,6 +579,91 @@ func (r *Runner) lookupAndOutput(host string) error { return nil } +func (r *Runner) runAutoWildcardFiltering() { + ipDomain := make(map[string]map[string]struct{}) + domainHosts := make(map[string][]string) + listIPs := []string{} + + // Scan all stored DNS data and group by IP and domain + r.hm.Scan(func(k, v []byte) error { + var dnsdata retryabledns.DNSData + if err := json.Unmarshal(v, &dnsdata); err != nil { + return nil + } + + host := string(k) + baseDomain := r.GetBaseDomain(host) + + // Track hosts per base domain + if _, ok := domainHosts[baseDomain]; !ok { + domainHosts[baseDomain] = make([]string, 0) + } + domainHosts[baseDomain] = append(domainHosts[baseDomain], host) + + for _, a := range dnsdata.A { + _, ok := ipDomain[a] + if !ok { + ipDomain[a] = make(map[string]struct{}) + listIPs = append(listIPs, a) + } + ipDomain[a][host] = struct{}{} + } + + return nil + }) + + gologger.Debug().Msgf("Found %d unique IPs across %d domains\n", len(listIPs), len(domainHosts)) + + // Start wildcard workers for auto detection + numThreads := r.options.Threads + if numThreads > len(listIPs) { + numThreads = len(listIPs) + } + for i := 0; i < numThreads; i++ { + r.wgwildcardworker.Add(1) + go r.wildcardWorkerAuto() + } + + // Send hosts to wildcard worker for checking + seen := make(map[string]struct{}) + for _, hosts := range domainHosts { + for _, host := range hosts { + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + r.wildcardworkerchan <- host + } + } + } + close(r.wildcardworkerchan) + r.wgwildcardworker.Wait() + + // Restart output worker + r.startOutputWorker() + + // Output non-wildcard results + seen = make(map[string]struct{}) + seenRemovedSubdomains := make(map[string]struct{}) + numRemovedSubdomains := 0 + + for _, hosts := range domainHosts { + for _, host := range hosts { + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + if !r.wildcards.Has(host) { + _ = r.lookupAndOutput(host) + } else { + numRemovedSubdomains++ + seenRemovedSubdomains[host] = struct{}{} + } + } + } + } + + close(r.outputchan) + r.wgoutputworker.Wait() + gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) +} + func (r *Runner) runStream() error { r.startWorkers() @@ -731,7 +823,7 @@ func (r *Runner) worker() { } } // if wildcard filtering just store the data - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { if err := r.storeDNSData(dnsData.DNSData); err != nil { gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err) } @@ -945,3 +1037,18 @@ func (r *Runner) wildcardWorker() { } } } + +func (r *Runner) wildcardWorkerAuto() { + defer r.wgwildcardworker.Done() + + for { + host, more := <-r.wildcardworkerchan + if !more { + break + } + if r.IsWildcardAuto(host) { + // mark this host as a wildcard subdomain + _ = r.wildcards.Set(host, struct{}{}) + } + } +} diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 5fdfc6a4..28b6718c 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -69,3 +69,106 @@ func (r *Runner) IsWildcard(host string) bool { return false } + +// IsWildcardDomain checks if a domain has wildcard DNS records by querying a random subdomain +func (r *Runner) IsWildcardDomain(domain string) bool { + // Check cache first + if hasWildcard, ok := r.autoWildcardDomains.Get(domain); ok { + return hasWildcard + } + + // Generate a random subdomain to test for wildcards + randomSubdomain := xid.New().String() + "." + domain + + in, err := r.dnsx.QueryOne(randomSubdomain) + if err != nil || in == nil { + // Random subdomain doesn't resolve, not a wildcard domain + _ = r.autoWildcardDomains.Set(domain, false) + return false + } + + // Random subdomain resolved, this is a wildcard domain + _ = r.autoWildcardDomains.Set(domain, true) + return true +} + +// GetBaseDomain extracts the base domain from a subdomain +func (r *Runner) GetBaseDomain(host string) string { + parts := strings.Split(host, ".") + if len(parts) <= 2 { + return host + } + // Return the last two parts as the base domain + return strings.Join(parts[len(parts)-2:], ".") +} + +// IsWildcardAuto checks if a host is wildcard using auto detection +func (r *Runner) IsWildcardAuto(host string) bool { + baseDomain := r.GetBaseDomain(host) + + // First check if this domain has wildcards + if !r.IsWildcardDomain(baseDomain) { + return false + } + + // Domain has wildcards, now check if this specific host is a wildcard + orig := make(map[string]struct{}) + wildcards := make(map[string]struct{}) + + in, err := r.dnsx.QueryOne(host) + if err != nil || in == nil { + return false + } + for _, A := range in.A { + orig[A] = struct{}{} + } + + // Get wildcard IPs by querying random subdomains at each level + subdomainPart := strings.TrimSuffix(host, "."+baseDomain) + subdomainTokens := strings.Split(subdomainPart, ".") + + var hosts []string + hosts = append(hosts, baseDomain) + + if len(subdomainTokens) > 0 && subdomainTokens[0] != "" { + for i := 1; i <= len(subdomainTokens); i++ { + newhost := strings.Join(subdomainTokens[i:], ".") + "." + baseDomain + if newhost != "" { + hosts = append(hosts, newhost) + } + } + } + + // Iterate over all the hosts generated for rand. + for _, h := range hosts { + r.wildcardscachemutex.Lock() + listip, ok := r.wildcardscache[h] + r.wildcardscachemutex.Unlock() + if !ok { + in, err := r.dnsx.QueryOne(xid.New().String() + "." + h) + if err != nil || in == nil { + continue + } + listip = in.A + r.wildcardscachemutex.Lock() + r.wildcardscache[h] = in.A + r.wildcardscachemutex.Unlock() + } + + // Get all the records and add them to the wildcard map + for _, A := range listip { + if _, ok := wildcards[A]; !ok { + wildcards[A] = struct{}{} + } + } + } + + // check if original ip are among wildcards + for a := range orig { + if _, ok := wildcards[a]; ok { + return true + } + } + + return false +} From bfe7a9acaa998aaa220b7c76cab3928bf62629a1 Mon Sep 17 00:00:00 2001 From: Pitrat-wav Date: Tue, 24 Feb 2026 05:29:45 +0500 Subject: [PATCH 2/2] fix(wildcard): use publicsuffix library for accurate base domain extraction Addresses Neo Security Audit feedback about GetBaseDomain() function. The previous naive approach using last 2 domain parts failed for multi-level TLDs like .co.uk. Now uses github.com/weppos/publicsuffix-go for correct eTLD+1 extraction. --- internal/runner/wildcard.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 28b6718c..1d4e452b 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/rs/xid" + "github.com/weppos/publicsuffix-go/publicsuffix" ) // IsWildcard checks if a host is wildcard @@ -94,6 +95,20 @@ func (r *Runner) IsWildcardDomain(domain string) bool { // GetBaseDomain extracts the base domain from a subdomain func (r *Runner) GetBaseDomain(host string) string { + // Trim trailing dot if present + host = strings.TrimSuffix(host, ".") + + // Use publicsuffix library for accurate base domain extraction + if domain, err := publicsuffix.Parse(host); err == nil { + // domain has fields: TLD, SLD, TRD + // For co.uk, TLD is "co.uk", SLD is "example", TRD is "sub" + // We want eTLD+1: SLD + "." + TLD + if domain.TLD != "" && domain.SLD != "" { + return domain.SLD + "." + domain.TLD + } + } + + // Fallback to original logic if publicsuffix parsing fails parts := strings.Split(host, ".") if len(parts) <= 2 { return host