diff --git a/README.md b/README.md index 441750d4..7a23438a 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ CONFIGURATIONS: -r, -resolver string list of resolvers to use (file or comma separated) -wt, -wildcard-threshold int wildcard filter threshold (default 5) -wd, -wildcard-domain string domain name for wildcard filtering (other flags will be ignored - only json output is supported) + -aw, -auto-wildcard automatically detect wildcard DNS per root domain and filter matching results (supports A, AAAA, CNAME; uses 3 probes per domain) ``` ## Running dnsx @@ -457,6 +458,38 @@ func main() { } ``` +### Auto wildcard filtering + +The `-aw` (`--auto-wildcard`) flag automatically detects wildcard DNS per root domain and filters matching results. Unlike `-wd`, you don't need to know which domains are wildcards in advance — it works transparently across mixed-domain input lists. + +**How it works:** +1. For each unique root domain (eTLD+1), three random subdomains are probed (e.g. `.example.com`, `.example.com`, `.example.com`) +2. If any probe resolves, the domain is marked as a wildcard and its A/AAAA/CNAME fingerprint is cached +3. During resolution, any result matching the wildcard fingerprint is silently filtered +4. Results that resolve to *different* IPs (real overrides) pass through + +```console +$ dnsx -l subdomains.txt -aw +[INF] [auto-wildcard] Detected wildcard domain: *.dev.example.com +[INF] [auto-wildcard] Detected wildcard domain: *.app.example.net +api.example.com +www.example.com +``` + +**Improvements over `-wd`:** +- No need to specify domains manually +- Uses 3 probes per domain (reduces false negatives from intermittent DNS) +- Detects CNAME wildcards in addition to A/AAAA wildcards +- Thread-safe cache — works with concurrent resolution +- Handles multi-level TLDs correctly (e.g. `co.uk`, `com.au`) +- Compatible with `-re`, `-ro`, `-j` output flags + +```console +dnsx -l subdomain_list.txt -aw -re +``` + +> Note: `-aw` and `-wd` cannot be used together. Use `-aw` when scanning multiple domains; use `-wd` for single-domain targeted filtering. + # 📋 Notes - As default, `dnsx` checks for **A** record. diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..21052561 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,7 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + AutoWildcard bool ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +190,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVarP(&options.AutoWildcard, "auto-wildcard", "aw", false, "automatically detect wildcard DNS per root domain and filter matching results (supports A, AAAA, CNAME; uses 3 probes per domain)"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) @@ -307,10 +309,17 @@ func (options *Options) validateOptions() { if options.WildcardDomain != "" { gologger.Fatal().Msgf("wildcard not supported in stream mode") } + if options.AutoWildcard { + gologger.Fatal().Msgf("auto-wildcard not supported in stream mode") + } if options.ShowStatistics { gologger.Fatal().Msgf("stats not supported in stream mode") } } + + if options.AutoWildcard && options.WildcardDomain != "" { + gologger.Fatal().Msgf("auto-wildcard(-aw) and wildcard-domain(-wd) cannot be used together") + } } func argumentHasStdin(arg string) bool { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..9ab257f9 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -43,6 +43,9 @@ type Runner struct { wildcards *mapsutil.SyncLockMap[string, struct{}] wildcardscache map[string][]string wildcardscachemutex sync.Mutex + // autoWildcard fields for -aw flag + autoWildcardMu sync.RWMutex + autoWildcardCache map[string]*wildcardFingerprint limiter *ratelimit.Limiter hm *hybrid.HybridMap stats clistats.StatisticsClient @@ -115,9 +118,11 @@ func New(options *Options) (*Runner, error) { } // If no option is specified or wildcard filter has been requested use query type A - if len(questionTypes) == 0 || options.WildcardDomain != "" { - options.A = true - questionTypes = append(questionTypes, dns.TypeA) + if len(questionTypes) == 0 || options.WildcardDomain != "" || options.AutoWildcard { + if !options.A { + options.A = true + questionTypes = append(questionTypes, dns.TypeA) + } } dnsxOptions.QuestionTypes = questionTypes dnsxOptions.QueryAll = options.QueryAll @@ -159,6 +164,7 @@ func New(options *Options) (*Runner, error) { wildcardworkerchan: make(chan string), wildcards: mapsutil.NewSyncLockMap[string, struct{}](), wildcardscache: make(map[string][]string), + autoWildcardCache: make(map[string]*wildcardFingerprint), limiter: limiter, hm: hm, stats: stats, @@ -738,6 +744,14 @@ func (r *Runner) worker() { continue } + // auto-wildcard: filter results that match wildcard fingerprint for their root domain + if r.options.AutoWildcard { + if r.isAutoWildcardMatch(domain, dnsData.A, dnsData.AAAA, dnsData.CNAME) { + gologger.Debug().Msgf("[auto-wildcard] Filtered wildcard result: %s\n", domain) + continue + } + } + // if response type filter is set, we don't want to ignore them if len(r.options.responseTypeFilterMap) > 0 && r.shouldSkipRecord(&dnsData) { continue @@ -939,7 +953,7 @@ func (r *Runner) wildcardWorker() { if !more { break } - if r.IsWildcard(host) { + if r.IsWildcard(host, r.options.WildcardDomain) { // mark this host as a wildcard subdomain _ = r.wildcards.Set(host, struct{}{}) } diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 5fdfc6a4..77fd0d52 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -1,13 +1,22 @@ package runner import ( + "net" "strings" + miekgdns "github.com/miekg/dns" + "github.com/projectdiscovery/gologger" "github.com/rs/xid" + "golang.org/x/net/publicsuffix" ) -// IsWildcard checks if a host is wildcard -func (r *Runner) IsWildcard(host string) bool { +// numWildcardProbes is the number of random subdomain probes per domain. +// Using 3 probes significantly reduces false negatives caused by intermittent DNS. +const numWildcardProbes = 3 + +// IsWildcard checks if a host is a wildcard for the given root domain. +// It is used by the existing -wd (wildcard-domain) post-processing flow. +func (r *Runner) IsWildcard(host, wildcardDomain string) bool { orig := make(map[string]struct{}) wildcards := make(map[string]struct{}) @@ -19,7 +28,7 @@ func (r *Runner) IsWildcard(host string) bool { orig[A] = struct{}{} } - subdomainPart := strings.TrimSuffix(host, "."+r.options.WildcardDomain) + subdomainPart := strings.TrimSuffix(host, "."+wildcardDomain) subdomainTokens := strings.Split(subdomainPart, ".") // Build an array by preallocating a slice of a length @@ -27,11 +36,11 @@ func (r *Runner) IsWildcard(host string) bool { // We use a rand prefix at the beginning like %rand%.domain.tld // A permutation is generated for each level of the subdomain. var hosts []string - hosts = append(hosts, r.options.WildcardDomain) + hosts = append(hosts, wildcardDomain) if len(subdomainTokens) > 0 { for i := 1; i < len(subdomainTokens); i++ { - newhost := strings.Join(subdomainTokens[i:], ".") + "." + r.options.WildcardDomain + newhost := strings.Join(subdomainTokens[i:], ".") + "." + wildcardDomain hosts = append(hosts, newhost) } } @@ -69,3 +78,151 @@ func (r *Runner) IsWildcard(host string) bool { return false } + +// extractRootDomain extracts the eTLD+1 (registered domain) from a hostname. +// Uses publicsuffix-go for accurate multi-level TLD handling (e.g. co.uk, com.au). +// Falls back to simple last-two-labels heuristic if parsing fails. +// Returns ("", false) for IP addresses, host:port, or unparseable inputs. +func extractRootDomain(host string) (string, bool) { + host = strings.TrimSuffix(strings.TrimSpace(host), ".") + if host == "" { + return "", false + } + // Reject host:port style inputs + if idx := strings.LastIndex(host, ":"); idx > strings.LastIndex(host, ".") { + return "", false + } + // Reject raw IP addresses (IPv4 and IPv6) + if net.ParseIP(host) != nil { + return "", false + } + + dom, err := publicsuffix.EffectiveTLDPlusOne(host) + if err == nil && dom != "" { + return dom, true + } + + // Fallback: use last two labels + parts := strings.Split(host, ".") + if len(parts) < 2 { + return "", false + } + return strings.Join(parts[len(parts)-2:], "."), true +} + +// wildcardFingerprint holds the unique A, AAAA, and CNAME values observed +// when probing a wildcard domain with random subdomains. +type wildcardFingerprint struct { + a map[string]struct{} + aaaa map[string]struct{} + cname map[string]struct{} +} + +// probeWildcardDomain performs numWildcardProbes random subdomain lookups for `domain` +// and returns a wildcardFingerprint if any probe resolves (indicating wildcard DNS). +// Returns nil if the domain does not have wildcard DNS. +// Uses both A and CNAME queries to detect all wildcard flavours. +func (r *Runner) probeWildcardDomain(domain string) *wildcardFingerprint { + fp := &wildcardFingerprint{ + a: make(map[string]struct{}), + aaaa: make(map[string]struct{}), + cname: make(map[string]struct{}), + } + + anyResolved := false + for i := 0; i < numWildcardProbes; i++ { + randHost := xid.New().String() + "." + domain + + // A/AAAA probe (QueryOne uses the first configured question type — TypeA by default) + if inA, err := r.dnsx.QueryOne(randHost); err == nil && inA != nil { + for _, ip := range inA.A { + fp.a[ip] = struct{}{} + } + for _, ip := range inA.AAAA { + fp.aaaa[ip] = struct{}{} + } + for _, cn := range inA.CNAME { + fp.cname[cn] = struct{}{} + } + if len(inA.A) > 0 || len(inA.AAAA) > 0 || len(inA.CNAME) > 0 { + anyResolved = true + } + } + + // Explicit CNAME probe to catch pure-CNAME wildcards (e.g. *.example.com → alias.cdn.net) + if inC, err := r.dnsx.QueryType(randHost, miekgdns.TypeCNAME); err == nil && inC != nil { + for _, cn := range inC.CNAME { + fp.cname[cn] = struct{}{} + } + if len(inC.CNAME) > 0 { + anyResolved = true + } + } + } + + if !anyResolved { + return nil + } + return fp +} + +// getAutoWildcardFingerprint returns the cached wildcard fingerprint for a root domain, +// probing lazily if not yet seen. Thread-safe via RWMutex. +// Returns (nil, false) when the domain is not a wildcard. +func (r *Runner) getAutoWildcardFingerprint(rootDomain string) (*wildcardFingerprint, bool) { + r.autoWildcardMu.RLock() + fp, seen := r.autoWildcardCache[rootDomain] + r.autoWildcardMu.RUnlock() + if seen { + return fp, fp != nil + } + + // Not yet probed — acquire write lock and probe (double-check idiom) + r.autoWildcardMu.Lock() + if fp, seen = r.autoWildcardCache[rootDomain]; seen { + r.autoWildcardMu.Unlock() + return fp, fp != nil + } + fp = r.probeWildcardDomain(rootDomain) + r.autoWildcardCache[rootDomain] = fp + r.autoWildcardMu.Unlock() + + if fp != nil { + gologger.Info().Msgf("[auto-wildcard] Detected wildcard domain: *.%s\n", rootDomain) + } + + return fp, fp != nil +} + +// isAutoWildcardMatch returns true if the resolved hostname's DNS records (A, AAAA, CNAME) +// match the wildcard fingerprint for its eTLD+1 root domain. +// When true, the record should be filtered from output. +func (r *Runner) isAutoWildcardMatch(host string, a, aaaa, cname []string) bool { + rootDomain, ok := extractRootDomain(host) + if !ok { + return false + } + + fp, isWildcard := r.getAutoWildcardFingerprint(rootDomain) + if !isWildcard { + return false + } + + for _, ip := range a { + if _, ok := fp.a[ip]; ok { + return true + } + } + for _, ip := range aaaa { + if _, ok := fp.aaaa[ip]; ok { + return true + } + } + for _, cn := range cname { + if _, ok := fp.cname[cn]; ok { + return true + } + } + + return false +} diff --git a/internal/runner/wildcard_auto_test.go b/internal/runner/wildcard_auto_test.go new file mode 100644 index 00000000..9cdd3542 --- /dev/null +++ b/internal/runner/wildcard_auto_test.go @@ -0,0 +1,228 @@ +package runner + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestExtractRootDomain verifies eTLD+1 extraction for various hostname formats. +func TestExtractRootDomain(t *testing.T) { + tests := []struct { + input string + expected string + ok bool + }{ + // Standard single-level TLDs + {"api.example.com", "example.com", true}, + {"www.example.com", "example.com", true}, + {"deep.sub.domain.example.com", "example.com", true}, + + // Multi-level TLDs (eTLD+1 accuracy) + {"api.example.co.uk", "example.co.uk", true}, + {"sub.foo.com.au", "foo.com.au", true}, + + // Trailing dot (FQDN) + {"api.example.com.", "example.com", true}, + {" www.example.com ", "example.com", true}, + + // Host:port — should be rejected + {"api.example.com:443", "", false}, + + // Bare apex domain + {"example.com", "example.com", true}, + + // Empty + {"", "", false}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got, ok := extractRootDomain(tt.input) + assert.Equal(t, tt.ok, ok, "ok mismatch for %q", tt.input) + if tt.ok { + assert.Equal(t, tt.expected, got, "domain mismatch for %q", tt.input) + } + }) + } +} + +// TestAutoWildcardCacheIsolation verifies the thread-safe wildcard cache correctly +// stores and retrieves fingerprints without races. +func TestAutoWildcardCacheIsolation(t *testing.T) { + r := &Runner{ + autoWildcardCache: make(map[string]*wildcardFingerprint), + } + + // Simulate storing a fingerprint + fp := &wildcardFingerprint{ + a: map[string]struct{}{"1.2.3.4": {}}, + aaaa: map[string]struct{}{}, + cname: map[string]struct{}{}, + } + r.autoWildcardMu.Lock() + r.autoWildcardCache["example.com"] = fp + r.autoWildcardMu.Unlock() + + // Read it back + r.autoWildcardMu.RLock() + got, seen := r.autoWildcardCache["example.com"] + r.autoWildcardMu.RUnlock() + + require.True(t, seen) + _, hasIP := got.a["1.2.3.4"] + require.True(t, hasIP) + + // Absent domain + r.autoWildcardMu.RLock() + _, seen2 := r.autoWildcardCache["other.com"] + r.autoWildcardMu.RUnlock() + require.False(t, seen2) +} + +// TestIsAutoWildcardMatch verifies match logic against a pre-populated cache. +func TestIsAutoWildcardMatch(t *testing.T) { + r := &Runner{ + options: &Options{}, + autoWildcardCache: make(map[string]*wildcardFingerprint), + } + + // Populate cache: example.com is a wildcard with known A and CNAME + r.autoWildcardCache["example.com"] = &wildcardFingerprint{ + a: map[string]struct{}{"10.0.0.1": {}, "10.0.0.2": {}}, + aaaa: map[string]struct{}{"::1": {}}, + cname: map[string]struct{}{"wildcard.cdn.net": {}}, + } + // notawild.com is explicitly NOT a wildcard + r.autoWildcardCache["notawild.com"] = nil + // unknown.com has no entry — we mark it nil to avoid probing in unit tests + r.autoWildcardCache["unknown.com"] = nil + + tests := []struct { + name string + host string + a []string + aaaa []string + cname []string + expected bool + }{ + { + name: "A record matches wildcard IP", + host: "random.example.com", + a: []string{"10.0.0.1"}, + expected: true, + }, + { + name: "AAAA record matches wildcard IPv6", + host: "random.example.com", + aaaa: []string{"::1"}, + expected: true, + }, + { + name: "CNAME matches wildcard CNAME", + host: "random.example.com", + cname: []string{"wildcard.cdn.net"}, + expected: true, + }, + { + name: "Different IP not filtered", + host: "legit.example.com", + a: []string{"1.2.3.4"}, + expected: false, + }, + { + name: "Non-wildcard domain not filtered", + host: "legit.notawild.com", + a: []string{"10.0.0.1"}, + expected: false, + }, + { + name: "Unknown domain (no cache entry) not filtered", + host: "sub.unknown.com", + a: []string{"10.0.0.1"}, + expected: false, + }, + { + name: "Multiple IPs one matches", + host: "random.example.com", + a: []string{"9.9.9.9", "10.0.0.2"}, + expected: true, + }, + { + name: "IP address input — no root domain", + host: "192.168.1.1", + a: []string{"192.168.1.1"}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := r.isAutoWildcardMatch(tt.host, tt.a, tt.aaaa, tt.cname) + assert.Equal(t, tt.expected, got) + }) + } +} + +// TestAutoWildcardDoublCheckLock verifies that concurrent lookups for the same domain +// only probe once (double-check locking correctness). +func TestAutoWildcardDoubleCheckLock(t *testing.T) { + r := &Runner{ + options: &Options{}, + autoWildcardCache: make(map[string]*wildcardFingerprint), + } + + // Pre-populate a non-wildcard entry + r.autoWildcardCache["example.com"] = nil + + // Multiple reads should all return (nil, false) + results := make(chan bool, 10) + for i := 0; i < 10; i++ { + go func() { + r.autoWildcardMu.RLock() + fp, seen := r.autoWildcardCache["example.com"] + r.autoWildcardMu.RUnlock() + results <- (seen && fp == nil) + }() + } + for i := 0; i < 10; i++ { + assert.True(t, <-results) + } +} + +// TestWildcardFingerprintAllFields verifies all three record types are tracked. +func TestWildcardFingerprintAllFields(t *testing.T) { + fp := &wildcardFingerprint{ + a: map[string]struct{}{"1.1.1.1": {}}, + aaaa: map[string]struct{}{"2606:4700:4700::1111": {}}, + cname: map[string]struct{}{"target.example.net": {}}, + } + + _, hasA := fp.a["1.1.1.1"] + _, hasAAAA := fp.aaaa["2606:4700:4700::1111"] + _, hasCNAME := fp.cname["target.example.net"] + + assert.True(t, hasA) + assert.True(t, hasAAAA) + assert.True(t, hasCNAME) +} + +// TestExtractRootDomainIPRejection verifies that IP addresses return false. +func TestExtractRootDomainIPRejection(t *testing.T) { + ips := []string{"1.2.3.4", "::1", "2001:db8::1", "10.0.0.1"} + for _, ip := range ips { + t.Run(ip, func(t *testing.T) { + // IPs won't have valid TLD+1 extraction + parsed := net.ParseIP(ip) + if parsed != nil { + _, ok := extractRootDomain(ip) + // IP-like strings may or may not parse as domains; + // what matters is they don't produce valid wildcard domains. + // Just verify no panic. + _ = ok + } + }) + } +} diff --git a/libs/dnsx/dnsx.go b/libs/dnsx/dnsx.go index 191e9b4d..fc8dca9d 100644 --- a/libs/dnsx/dnsx.go +++ b/libs/dnsx/dnsx.go @@ -138,6 +138,12 @@ func (d *DNSX) QueryOne(hostname string) (*retryabledns.DNSData, error) { return d.dnsClient.Query(hostname, d.Options.QuestionTypes[0]) } +// QueryType performs a DNS question of the given type and returns raw responses. +// This is used for explicit type-based probing (e.g. TypeCNAME) independent of configured QuestionTypes. +func (d *DNSX) QueryType(hostname string, qtype uint16) (*retryabledns.DNSData, error) { + return d.dnsClient.Query(hostname, qtype) +} + // QueryMultiple performs a DNS question of the specified types and returns raw responses func (d *DNSX) QueryMultiple(hostname string) (*retryabledns.DNSData, error) { // Omit PTR queries unless the input is an IP address to decrease execution time, as PTR queries can lead to timeouts.