From ce6dd52f0506fdeecc2ba0aa105c6dea70cd6ea0 Mon Sep 17 00:00:00 2001 From: Adel Assakaf Date: Mon, 16 Feb 2026 00:36:50 +0100 Subject: [PATCH 1/2] feat: add -sw (strict-wildcard) flag for automatic wildcard detection and filtering --- internal/runner/options.go | 11 ++++ internal/runner/runner.go | 94 ++++++++++++++++++++++++++++++-- internal/runner/wildcard.go | 88 ++++++++++++++++++++++++++++++ internal/runner/wildcard_test.go | 91 +++++++++++++++++++++++++++++++ 4 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 internal/runner/wildcard_test.go diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..530878d0 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,8 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + StrictWildcard bool + WildcardRetry int ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +191,8 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVarP(&options.StrictWildcard, "strict-wildcard", "sw", false, "perform strict wildcard check on all found subdomains"), + flagSet.IntVar(&options.WildcardRetry, "wildcard-retry", 5, "number of dns retries for wildcard detection (used with -sw)"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) @@ -307,10 +311,17 @@ func (options *Options) validateOptions() { if options.WildcardDomain != "" { gologger.Fatal().Msgf("wildcard not supported in stream mode") } + if options.StrictWildcard { + gologger.Fatal().Msgf("strict wildcard not supported in stream mode") + } if options.ShowStatistics { gologger.Fatal().Msgf("stats not supported in stream mode") } } + + if options.StrictWildcard && options.WildcardDomain != "" { + gologger.Fatal().Msgf("strict-wildcard(sw) and wildcard-domain(wd) can't be used at the same time") + } } func argumentHasStdin(arg string) bool { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..6de76a40 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -34,6 +34,7 @@ import ( type Runner struct { options *Options dnsx *dnsx.DNSX + wildcardDnsx *dnsx.DNSX wgoutputworker *sync.WaitGroup wgresolveworkers *sync.WaitGroup wgwildcardworker *sync.WaitGroup @@ -115,9 +116,11 @@ func New(options *Options) (*Runner, error) { } // If no option is specified or wildcard filter has been requested use query type A - if len(questionTypes) == 0 || options.WildcardDomain != "" { - options.A = true - questionTypes = append(questionTypes, dns.TypeA) + if len(questionTypes) == 0 || options.WildcardDomain != "" || options.StrictWildcard { + if !options.A { + options.A = true + questionTypes = append(questionTypes, dns.TypeA) + } } dnsxOptions.QuestionTypes = questionTypes dnsxOptions.QueryAll = options.QueryAll @@ -149,9 +152,21 @@ func New(options *Options) (*Runner, error) { options.NoColor = true } + // create a DNS client for wildcard testing with dedicated retry count + var wildcardDnsX *dnsx.DNSX + if options.StrictWildcard { + wOpts := dnsxOptions + wOpts.MaxRetries = options.WildcardRetry + wildcardDnsX, err = dnsx.New(wOpts) + if err != nil { + return nil, err + } + } + r := Runner{ options: options, dnsx: dnsX, + wildcardDnsx: wildcardDnsX, wgoutputworker: &sync.WaitGroup{}, wgresolveworkers: &sync.WaitGroup{}, wgwildcardworker: &sync.WaitGroup{}, @@ -548,9 +563,63 @@ func (r *Runner) run() error { gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) } + if r.options.StrictWildcard { + gologger.Info().Msgf("Detecting wildcard root subdomains") + + // collect all resolved hostnames from the HybridMap (only those with A records) + var allHosts []string + r.hm.Scan(func(k, v []byte) error { + if v == nil { + return nil + } + var dnsData retryabledns.DNSData + if err := json.Unmarshal(v, &dnsData); err != nil { + return nil + } + if len(dnsData.A) == 0 { + return nil + } + allHosts = append(allHosts, string(k)) + return nil + }) + + // detect wildcard roots + wildcardRoots := r.detectWildcardRoots(allHosts) + + if len(wildcardRoots) > 0 { + gologger.Info().Msgf("Found %d wildcard root%s:", len(wildcardRoots), plural(len(wildcardRoots))) + for root := range wildcardRoots { + gologger.Info().Msgf(" *.%s", root) + } + } else { + gologger.Info().Msgf("Found 0 wildcard roots") + } + + // restart output worker and filter results + r.startOutputWorker() + numFiltered := 0 + for _, host := range allHosts { + if isSubdomainOfWildcard(host, wildcardRoots) { + numFiltered++ + } else { + _ = r.lookupAndOutput(host) + } + } + close(r.outputchan) + r.wgoutputworker.Wait() + gologger.Info().Msgf("Found %d non-wildcard domains (%d wildcard subdomains filtered)", len(allHosts)-numFiltered, numFiltered) + } + return nil } +func plural(n int) string { + if n != 1 { + return "s" + } + return "" +} + func (r *Runner) lookupAndOutput(host string) error { if r.options.JSON { if data, ok := r.hm.Get(host); ok { @@ -568,6 +637,23 @@ func (r *Runner) lookupAndOutput(host string) error { } } + if r.options.Response || r.options.ResponseOnly { + if data, ok := r.hm.Get(host); ok { + var dnsData retryabledns.DNSData + if err := json.Unmarshal(data, &dnsData); err != nil { + return err + } + for _, a := range dnsData.A { + if r.options.ResponseOnly { + r.outputchan <- a + } else { + r.outputchan <- fmt.Sprintf("%s [%s] [%s]", host, r.aurora.Magenta("A"), r.aurora.Green(a)) + } + } + return nil + } + } + r.outputchan <- host return nil } @@ -731,7 +817,7 @@ func (r *Runner) worker() { } } // if wildcard filtering just store the data - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.StrictWildcard { if err := r.storeDNSData(dnsData.DNSData); err != nil { gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err) } diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 5fdfc6a4..dc5f009b 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -2,6 +2,7 @@ package runner import ( "strings" + "sync" "github.com/rs/xid" ) @@ -69,3 +70,90 @@ func (r *Runner) IsWildcard(host string) bool { return false } + +// isStrictWildcard checks if a domain is a wildcard root. +// Unlike IsWildcard (which compares IPs and requires -wd to specify the domain), +// this just checks if a random subdomain resolves at all — works for load-balanced wildcards. +func (r *Runner) isStrictWildcard(domain string) bool { + randomHost := xid.New().String() + "." + domain + resp, err := r.wildcardDnsx.QueryOne(randomHost) + if err != nil || resp == nil { + return false + } + return len(resp.A) > 0 +} + +// detectWildcardRoots finds all wildcard roots from a list of resolved hosts. +// Tests candidates top-down (shallowest first) so that finding *.example.com +// skips testing *.sub.example.com. +func (r *Runner) detectWildcardRoots(hosts []string) map[string]struct{} { + allCandidates := make(map[string]struct{}) + for _, host := range hosts { + parts := strings.Split(host, ".") + if len(parts) < 3 { + continue + } + for i := 1; i < len(parts)-1; i++ { + candidate := strings.Join(parts[i:], ".") + allCandidates[candidate] = struct{}{} + } + } + + byDepth := make(map[int][]string) + maxDepth := 0 + for candidate := range allCandidates { + depth := strings.Count(candidate, ".") + 1 + byDepth[depth] = append(byDepth[depth], candidate) + if depth > maxDepth { + maxDepth = depth + } + } + + roots := make(map[string]struct{}) + + for depth := 2; depth <= maxDepth; depth++ { + candidates := byDepth[depth] + if len(candidates) == 0 { + continue + } + + var toTest []string + for _, c := range candidates { + if !isSubdomainOfWildcard(c, roots) { + toTest = append(toTest, c) + } + } + if len(toTest) == 0 { + continue + } + + var wg sync.WaitGroup + sem := make(chan struct{}, r.options.Threads) + var mu sync.Mutex + for _, domain := range toTest { + wg.Add(1) + sem <- struct{}{} + go func(d string) { + defer wg.Done() + defer func() { <-sem }() + if r.isStrictWildcard(d) { + mu.Lock() + roots[d] = struct{}{} + mu.Unlock() + } + }(domain) + } + wg.Wait() + } + + return roots +} + +func isSubdomainOfWildcard(host string, roots map[string]struct{}) bool { + for root := range roots { + if host != root && strings.HasSuffix(host, "."+root) { + return true + } + } + return false +} diff --git a/internal/runner/wildcard_test.go b/internal/runner/wildcard_test.go new file mode 100644 index 00000000..976c3847 --- /dev/null +++ b/internal/runner/wildcard_test.go @@ -0,0 +1,91 @@ +package runner + +import ( + "testing" + + "github.com/projectdiscovery/dnsx/libs/dnsx" + "github.com/stretchr/testify/require" +) + +// newTestRunner creates a minimal Runner with a real DNS client for wildcard testing. +func newTestRunner(t *testing.T) *Runner { + t.Helper() + options := dnsx.DefaultOptions + options.QuestionTypes = []uint16{1} // TypeA + options.MaxRetries = 3 + dnsX, err := dnsx.New(options) + require.NoError(t, err) + return &Runner{ + dnsx: dnsX, + wildcardDnsx: dnsX, + options: &Options{ + Threads: 100, + }, + } +} + +func TestIsSubdomainOfWildcard(t *testing.T) { + roots := map[string]struct{}{ + "dev.projectdiscovery.io": {}, + } + + tests := []struct { + host string + expected bool + }{ + {"bob.dev.projectdiscovery.io", true}, + {"a.b.dev.projectdiscovery.io", true}, + {"dev.projectdiscovery.io", false}, // root itself is not filtered + {"projectdiscovery.io", false}, // parent domain + {"notprojectdiscovery.io", false}, // different domain + {"dev.projectdiscovery.io.evil.com", false}, // suffix trick + } + + for _, tt := range tests { + t.Run(tt.host, func(t *testing.T) { + got := isSubdomainOfWildcard(tt.host, roots) + require.Equal(t, tt.expected, got) + }) + } +} + +func TestIsStrictWildcard(t *testing.T) { + r := newTestRunner(t) + + tests := []struct { + domain string + expected bool + }{ + {"dev.projectdiscovery.io", true}, // wildcard + {"projectdiscovery.io", false}, // not wildcard + } + + for _, tt := range tests { + t.Run(tt.domain, func(t *testing.T) { + got := r.isStrictWildcard(tt.domain) + require.Equal(t, tt.expected, got) + }) + } +} + +func TestDetectWildcardRoots(t *testing.T) { + r := newTestRunner(t) + + hosts := []string{ + "bob.dev.projectdiscovery.io", + "alice.dev.projectdiscovery.io", + "blog.projectdiscovery.io", + } + + roots := r.detectWildcardRoots(hosts) + + _, hasDev := roots["dev.projectdiscovery.io"] + require.True(t, hasDev, "dev.projectdiscovery.io should be detected as wildcard root") + + _, hasPD := roots["projectdiscovery.io"] + require.False(t, hasPD, "projectdiscovery.io should not be detected as wildcard root") + + // subdomains should be filtered, non-wildcard siblings should not + require.True(t, isSubdomainOfWildcard("bob.dev.projectdiscovery.io", roots)) + require.False(t, isSubdomainOfWildcard("blog.projectdiscovery.io", roots)) +} From 804a7187c4a4642c0a78867e3db8c494eaee311b Mon Sep 17 00:00:00 2001 From: Adel Assakaf Date: Wed, 18 Feb 2026 23:42:12 +0100 Subject: [PATCH 2/2] docs: add strict wildcard flags in README.md --- README.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 441750d4..483618c8 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,8 @@ CONFIGURATIONS: -r, -resolver string list of resolvers to use (file or comma separated) -wt, -wildcard-threshold int wildcard filter threshold (default 5) -wd, -wildcard-domain string domain name for wildcard filtering (other flags will be ignored - only json output is supported) + -sw, -strict-wildcard perform strict wildcard check on all found subdomains + -wildcard-retry int number of dns retries for wildcard detection (used with -sw) (default 5) ``` ## Running dnsx @@ -405,6 +407,35 @@ A special feature of `dnsx` is its ability to handle **multi-level DNS based wil dnsx -l subdomain_list.txt -wd airbnb.com -o output.txt ``` +### Strict wildcard filtering + +The `-sw` flag provides automatic wildcard detection and filtering without requiring you to specify which domains are wildcards. It works by querying random subdomains for each parent domain — if a random subdomain resolves, the parent is a wildcard root and all its subdomains are filtered from the output. + +```console +$ dnsx -l domains.txt -sw + +[INF] Detecting wildcard root subdomains +[INF] Found 8 wildcard roots: +[INF] *.netlify.com +[INF] *.dev.projectdiscovery.io +[INF] *.netlify.app +[INF] *.ngrok.io +[INF] *.wordpress.com +[INF] *.vercel.app +[INF] *.github.io +[INF] *.herokuapp.com +cloud.projectdiscovery.io +docs.projectdiscovery.io +www.example.com +[INF] Found 3 non-wildcard domains (11 wildcard subdomains filtered) +``` + +The wildcard retry count can be configured with `-wildcard-retry` (default 5): + +```console +dnsx -l subdomain_list.txt -sw -wildcard-retry 10 +``` + --------- ### Dnsx as a library @@ -462,8 +493,8 @@ func main() { - As default, `dnsx` checks for **A** record. - As default `dnsx` uses Google, Cloudflare, Quad9 [resolver](https://github.com/projectdiscovery/dnsx/blob/43af78839e237ea8cbafe571df1ab0d6cbe7f445/libs/dnsx/dnsx.go#L31). - Custom resolver list can be loaded using the `r` flag. -- Domain name (`wd`) input is mandatory for wildcard elimination. -- DNS record flag can not be used when using wildcard filtering. +- Domain name (`wd`) input is required for wildcard filtering with `-wd`. The `-sw` flag provides automatic wildcard detection without needing `-wd`. +- DNS record flags cannot be used when using wildcard filtering (`-wd` or `-sw`). - DNS resolution (`l`) and DNS brute-forcing (`w`) can't be used together. - VPN operators tend to filter high DNS/UDP traffic, therefore the tool might experience packets loss (eg. [Mullvad VPN](https://github.com/projectdiscovery/dnsx/issues/221)). Check [this potential solution](./MULLVAD.md).