From 64b5c10ee02796815478ca36e1c9024d434fb23c Mon Sep 17 00:00:00 2001 From: Jujubee-LLM Date: Tue, 24 Feb 2026 19:40:51 +0800 Subject: [PATCH 1/6] feat(wildcard): add --auto-wildcard multi-domain detection mode --- internal/runner/options.go | 4 ++- internal/runner/runner.go | 54 ++++++++++++++++++++++++++++++------- internal/runner/wildcard.go | 10 +++---- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..1df917ea 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,7 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + AutoWildcard bool ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +190,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVar(&options.AutoWildcard, "auto-wildcard", false, "auto-detect wildcard root domain per host (multi-domain mode)"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) @@ -304,7 +306,7 @@ func (options *Options) validateOptions() { if options.Resume { gologger.Fatal().Msgf("resume not supported in stream mode") } - if options.WildcardDomain != "" { + if options.WildcardDomain != "" || options.AutoWildcard { gologger.Fatal().Msgf("wildcard not supported in stream mode") } if options.ShowStatistics { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..44f2685c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -24,6 +24,7 @@ import ( "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/retryabledns" + "golang.org/x/net/publicsuffix" fileutil "github.com/projectdiscovery/utils/file" iputil "github.com/projectdiscovery/utils/ip" mapsutil "github.com/projectdiscovery/utils/maps" @@ -31,6 +32,11 @@ import ( ) // Runner is a client for running the enumeration process. +type wildcardTask struct { + host string + domain string +} + type Runner struct { options *Options dnsx *dnsx.DNSX @@ -39,7 +45,7 @@ type Runner struct { wgwildcardworker *sync.WaitGroup workerchan chan string outputchan chan string - wildcardworkerchan chan string + wildcardworkerchan chan wildcardTask wildcards *mapsutil.SyncLockMap[string, struct{}] wildcardscache map[string][]string wildcardscachemutex sync.Mutex @@ -115,7 +121,7 @@ func New(options *Options) (*Runner, error) { } // If no option is specified or wildcard filter has been requested use query type A - if len(questionTypes) == 0 || options.WildcardDomain != "" { + if len(questionTypes) == 0 || options.WildcardDomain != "" || options.AutoWildcard { options.A = true questionTypes = append(questionTypes, dns.TypeA) } @@ -156,7 +162,7 @@ func New(options *Options) (*Runner, error) { wgresolveworkers: &sync.WaitGroup{}, wgwildcardworker: &sync.WaitGroup{}, workerchan: make(chan string), - wildcardworkerchan: make(chan string), + wildcardworkerchan: make(chan wildcardTask), wildcards: mapsutil.NewSyncLockMap[string, struct{}](), wildcardscache: make(map[string][]string), limiter: limiter, @@ -467,7 +473,7 @@ func (r *Runner) run() error { close(r.outputchan) r.wgoutputworker.Wait() - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { gologger.Print().Msgf("Starting to filter wildcard subdomains\n") ipDomain := make(map[string]map[string]struct{}) listIPs := []string{} @@ -503,13 +509,19 @@ func (r *Runner) run() error { } seen := make(map[string]struct{}) + hostToWildcardDomain := make(map[string]string) for _, a := range listIPs { hosts := ipDomain[a] if len(hosts) >= r.options.WildcardThreshold { for host := range hosts { + wildcardDomain, ok := r.getWildcardDomainForHost(host) + if !ok { + continue + } + hostToWildcardDomain[host] = wildcardDomain if _, ok := seen[host]; !ok { seen[host] = struct{}{} - r.wildcardworkerchan <- host + r.wildcardworkerchan <- wildcardTask{host: host, domain: wildcardDomain} } } } @@ -524,7 +536,8 @@ func (r *Runner) run() error { numRemovedSubdomains := 0 for _, A := range listIPs { for host := range ipDomain[A] { - if host == r.options.WildcardDomain { + wildcardDomain, hasDomain := hostToWildcardDomain[host] + if hasDomain && host == wildcardDomain { if _, ok := seen[host]; !ok { seen[host] = struct{}{} _ = r.lookupAndOutput(host) @@ -551,6 +564,27 @@ func (r *Runner) run() error { return nil } +func (r *Runner) getWildcardDomainForHost(host string) (string, bool) { + if r.options.WildcardDomain != "" { + return r.options.WildcardDomain, true + } + if !r.options.AutoWildcard { + return "", false + } + h := strings.TrimSpace(host) + if h == "" { + return "", false + } + if strings.Contains(h, ":") { + return "", false + } + domain, err := publicsuffix.EffectiveTLDPlusOne(h) + if err != nil || domain == "" { + return "", false + } + return domain, true +} + func (r *Runner) lookupAndOutput(host string) error { if r.options.JSON { if data, ok := r.hm.Get(host); ok { @@ -731,7 +765,7 @@ func (r *Runner) worker() { } } // if wildcard filtering just store the data - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { if err := r.storeDNSData(dnsData.DNSData); err != nil { gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err) } @@ -935,13 +969,13 @@ func (r *Runner) wildcardWorker() { defer r.wgwildcardworker.Done() for { - host, more := <-r.wildcardworkerchan + task, more := <-r.wildcardworkerchan if !more { break } - if r.IsWildcard(host) { + if r.IsWildcard(task.host, task.domain) { // mark this host as a wildcard subdomain - _ = r.wildcards.Set(host, struct{}{}) + _ = r.wildcards.Set(task.host, struct{}{}) } } } diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 5fdfc6a4..52dae69e 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -6,8 +6,8 @@ import ( "github.com/rs/xid" ) -// IsWildcard checks if a host is wildcard -func (r *Runner) IsWildcard(host string) bool { +// IsWildcard checks if a host is wildcard for a specific root domain +func (r *Runner) IsWildcard(host, wildcardDomain string) bool { orig := make(map[string]struct{}) wildcards := make(map[string]struct{}) @@ -19,7 +19,7 @@ func (r *Runner) IsWildcard(host string) bool { orig[A] = struct{}{} } - subdomainPart := strings.TrimSuffix(host, "."+r.options.WildcardDomain) + subdomainPart := strings.TrimSuffix(host, "."+wildcardDomain) subdomainTokens := strings.Split(subdomainPart, ".") // Build an array by preallocating a slice of a length @@ -27,11 +27,11 @@ func (r *Runner) IsWildcard(host string) bool { // We use a rand prefix at the beginning like %rand%.domain.tld // A permutation is generated for each level of the subdomain. var hosts []string - hosts = append(hosts, r.options.WildcardDomain) + hosts = append(hosts, wildcardDomain) if len(subdomainTokens) > 0 { for i := 1; i < len(subdomainTokens); i++ { - newhost := strings.Join(subdomainTokens[i:], ".") + "." + r.options.WildcardDomain + newhost := strings.Join(subdomainTokens[i:], ".") + "." + wildcardDomain hosts = append(hosts, newhost) } } From 71e3573ecc896d96c24ce72b3ee414c5a6ef8f65 Mon Sep 17 00:00:00 2001 From: Jujubee-LLM Date: Tue, 24 Feb 2026 20:06:25 +0800 Subject: [PATCH 2/6] chore(runner): format imports for auto-wildcard patch --- internal/runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 44f2685c..86988a6e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -24,11 +24,11 @@ import ( "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/retryabledns" - "golang.org/x/net/publicsuffix" fileutil "github.com/projectdiscovery/utils/file" iputil "github.com/projectdiscovery/utils/ip" mapsutil "github.com/projectdiscovery/utils/maps" sliceutil "github.com/projectdiscovery/utils/slice" + "golang.org/x/net/publicsuffix" ) // Runner is a client for running the enumeration process. From 05b82ba4a16c50246aafa78c7234b1a8b4bf0355 Mon Sep 17 00:00:00 2001 From: Jujubee-LLM Date: Tue, 24 Feb 2026 20:13:07 +0800 Subject: [PATCH 3/6] test(runner): add coverage for auto-wildcard domain derivation --- internal/runner/runner_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 179c639a..2469a90d 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -141,6 +141,34 @@ func TestRunner_fileInput_prepareInput(t *testing.T) { require.ElementsMatch(t, expected, got, "could not match expected output") } +func TestRunner_getWildcardDomainForHost(t *testing.T) { + t.Run("uses explicit wildcard domain when provided", func(t *testing.T) { + r := Runner{options: &Options{WildcardDomain: "example.com"}} + domain, ok := r.getWildcardDomainForHost("api.foo.bar") + require.True(t, ok) + require.Equal(t, "example.com", domain) + }) + + t.Run("derives effective tld+1 when auto wildcard enabled", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + domain, ok := r.getWildcardDomainForHost("api.dev.example.co.uk") + require.True(t, ok) + require.Equal(t, "example.co.uk", domain) + }) + + t.Run("returns false for ip input", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + _, ok := r.getWildcardDomainForHost("1.1.1.1") + require.False(t, ok) + }) + + t.Run("returns false when feature is disabled", func(t *testing.T) { + r := Runner{options: &Options{}} + _, ok := r.getWildcardDomainForHost("api.example.com") + require.False(t, ok) + }) +} + func TestRunner_InputWorkerStream(t *testing.T) { options := &Options{ Hosts: "tests/stream_input.txt", From cc8b7ed36547a389c0f1c3e13608e4ca8f97534e Mon Sep 17 00:00:00 2001 From: Jujubee-LLM Date: Sat, 28 Feb 2026 10:18:52 +0800 Subject: [PATCH 4/6] refactor(wildcard): simplify host validation flow in wildcard domain resolver --- internal/runner/runner.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 86988a6e..0d6ee11b 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -565,19 +565,21 @@ func (r *Runner) run() error { } func (r *Runner) getWildcardDomainForHost(host string) (string, bool) { - if r.options.WildcardDomain != "" { - return r.options.WildcardDomain, true + if domain := r.options.WildcardDomain; domain != "" { + return domain, true } if !r.options.AutoWildcard { return "", false } + h := strings.TrimSpace(host) - if h == "" { + switch { + case h == "": return "", false - } - if strings.Contains(h, ":") { + case strings.Contains(h, ":"): return "", false } + domain, err := publicsuffix.EffectiveTLDPlusOne(h) if err != nil || domain == "" { return "", false From 167477979138321493cc8edb6e522477d7dae11e Mon Sep 17 00:00:00 2001 From: Jujubee-LLM Date: Sat, 28 Feb 2026 10:36:49 +0800 Subject: [PATCH 5/6] test+docs(wildcard): add coverage for host normalization and port input --- internal/runner/runner.go | 6 ++++++ internal/runner/runner_test.go | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 0d6ee11b..99de8c5c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -32,6 +32,7 @@ import ( ) // Runner is a client for running the enumeration process. +// wildcardTask carries the input host and the wildcard root domain used for matching. type wildcardTask struct { host string domain string @@ -564,6 +565,11 @@ func (r *Runner) run() error { return nil } +// getWildcardDomainForHost resolves the wildcard root domain for a host. +// +// Resolution order: +// - explicit options.WildcardDomain +// - auto-derived eTLD+1 when options.AutoWildcard is enabled func (r *Runner) getWildcardDomainForHost(host string) (string, bool) { if domain := r.options.WildcardDomain; domain != "" { return domain, true diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 2469a90d..bdb3719a 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -167,6 +167,19 @@ func TestRunner_getWildcardDomainForHost(t *testing.T) { _, ok := r.getWildcardDomainForHost("api.example.com") require.False(t, ok) }) + + t.Run("trims whitespace before deriving domain", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + domain, ok := r.getWildcardDomainForHost(" www.projectdiscovery.io ") + require.True(t, ok) + require.Equal(t, "projectdiscovery.io", domain) + }) + + t.Run("returns false for host with port", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + _, ok := r.getWildcardDomainForHost("api.example.com:443") + require.False(t, ok) + }) } func TestRunner_InputWorkerStream(t *testing.T) { From ffd7b9842e36ba00a7ace8d715a526b4e5cfeaf7 Mon Sep 17 00:00:00 2001 From: Jujubee-LLM Date: Tue, 3 Mar 2026 12:31:10 +0800 Subject: [PATCH 6/6] runner: allow wildcard filter with stream fallback path --- internal/runner/options.go | 4 +--- internal/runner/runner.go | 21 ++++++++++++++++++--- internal/runner/runner_test.go | 15 +++++++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index 1df917ea..1c08ba88 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -306,9 +306,7 @@ func (options *Options) validateOptions() { if options.Resume { gologger.Fatal().Msgf("resume not supported in stream mode") } - if options.WildcardDomain != "" || options.AutoWildcard { - gologger.Fatal().Msgf("wildcard not supported in stream mode") - } + // wildcard filtering is handled via buffered execution path even when stream is enabled if options.ShowStatistics { gologger.Fatal().Msgf("stats not supported in stream mode") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 99de8c5c..d507f882 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -57,6 +57,10 @@ type Runner struct { aurora aurora.Aurora } +func normalizeHostname(host string) string { + return strings.TrimSuffix(strings.TrimSpace(host), ".") +} + func New(options *Options) (*Runner, error) { retryabledns.CheckInternalIPs = true @@ -444,6 +448,10 @@ func (r *Runner) SaveResumeConfig() error { func (r *Runner) Run() error { if r.options.Stream { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { + gologger.Warning().Msgf("Wildcard filtering enabled in stream mode: falling back to buffered execution") + return r.run() + } return r.runStream() } @@ -520,6 +528,9 @@ func (r *Runner) run() error { continue } hostToWildcardDomain[host] = wildcardDomain + if r.isWildcardApexHost(host, wildcardDomain) { + continue + } if _, ok := seen[host]; !ok { seen[host] = struct{}{} r.wildcardworkerchan <- wildcardTask{host: host, domain: wildcardDomain} @@ -538,7 +549,7 @@ func (r *Runner) run() error { for _, A := range listIPs { for host := range ipDomain[A] { wildcardDomain, hasDomain := hostToWildcardDomain[host] - if hasDomain && host == wildcardDomain { + if hasDomain && r.isWildcardApexHost(host, wildcardDomain) { if _, ok := seen[host]; !ok { seen[host] = struct{}{} _ = r.lookupAndOutput(host) @@ -571,14 +582,14 @@ func (r *Runner) run() error { // - explicit options.WildcardDomain // - auto-derived eTLD+1 when options.AutoWildcard is enabled func (r *Runner) getWildcardDomainForHost(host string) (string, bool) { - if domain := r.options.WildcardDomain; domain != "" { + if domain := normalizeHostname(r.options.WildcardDomain); domain != "" { return domain, true } if !r.options.AutoWildcard { return "", false } - h := strings.TrimSpace(host) + h := normalizeHostname(host) switch { case h == "": return "", false @@ -593,6 +604,10 @@ func (r *Runner) getWildcardDomainForHost(host string) (string, bool) { return domain, true } +func (r *Runner) isWildcardApexHost(host, wildcardDomain string) bool { + return normalizeHostname(host) == normalizeHostname(wildcardDomain) +} + func (r *Runner) lookupAndOutput(host string) error { if r.options.JSON { if data, ok := r.hm.Get(host); ok { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index bdb3719a..202f895f 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -175,6 +175,13 @@ func TestRunner_getWildcardDomainForHost(t *testing.T) { require.Equal(t, "projectdiscovery.io", domain) }) + t.Run("trims trailing dot before deriving domain", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + domain, ok := r.getWildcardDomainForHost("api.projectdiscovery.io.") + require.True(t, ok) + require.Equal(t, "projectdiscovery.io", domain) + }) + t.Run("returns false for host with port", func(t *testing.T) { r := Runner{options: &Options{AutoWildcard: true}} _, ok := r.getWildcardDomainForHost("api.example.com:443") @@ -182,6 +189,14 @@ func TestRunner_getWildcardDomainForHost(t *testing.T) { }) } +func TestRunner_isWildcardApexHost(t *testing.T) { + r := Runner{} + + require.True(t, r.isWildcardApexHost("example.com", "example.com")) + require.True(t, r.isWildcardApexHost("example.com.", "example.com")) + require.False(t, r.isWildcardApexHost("api.example.com", "example.com")) +} + func TestRunner_InputWorkerStream(t *testing.T) { options := &Options{ Hosts: "tests/stream_input.txt",