diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..1c08ba88 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,7 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + AutoWildcard bool ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +190,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVar(&options.AutoWildcard, "auto-wildcard", false, "auto-detect wildcard root domain per host (multi-domain mode)"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) @@ -304,9 +306,7 @@ func (options *Options) validateOptions() { if options.Resume { gologger.Fatal().Msgf("resume not supported in stream mode") } - if options.WildcardDomain != "" { - gologger.Fatal().Msgf("wildcard not supported in stream mode") - } + // wildcard filtering is handled via buffered execution path even when stream is enabled if options.ShowStatistics { gologger.Fatal().Msgf("stats not supported in stream mode") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..d507f882 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -28,9 +28,16 @@ import ( iputil "github.com/projectdiscovery/utils/ip" mapsutil "github.com/projectdiscovery/utils/maps" sliceutil "github.com/projectdiscovery/utils/slice" + "golang.org/x/net/publicsuffix" ) // Runner is a client for running the enumeration process. +// wildcardTask carries the input host and the wildcard root domain used for matching. +type wildcardTask struct { + host string + domain string +} + type Runner struct { options *Options dnsx *dnsx.DNSX @@ -39,7 +46,7 @@ type Runner struct { wgwildcardworker *sync.WaitGroup workerchan chan string outputchan chan string - wildcardworkerchan chan string + wildcardworkerchan chan wildcardTask wildcards *mapsutil.SyncLockMap[string, struct{}] wildcardscache map[string][]string wildcardscachemutex sync.Mutex @@ -50,6 +57,10 @@ type Runner struct { aurora aurora.Aurora } +func normalizeHostname(host string) string { + return strings.TrimSuffix(strings.TrimSpace(host), ".") +} + func New(options *Options) (*Runner, error) { retryabledns.CheckInternalIPs = true @@ -115,7 +126,7 @@ func New(options *Options) (*Runner, error) { } // If no option is specified or wildcard filter has been requested use query type A - if len(questionTypes) == 0 || options.WildcardDomain != "" { + if len(questionTypes) == 0 || options.WildcardDomain != "" || options.AutoWildcard { options.A = true questionTypes = append(questionTypes, dns.TypeA) } @@ -156,7 +167,7 @@ func New(options *Options) (*Runner, error) { wgresolveworkers: &sync.WaitGroup{}, wgwildcardworker: &sync.WaitGroup{}, workerchan: make(chan string), - wildcardworkerchan: make(chan string), + wildcardworkerchan: make(chan wildcardTask), wildcards: mapsutil.NewSyncLockMap[string, struct{}](), wildcardscache: make(map[string][]string), limiter: limiter, @@ -437,6 +448,10 @@ func (r *Runner) SaveResumeConfig() error { func (r *Runner) Run() error { if r.options.Stream { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { + gologger.Warning().Msgf("Wildcard filtering enabled in stream mode: falling back to buffered execution") + return r.run() + } return r.runStream() } @@ -467,7 +482,7 @@ func (r *Runner) run() error { close(r.outputchan) r.wgoutputworker.Wait() - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { gologger.Print().Msgf("Starting to filter wildcard subdomains\n") ipDomain := make(map[string]map[string]struct{}) listIPs := []string{} @@ -503,13 +518,22 @@ func (r *Runner) run() error { } seen := make(map[string]struct{}) + hostToWildcardDomain := make(map[string]string) for _, a := range listIPs { hosts := ipDomain[a] if len(hosts) >= r.options.WildcardThreshold { for host := range hosts { + wildcardDomain, ok := r.getWildcardDomainForHost(host) + if !ok { + continue + } + hostToWildcardDomain[host] = wildcardDomain + if r.isWildcardApexHost(host, wildcardDomain) { + continue + } if _, ok := seen[host]; !ok { seen[host] = struct{}{} - r.wildcardworkerchan <- host + r.wildcardworkerchan <- wildcardTask{host: host, domain: wildcardDomain} } } } @@ -524,7 +548,8 @@ func (r *Runner) run() error { numRemovedSubdomains := 0 for _, A := range listIPs { for host := range ipDomain[A] { - if host == r.options.WildcardDomain { + wildcardDomain, hasDomain := hostToWildcardDomain[host] + if hasDomain && r.isWildcardApexHost(host, wildcardDomain) { if _, ok := seen[host]; !ok { seen[host] = struct{}{} _ = r.lookupAndOutput(host) @@ -551,6 +576,38 @@ func (r *Runner) run() error { return nil } +// getWildcardDomainForHost resolves the wildcard root domain for a host. +// +// Resolution order: +// - explicit options.WildcardDomain +// - auto-derived eTLD+1 when options.AutoWildcard is enabled +func (r *Runner) getWildcardDomainForHost(host string) (string, bool) { + if domain := normalizeHostname(r.options.WildcardDomain); domain != "" { + return domain, true + } + if !r.options.AutoWildcard { + return "", false + } + + h := normalizeHostname(host) + switch { + case h == "": + return "", false + case strings.Contains(h, ":"): + return "", false + } + + domain, err := publicsuffix.EffectiveTLDPlusOne(h) + if err != nil || domain == "" { + return "", false + } + return domain, true +} + +func (r *Runner) isWildcardApexHost(host, wildcardDomain string) bool { + return normalizeHostname(host) == normalizeHostname(wildcardDomain) +} + func (r *Runner) lookupAndOutput(host string) error { if r.options.JSON { if data, ok := r.hm.Get(host); ok { @@ -731,7 +788,7 @@ func (r *Runner) worker() { } } // if wildcard filtering just store the data - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { if err := r.storeDNSData(dnsData.DNSData); err != nil { gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err) } @@ -935,13 +992,13 @@ func (r *Runner) wildcardWorker() { defer r.wgwildcardworker.Done() for { - host, more := <-r.wildcardworkerchan + task, more := <-r.wildcardworkerchan if !more { break } - if r.IsWildcard(host) { + if r.IsWildcard(task.host, task.domain) { // mark this host as a wildcard subdomain - _ = r.wildcards.Set(host, struct{}{}) + _ = r.wildcards.Set(task.host, struct{}{}) } } } diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 179c639a..202f895f 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -141,6 +141,62 @@ func TestRunner_fileInput_prepareInput(t *testing.T) { require.ElementsMatch(t, expected, got, "could not match expected output") } +func TestRunner_getWildcardDomainForHost(t *testing.T) { + t.Run("uses explicit wildcard domain when provided", func(t *testing.T) { + r := Runner{options: &Options{WildcardDomain: "example.com"}} + domain, ok := r.getWildcardDomainForHost("api.foo.bar") + require.True(t, ok) + require.Equal(t, "example.com", domain) + }) + + t.Run("derives effective tld+1 when auto wildcard enabled", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + domain, ok := r.getWildcardDomainForHost("api.dev.example.co.uk") + require.True(t, ok) + require.Equal(t, "example.co.uk", domain) + }) + + t.Run("returns false for ip input", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + _, ok := r.getWildcardDomainForHost("1.1.1.1") + require.False(t, ok) + }) + + t.Run("returns false when feature is disabled", func(t *testing.T) { + r := Runner{options: &Options{}} + _, ok := r.getWildcardDomainForHost("api.example.com") + require.False(t, ok) + }) + + t.Run("trims whitespace before deriving domain", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + domain, ok := r.getWildcardDomainForHost(" www.projectdiscovery.io ") + require.True(t, ok) + require.Equal(t, "projectdiscovery.io", domain) + }) + + t.Run("trims trailing dot before deriving domain", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + domain, ok := r.getWildcardDomainForHost("api.projectdiscovery.io.") + require.True(t, ok) + require.Equal(t, "projectdiscovery.io", domain) + }) + + t.Run("returns false for host with port", func(t *testing.T) { + r := Runner{options: &Options{AutoWildcard: true}} + _, ok := r.getWildcardDomainForHost("api.example.com:443") + require.False(t, ok) + }) +} + +func TestRunner_isWildcardApexHost(t *testing.T) { + r := Runner{} + + require.True(t, r.isWildcardApexHost("example.com", "example.com")) + require.True(t, r.isWildcardApexHost("example.com.", "example.com")) + require.False(t, r.isWildcardApexHost("api.example.com", "example.com")) +} + func TestRunner_InputWorkerStream(t *testing.T) { options := &Options{ Hosts: "tests/stream_input.txt", diff --git a/internal/runner/wildcard.go b/internal/runner/wildcard.go index 5fdfc6a4..52dae69e 100644 --- a/internal/runner/wildcard.go +++ b/internal/runner/wildcard.go @@ -6,8 +6,8 @@ import ( "github.com/rs/xid" ) -// IsWildcard checks if a host is wildcard -func (r *Runner) IsWildcard(host string) bool { +// IsWildcard checks if a host is wildcard for a specific root domain +func (r *Runner) IsWildcard(host, wildcardDomain string) bool { orig := make(map[string]struct{}) wildcards := make(map[string]struct{}) @@ -19,7 +19,7 @@ func (r *Runner) IsWildcard(host string) bool { orig[A] = struct{}{} } - subdomainPart := strings.TrimSuffix(host, "."+r.options.WildcardDomain) + subdomainPart := strings.TrimSuffix(host, "."+wildcardDomain) subdomainTokens := strings.Split(subdomainPart, ".") // Build an array by preallocating a slice of a length @@ -27,11 +27,11 @@ func (r *Runner) IsWildcard(host string) bool { // We use a rand prefix at the beginning like %rand%.domain.tld // A permutation is generated for each level of the subdomain. var hosts []string - hosts = append(hosts, r.options.WildcardDomain) + hosts = append(hosts, wildcardDomain) if len(subdomainTokens) > 0 { for i := 1; i < len(subdomainTokens); i++ { - newhost := strings.Join(subdomainTokens[i:], ".") + "." + r.options.WildcardDomain + newhost := strings.Join(subdomainTokens[i:], ".") + "." + wildcardDomain hosts = append(hosts, newhost) } }