package runner import ( "context" "fmt" "sync/atomic" "time" "github.com/pkg/errors" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/utils" stringsutil "github.com/projectdiscovery/utils/strings" syncutil "github.com/projectdiscovery/utils/sync" ) // initializeTemplatesHTTPInput initializes the http form of input // for any loaded http templates if input is in non-standard format. func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { hm, err := hybrid.New(hybrid.DefaultDiskOptions) if err != nil { return nil, errors.Wrap(err, "could not create temporary input file") } if r.inputProvider.InputType() == provider.MultiFormatInputProvider { // currently http probing for input mode types is not supported return hm, nil } r.Logger.Info().Msgf("Running httpx on input host") httpxOptions := httpx.DefaultOptions if r.options.AliveHttpProxy != "" { httpxOptions.Proxy = r.options.AliveHttpProxy } else if r.options.AliveSocksProxy != "" { httpxOptions.Proxy = r.options.AliveSocksProxy } httpxOptions.RetryMax = r.options.Retries httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second dialers := protocolstate.GetDialersWithId(r.options.ExecutionId) if dialers == nil { return nil, fmt.Errorf("dialers not initialized for %s", r.options.ExecutionId) } httpxOptions.NetworkPolicy = dialers.NetworkPolicy httpxClient, err := httpx.New(&httpxOptions) if err != nil { return nil, errors.Wrap(err, "could not create httpx client") } // Probe the non-standard URLs and store them in cache swg, err := syncutil.New(syncutil.WithSize(r.options.BulkSize)) if err != nil { return nil, errors.Wrap(err, "could not create adaptive group") } var count atomic.Int32 r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool { if stringsutil.HasPrefixAny(value.Input, "http://", "https://") { return true } if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency { if err := swg.Resize(context.Background(), r.options.ProbeConcurrency); err != nil { r.Logger.Error().Msgf("Could not resize workpool: %s\n", err) } } swg.Add() go func(input *contextargs.MetaInput) { defer swg.Done() if result := utils.ProbeURL(input.Input, httpxClient); result != "" { count.Add(1) _ = hm.Set(input.Input, []byte(result)) } }(value) return true }) swg.Wait() r.Logger.Info().Msgf("Found %d URL from httpx", count.Load()) return hm, nil }