mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-17 22:45:28 +00:00
3e54ca54b0
* feat: fixed leak * add go leak unit test in sdk * added goleak unit tests * bugfix: add random user agents to fuzzing requests * misc * misc * fix lint + use utils pr + misc * fix ratelimit memleak in sdk * close protocolstate shared resources in nuclei sdk/lib * add missing close references * ignore read/write loop of intransit connections * close unnecessary idle conns * add ignore method * using fixed utils * dep update --------- Co-authored-by: Ice3man <nizamulrana@gmail.com> Co-authored-by: mzack <marco.rivoli.nvh@gmail.com> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
75 lines
2.4 KiB
Go
75 lines
2.4 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/projectdiscovery/gologger"
|
|
"github.com/projectdiscovery/hmap/store/hybrid"
|
|
"github.com/projectdiscovery/httpx/common/httpx"
|
|
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
|
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
|
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
|
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
|
stringsutil "github.com/projectdiscovery/utils/strings"
|
|
syncutil "github.com/projectdiscovery/utils/sync"
|
|
)
|
|
|
|
// initializeTemplatesHTTPInput initializes the http form of input
|
|
// for any loaded http templates if input is in non-standard format.
|
|
func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
|
|
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "could not create temporary input file")
|
|
}
|
|
if r.inputProvider.InputType() == provider.MultiFormatInputProvider {
|
|
// currently http probing for input mode types is not supported
|
|
return hm, nil
|
|
}
|
|
gologger.Info().Msgf("Running httpx on input host")
|
|
|
|
httpxOptions := httpx.DefaultOptions
|
|
httpxOptions.RetryMax = r.options.Retries
|
|
httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
|
|
httpxOptions.NetworkPolicy = protocolstate.NetworkPolicy
|
|
httpxClient, err := httpx.New(&httpxOptions)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "could not create httpx client")
|
|
}
|
|
|
|
// Probe the non-standard URLs and store them in cache
|
|
swg, err := syncutil.New(syncutil.WithSize(r.options.BulkSize))
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "could not create adaptive group")
|
|
}
|
|
var count atomic.Int32
|
|
r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
|
|
if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
|
|
return true
|
|
}
|
|
|
|
if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency {
|
|
if err := swg.Resize(context.Background(), r.options.ProbeConcurrency); err != nil {
|
|
gologger.Error().Msgf("Could not resize workpool: %s\n", err)
|
|
}
|
|
}
|
|
|
|
swg.Add()
|
|
go func(input *contextargs.MetaInput) {
|
|
defer swg.Done()
|
|
|
|
if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
|
|
count.Add(1)
|
|
_ = hm.Set(input.Input, []byte(result))
|
|
}
|
|
}(value)
|
|
return true
|
|
})
|
|
swg.Wait()
|
|
|
|
gologger.Info().Msgf("Found %d URL from httpx", count.Load())
|
|
return hm, nil
|
|
}
|