init- using resizable components

This commit is contained in:
Mzack9999 2024-04-03 17:50:57 +02:00
parent e99420603f
commit a8d1393e96
20 changed files with 126 additions and 105 deletions

View File

@ -2,7 +2,7 @@ package main
import ( import (
nuclei "github.com/projectdiscovery/nuclei/v3/lib" nuclei "github.com/projectdiscovery/nuclei/v3/lib"
"github.com/remeh/sizedwaitgroup" syncutil "github.com/projectdiscovery/utils/sync"
) )
func main() { func main() {
@ -12,7 +12,10 @@ func main() {
panic(err) panic(err)
} }
// setup sizedWaitgroup to handle concurrency // setup sizedWaitgroup to handle concurrency
sg := sizedwaitgroup.New(10) sg, err := syncutil.New(syncutil.WithSize(10))
if err != nil {
panic(err)
}
// scan 1 = run dns templates on scanme.sh // scan 1 = run dns templates on scanme.sh
sg.Add() sg.Add()

2
go.mod
View File

@ -142,6 +142,8 @@ require (
github.com/docker/cli v24.0.5+incompatible // indirect github.com/docker/cli v24.0.5+incompatible // indirect
github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/docker v24.0.9+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-connections v0.4.0 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/fatih/color v1.15.0 // indirect github.com/fatih/color v1.15.0 // indirect
github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect

3
go.sum
View File

@ -296,8 +296,11 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=

View File

@ -12,7 +12,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/nuclei/v3/pkg/utils"
stringsutil "github.com/projectdiscovery/utils/strings" stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/remeh/sizedwaitgroup" syncutil "github.com/projectdiscovery/utils/sync"
) )
const probeBulkSize = 50 const probeBulkSize = 50
@ -45,8 +45,11 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
} }
// Probe the non-standard URLs and store them in cache // Probe the non-standard URLs and store them in cache
swg := sizedwaitgroup.New(bulkSize) swg, err := syncutil.New(syncutil.WithSize(bulkSize))
count := int32(0) if err != nil {
return nil, errors.Wrap(err, "could not create adaptive group")
}
var count atomic.Int32
r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool { r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
if stringsutil.HasPrefixAny(value.Input, "http://", "https://") { if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
return true return true
@ -57,7 +60,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
defer swg.Done() defer swg.Done()
if result := utils.ProbeURL(input.Input, httpxClient); result != "" { if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
atomic.AddInt32(&count, 1) count.Add(1)
_ = hm.Set(input.Input, []byte(result)) _ = hm.Set(input.Input, []byte(result))
} }
}(value) }(value)
@ -65,6 +68,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
}) })
swg.Wait() swg.Wait()
gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count)) gologger.Info().Msgf("Found %d URL from httpx", count.Load())
return hm, nil return hm, nil
} }

View File

@ -4,8 +4,6 @@ import (
"sync" "sync"
"sync/atomic" "sync/atomic"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/output"
@ -14,6 +12,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy" "github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
stringsutil "github.com/projectdiscovery/utils/strings" stringsutil "github.com/projectdiscovery/utils/strings"
syncutil "github.com/projectdiscovery/utils/sync"
) )
// Execute takes a list of templates/workflows that have been compiled // Execute takes a list of templates/workflows that have been compiled
@ -111,7 +110,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
for _, template := range templatesList { for _, template := range templatesList {
templateType := template.Type() templateType := template.Type()
var wg *sizedwaitgroup.SizedWaitGroup var wg *syncutil.AdaptiveWaitGroup
if templateType == types.HeadlessProtocol { if templateType == types.HeadlessProtocol {
wg = wp.Headless wg = wp.Headless
} else { } else {
@ -134,7 +133,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
// executeHostSpray executes scan using host spray strategy where templates are iterated over each target // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool { func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
results := &atomic.Bool{} results := &atomic.Bool{}
wp := sizedwaitgroup.New(e.options.BulkSize + e.options.HeadlessBulkSize) wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
target.Iterate(func(value *contextargs.MetaInput) bool { target.Iterate(func(value *contextargs.MetaInput) bool {
wp.Add() wp.Add()

View File

@ -11,7 +11,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types" generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/remeh/sizedwaitgroup" syncutil "github.com/projectdiscovery/utils/sync"
) )
// Executors are low level executors that deals with template execution on a target // Executors are low level executors that deals with template execution on a target
@ -104,9 +104,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
return true return true
} }
wg.WaitGroup.Add() wg.Add()
go func(index uint32, skip bool, value *contextargs.MetaInput) { go func(index uint32, skip bool, value *contextargs.MetaInput) {
defer wg.WaitGroup.Done() defer wg.Done()
defer cleanupInFlight(index) defer cleanupInFlight(index)
if skip { if skip {
return return
@ -140,7 +140,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
index++ index++
return true return true
}) })
wg.WaitGroup.Wait() wg.Wait()
// on completion marks the template as completed // on completion marks the template as completed
currentInfo.Lock() currentInfo.Lock()
@ -158,14 +158,14 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
wp := e.GetWorkPool() wp := e.GetWorkPool()
for _, tpl := range alltemplates { for _, tpl := range alltemplates {
var sg *sizedwaitgroup.SizedWaitGroup var sg *syncutil.AdaptiveWaitGroup
if tpl.Type() == types.HeadlessProtocol { if tpl.Type() == types.HeadlessProtocol {
sg = wp.Headless sg = wp.Headless
} else { } else {
sg = wp.Default sg = wp.Default
} }
sg.Add() sg.Add()
go func(template *templates.Template, value *contextargs.MetaInput, wg *sizedwaitgroup.SizedWaitGroup) { go func(template *templates.Template, value *contextargs.MetaInput, wg *syncutil.AdaptiveWaitGroup) {
defer wg.Done() defer wg.Done()
var match bool var match bool
@ -213,7 +213,7 @@ func (e *ChildExecuter) Close() *atomic.Bool {
func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) { func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
templateType := template.Type() templateType := template.Type()
var wg *sizedwaitgroup.SizedWaitGroup var wg *syncutil.AdaptiveWaitGroup
if templateType == types.HeadlessProtocol { if templateType == types.HeadlessProtocol {
wg = e.e.workPool.Headless wg = e.e.workPool.Headless
} else { } else {

View File

@ -5,13 +5,12 @@ import (
"net/http/cookiejar" "net/http/cookiejar"
"sync/atomic" "sync/atomic"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/scan"
"github.com/projectdiscovery/nuclei/v3/pkg/workflows" "github.com/projectdiscovery/nuclei/v3/pkg/workflows"
syncutil "github.com/projectdiscovery/utils/sync"
) )
const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n" const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
@ -32,7 +31,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
if templateThreads == 1 { if templateThreads == 1 {
templateThreads++ templateThreads++
} }
swg := sizedwaitgroup.New(templateThreads) swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
for _, template := range w.Workflows { for _, template := range w.Workflows {
swg.Add() swg.Add()
@ -40,7 +39,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
func(template *workflows.WorkflowTemplate) { func(template *workflows.WorkflowTemplate) {
defer swg.Done() defer swg.Done()
if err := e.runWorkflowStep(template, ctx, results, &swg, w); err != nil { if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
} }
}(template) }(template)
@ -51,7 +50,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
// runWorkflowStep runs a workflow step for the workflow. It executes the workflow // runWorkflowStep runs a workflow step for the workflow. It executes the workflow
// in a recursive manner running all subtemplates and matchers. // in a recursive manner running all subtemplates and matchers.
func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error {
var firstMatched bool var firstMatched bool
var err error var err error
var mainErr error var mainErr error

View File

@ -1,9 +1,8 @@
package core package core
import ( import (
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
syncutil "github.com/projectdiscovery/utils/sync"
) )
// WorkPool implements an execution pool for executing different // WorkPool implements an execution pool for executing different
@ -12,8 +11,8 @@ import (
// It also allows Configuration of such requirements. This is used // It also allows Configuration of such requirements. This is used
// for per-module like separate headless concurrency etc. // for per-module like separate headless concurrency etc.
type WorkPool struct { type WorkPool struct {
Headless *sizedwaitgroup.SizedWaitGroup Headless *syncutil.AdaptiveWaitGroup
Default *sizedwaitgroup.SizedWaitGroup Default *syncutil.AdaptiveWaitGroup
config WorkPoolConfig config WorkPoolConfig
} }
@ -31,13 +30,13 @@ type WorkPoolConfig struct {
// NewWorkPool returns a new WorkPool instance // NewWorkPool returns a new WorkPool instance
func NewWorkPool(config WorkPoolConfig) *WorkPool { func NewWorkPool(config WorkPoolConfig) *WorkPool {
headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency) headlessWg, _ := syncutil.New(syncutil.WithSize(config.HeadlessTypeConcurrency))
defaultWg := sizedwaitgroup.New(config.TypeConcurrency) defaultWg, _ := syncutil.New(syncutil.WithSize(config.TypeConcurrency))
return &WorkPool{ return &WorkPool{
config: config, config: config,
Headless: &headlessWg, Headless: headlessWg,
Default: &defaultWg, Default: defaultWg,
} }
} }
@ -47,19 +46,14 @@ func (w *WorkPool) Wait() {
w.Headless.Wait() w.Headless.Wait()
} }
// InputWorkPool is a work pool per-input
type InputWorkPool struct {
WaitGroup *sizedwaitgroup.SizedWaitGroup
}
// InputPool returns a work pool for an input type // InputPool returns a work pool for an input type
func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool { func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.AdaptiveWaitGroup {
var count int var count int
if templateType == types.HeadlessProtocol { if templateType == types.HeadlessProtocol {
count = w.config.HeadlessInputConcurrency count = w.config.HeadlessInputConcurrency
} else { } else {
count = w.config.InputConcurrency count = w.config.InputConcurrency
} }
swg := sizedwaitgroup.New(count) swg, _ := syncutil.New(syncutil.WithSize(count))
return &InputWorkPool{WaitGroup: &swg} return swg
} }

View File

@ -4,13 +4,13 @@ import (
"sync" "sync"
"github.com/dop251/goja" "github.com/dop251/goja"
"github.com/remeh/sizedwaitgroup" syncutil "github.com/projectdiscovery/utils/sync"
) )
var ( var (
ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency) ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
lazyFixedSgInit = sync.OnceFunc(func() { lazyFixedSgInit = sync.OnceFunc(func() {
ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency) ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
}) })
) )

View File

@ -36,7 +36,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole" "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
stringsutil "github.com/projectdiscovery/utils/strings" stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/remeh/sizedwaitgroup" syncutil "github.com/projectdiscovery/utils/sync"
) )
const ( const (
@ -51,9 +51,9 @@ var (
// autoregister console node module with default printer it uses gologger backend // autoregister console node module with default printer it uses gologger backend
require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter())) require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter()))
}) })
pooljsc sizedwaitgroup.SizedWaitGroup pooljsc *syncutil.AdaptiveWaitGroup
lazySgInit = sync.OnceFunc(func() { lazySgInit = sync.OnceFunc(func() {
pooljsc = sizedwaitgroup.New(PoolingJsVmConcurrency) pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
}) })
) )

View File

@ -30,8 +30,8 @@ import (
mapsutil "github.com/projectdiscovery/utils/maps" mapsutil "github.com/projectdiscovery/utils/maps"
sliceutil "github.com/projectdiscovery/utils/slice" sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings" stringsutil "github.com/projectdiscovery/utils/strings"
syncutil "github.com/projectdiscovery/utils/sync"
wappalyzer "github.com/projectdiscovery/wappalyzergo" wappalyzer "github.com/projectdiscovery/wappalyzergo"
"github.com/remeh/sizedwaitgroup"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
@ -128,7 +128,10 @@ func (s *Service) Close() bool {
func (s *Service) Execute() error { func (s *Service) Execute() error {
gologger.Info().Msgf("Executing Automatic scan on %d target[s]", s.target.Count()) gologger.Info().Msgf("Executing Automatic scan on %d target[s]", s.target.Count())
// setup host concurrency // setup host concurrency
sg := sizedwaitgroup.New(s.opts.Options.BulkSize) sg, err := syncutil.New(syncutil.WithSize(s.opts.Options.BulkSize))
if err != nil {
return err
}
s.target.Iterate(func(value *contextargs.MetaInput) bool { s.target.Iterate(func(value *contextargs.MetaInput) bool {
sg.Add() sg.Add()
go func(input *contextargs.MetaInput) { go func(input *contextargs.MetaInput) {
@ -246,7 +249,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
// execute tech detection templates on target // execute tech detection templates on target
tags := map[string]struct{}{} tags := map[string]struct{}{}
m := &sync.Mutex{} m := &sync.Mutex{}
sg := sizedwaitgroup.New(s.opts.Options.TemplateThreads) sg, _ := syncutil.New(syncutil.WithSize(s.opts.Options.TemplateThreads))
counter := atomic.Uint32{} counter := atomic.Uint32{}
for _, t := range s.techTemplates { for _, t := range s.techTemplates {

View File

@ -9,7 +9,6 @@ import (
"github.com/miekg/dns" "github.com/miekg/dns"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/multierr" "go.uber.org/multierr"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
@ -27,6 +26,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/retryabledns" "github.com/projectdiscovery/retryabledns"
iputil "github.com/projectdiscovery/utils/ip" iputil "github.com/projectdiscovery/utils/ip"
syncutil "github.com/projectdiscovery/utils/sync"
) )
var _ protocols.Request = &Request{} var _ protocols.Request = &Request{}
@ -64,7 +64,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
if request.generator != nil { if request.generator != nil {
iterator := request.generator.NewIterator() iterator := request.generator.NewIterator()
swg := sizedwaitgroup.New(request.Threads) swg, err := syncutil.New(syncutil.WithSize(request.Threads))
if err != nil {
return err
}
var multiErr error var multiErr error
m := &sync.Mutex{} m := &sync.Mutex{}

View File

@ -11,7 +11,6 @@ import (
"github.com/docker/go-units" "github.com/docker/go-units"
"github.com/mholt/archiver" "github.com/mholt/archiver"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/operators"
@ -24,6 +23,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
sliceutil "github.com/projectdiscovery/utils/slice" sliceutil "github.com/projectdiscovery/utils/slice"
syncutil "github.com/projectdiscovery/utils/sync"
) )
var _ protocols.Request = &Request{} var _ protocols.Request = &Request{}
@ -47,8 +47,11 @@ var errEmptyResult = errors.New("Empty result")
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize) wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
err := request.getInputPaths(input.MetaInput.Input, func(filePath string) { if err != nil {
return err
}
err = request.getInputPaths(input.MetaInput.Input, func(filePath string) {
wg.Add() wg.Add()
func(filePath string) { func(filePath string) {
defer wg.Done() defer wg.Done()

View File

@ -214,7 +214,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti
} }
// ActionAddHeader executes a AddHeader action. // ActionAddHeader executes a AddHeader action.
func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
in := p.getActionArgWithDefaultValues(act, "part") in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string) args := make(map[string]string)
@ -225,7 +225,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review
} }
// ActionSetHeader executes a SetHeader action. // ActionSetHeader executes a SetHeader action.
func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
in := p.getActionArgWithDefaultValues(act, "part") in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string) args := make(map[string]string)
@ -236,7 +236,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review
} }
// ActionDeleteHeader executes a DeleteHeader action. // ActionDeleteHeader executes a DeleteHeader action.
func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error {
in := p.getActionArgWithDefaultValues(act, "part") in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string) args := make(map[string]string)
@ -343,7 +343,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error {
} }
// ClickElement executes click actions for an element. // ClickElement executes click actions for an element.
func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) ClickElement(act *Action, out map[string]string) error {
element, err := p.pageElementBy(act.Data) element, err := p.pageElementBy(act.Data)
if err != nil { if err != nil {
return errors.Wrap(err, errCouldNotGetElement) return errors.Wrap(err, errCouldNotGetElement)
@ -358,12 +358,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unu
} }
// KeyboardAction executes a keyboard action on the page. // KeyboardAction executes a keyboard action on the page.
func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) KeyboardAction(act *Action, out map[string]string) error {
return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...) return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...)
} }
// RightClickElement executes right click actions for an element. // RightClickElement executes right click actions for an element.
func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) RightClickElement(act *Action, out map[string]string) error {
element, err := p.pageElementBy(act.Data) element, err := p.pageElementBy(act.Data)
if err != nil { if err != nil {
return errors.Wrap(err, errCouldNotGetElement) return errors.Wrap(err, errCouldNotGetElement)
@ -441,7 +441,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error {
} }
// InputElement executes input element actions for an element. // InputElement executes input element actions for an element.
func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) InputElement(act *Action, out map[string]string) error {
value := p.getActionArgWithDefaultValues(act, "value") value := p.getActionArgWithDefaultValues(act, "value")
if value == "" { if value == "" {
return errinvalidArguments return errinvalidArguments
@ -460,7 +460,7 @@ func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unu
} }
// TimeInputElement executes time input on an element // TimeInputElement executes time input on an element
func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
value := p.getActionArgWithDefaultValues(act, "value") value := p.getActionArgWithDefaultValues(act, "value")
if value == "" { if value == "" {
return errinvalidArguments return errinvalidArguments
@ -483,7 +483,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review
} }
// SelectInputElement executes select input statement action on a element // SelectInputElement executes select input statement action on a element
func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
value := p.getActionArgWithDefaultValues(act, "value") value := p.getActionArgWithDefaultValues(act, "value")
if value == "" { if value == "" {
return errinvalidArguments return errinvalidArguments
@ -508,7 +508,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi
} }
// WaitLoad waits for the page to load // WaitLoad waits for the page to load
func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) WaitLoad(act *Action, out map[string]string) error {
p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
// Wait for the window.onload event and also wait for the network requests // Wait for the window.onload event and also wait for the network requests
@ -538,7 +538,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error {
} }
// FilesInput acts with a file input element on page // FilesInput acts with a file input element on page
func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) FilesInput(act *Action, out map[string]string) error {
element, err := p.pageElementBy(act.Data) element, err := p.pageElementBy(act.Data)
if err != nil { if err != nil {
return errors.Wrap(err, errCouldNotGetElement) return errors.Wrap(err, errCouldNotGetElement)
@ -589,7 +589,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error {
} }
// WaitEvent waits for an event to happen on the page. // WaitEvent waits for an event to happen on the page.
func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused parameter*/) (func() error, error) { func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) {
event := p.getActionArgWithDefaultValues(act, "event") event := p.getActionArgWithDefaultValues(act, "event")
if event == "" { if event == "" {
return nil, errors.New("event not recognized") return nil, errors.New("event not recognized")
@ -661,14 +661,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) {
} }
// DebugAction enables debug action on a page. // DebugAction enables debug action on a page.
func (p *Page) DebugAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) DebugAction(act *Action, out map[string]string) error {
p.instance.browser.engine.SlowMotion(5 * time.Second) p.instance.browser.engine.SlowMotion(5 * time.Second)
p.instance.browser.engine.Trace(true) p.instance.browser.engine.Trace(true)
return nil return nil
} }
// SleepAction sleeps on the page for a specified duration // SleepAction sleeps on the page for a specified duration
func (p *Page) SleepAction(act *Action, out map[string]string /*TODO review unused parameter*/) error { func (p *Page) SleepAction(act *Action, out map[string]string) error {
seconds := act.Data["duration"] seconds := act.Data["duration"]
if seconds == "" { if seconds == "" {
seconds = "5" seconds = "5"

View File

@ -4,7 +4,7 @@ import (
"context" "context"
"sync" "sync"
"github.com/remeh/sizedwaitgroup" syncutil "github.com/projectdiscovery/utils/sync"
) )
// WorkPoolType is the type of work pool to use // WorkPoolType is the type of work pool to use
@ -26,7 +26,7 @@ type StopAtFirstMatchHandler[T any] struct {
// work pool and its type // work pool and its type
poolType WorkPoolType poolType WorkPoolType
sgPool sizedwaitgroup.SizedWaitGroup sgPool *syncutil.AdaptiveWaitGroup
wgPool *sync.WaitGroup wgPool *sync.WaitGroup
// internal / unexported // internal / unexported
@ -40,10 +40,13 @@ type StopAtFirstMatchHandler[T any] struct {
// NewBlockingSPMHandler creates a new stop at first match handler // NewBlockingSPMHandler creates a new stop at first match handler
func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] { func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
ctx1, cancel := context.WithCancel(ctx) ctx1, cancel := context.WithCancel(ctx)
awg, _ := syncutil.New(syncutil.WithSize(size))
s := &StopAtFirstMatchHandler[T]{ s := &StopAtFirstMatchHandler[T]{
ResultChan: make(chan T, 1), ResultChan: make(chan T, 1),
poolType: Blocking, poolType: Blocking,
sgPool: sizedwaitgroup.New(size), sgPool: awg,
internalWg: &sync.WaitGroup{}, internalWg: &sync.WaitGroup{},
ctx: ctx1, ctx: ctx1,
cancel: cancel, cancel: cancel,

View File

@ -32,8 +32,8 @@ import (
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/types"
errorutil "github.com/projectdiscovery/utils/errors" errorutil "github.com/projectdiscovery/utils/errors"
syncutil "github.com/projectdiscovery/utils/sync"
urlutil "github.com/projectdiscovery/utils/url" urlutil "github.com/projectdiscovery/utils/url"
"github.com/remeh/sizedwaitgroup"
) )
// Request is a request for the javascript protocol // Request is a request for the javascript protocol
@ -404,7 +404,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
requestOptions := request.options requestOptions := request.options
gotmatches := &atomic.Bool{} gotmatches := &atomic.Bool{}
sg := sizedwaitgroup.New(threads) sg, _ := syncutil.New(syncutil.WithSize(threads))
if request.generator != nil { if request.generator != nil {
iterator := request.generator.NewIterator() iterator := request.generator.NewIterator()
for { for {

View File

@ -11,7 +11,7 @@ var (
) )
// Init initializes the clientpool implementation // Init initializes the clientpool implementation
func Init(options *types.Options /*TODO review unused parameter*/) error { func Init(options *types.Options) error {
// Don't create clients if already created in the past. // Don't create clients if already created in the past.
if normalClient != nil { if normalClient != nil {
return nil return nil

View File

@ -12,7 +12,6 @@ import (
"time" "time"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/multierr" "go.uber.org/multierr"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
@ -34,6 +33,7 @@ import (
errorutil "github.com/projectdiscovery/utils/errors" errorutil "github.com/projectdiscovery/utils/errors"
mapsutil "github.com/projectdiscovery/utils/maps" mapsutil "github.com/projectdiscovery/utils/maps"
"github.com/projectdiscovery/utils/reader" "github.com/projectdiscovery/utils/reader"
syncutil "github.com/projectdiscovery/utils/sync"
) )
var ( var (
@ -178,7 +178,10 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
iterator := request.generator.NewIterator() iterator := request.generator.NewIterator()
var multiErr error var multiErr error
m := &sync.Mutex{} m := &sync.Mutex{}
swg := sizedwaitgroup.New(request.Threads) swg, err := syncutil.New(syncutil.WithSize(request.Threads))
if err != nil {
return err
}
for { for {
value, ok := iterator.Value() value, ok := iterator.Value()

View File

@ -6,7 +6,6 @@ import (
"os" "os"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/output"
@ -17,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/utils/conversion" "github.com/projectdiscovery/utils/conversion"
syncutil "github.com/projectdiscovery/utils/sync"
) )
var _ protocols.Request = &Request{} var _ protocols.Request = &Request{}
@ -29,10 +29,13 @@ func (request *Request) Type() templateTypes.ProtocolType {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize) wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
if err != nil {
return err
}
err := request.getInputPaths(input.MetaInput.Input, func(data string) { err = request.getInputPaths(input.MetaInput.Input, func(data string) {
wg.Add() wg.Add()
go func(data string) { go func(data string) {

View File

@ -1,6 +1,7 @@
package flow package flow
import ( import (
"context"
"reflect" "reflect"
"sync" "sync"
@ -12,33 +13,10 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin"
"github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/utils/sync/sizedpool"
) )
type jsWaitGroup struct { var jsOnce sync.Once
sync.Once
sg sizedwaitgroup.SizedWaitGroup
}
var jsPool = &jsWaitGroup{}
// GetJSRuntime returns a new JS runtime from pool
func GetJSRuntime(opts *types.Options) *goja.Runtime {
jsPool.Do(func() {
if opts.JsConcurrency < 100 {
opts.JsConcurrency = 100
}
jsPool.sg = sizedwaitgroup.New(opts.JsConcurrency)
})
jsPool.sg.Add()
return gojapool.Get().(*goja.Runtime)
}
// PutJSRuntime returns a JS runtime to pool
func PutJSRuntime(runtime *goja.Runtime) {
defer jsPool.sg.Done()
gojapool.Put(runtime)
}
// js runtime pool using sync.Pool // js runtime pool using sync.Pool
var gojapool = &sync.Pool{ var gojapool = &sync.Pool{
@ -49,8 +27,29 @@ var gojapool = &sync.Pool{
}, },
} }
func registerBuiltins(runtime *goja.Runtime) { var sizedgojapool *sizedpool.SizedPool[*goja.Runtime]
// GetJSRuntime returns a new JS runtime from pool
func GetJSRuntime(opts *types.Options) *goja.Runtime {
jsOnce.Do(func() {
if opts.JsConcurrency < 100 {
opts.JsConcurrency = 100
}
sizedgojapool, _ = sizedpool.New[*goja.Runtime](
sizedpool.WithPool[*goja.Runtime](gojapool),
sizedpool.WithSize[*goja.Runtime](int64(opts.JsConcurrency)),
)
})
runtime, _ := sizedgojapool.Get(context.TODO())
return runtime
}
// PutJSRuntime returns a JS runtime to pool
func PutJSRuntime(runtime *goja.Runtime) {
sizedgojapool.Put(runtime)
}
func registerBuiltins(runtime *goja.Runtime) {
_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "log", Name: "log",
Description: "Logs a given object/message to stdout (only for debugging purposes)", Description: "Logs a given object/message to stdout (only for debugging purposes)",