diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 92ead8b14..fbe401d00 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -31,8 +31,6 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri workflowCount++ } sev := strings.ToLower(types.ToString(t.Info["severity"])) - - fmt.Printf("info: %+v\n", t.Info) if !filterBySeverity || hasMatchingSeverity(sev, severities) { parsedTemplates[t.ID] = t gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev)) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1f154460a..d19030ca4 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -27,10 +27,10 @@ import ( const defaultMaxWorkers = 150 // executeRaceRequest executes race condition request for a URL -func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := e.newGenerator() +func (r *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := r.newGenerator() - maxWorkers := e.RaceNumberRequests + maxWorkers := r.RaceNumberRequests swg := sizedwaitgroup.New(maxWorkers) var requestErr error @@ -40,10 +40,10 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp if err != nil { return err } - for i := 0; i < e.RaceNumberRequests; i++ { + for i := 0; i < r.RaceNumberRequests; i++ { swg.Add() go func(httpRequest *generatedRequest) { - err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) + err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) @@ -56,12 +56,12 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp return requestErr } -// executeRaceRequest executes race condition request for a URL -func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := e.newGenerator() +// executeRaceRequest executes parallel requests for a template +func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := r.newGenerator() // Workers that keeps enqueuing new requests - maxWorkers := e.Threads + maxWorkers := r.Threads swg := sizedwaitgroup.New(maxWorkers) var requestErr error @@ -72,30 +72,30 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Total())) + r.options.Progress.DecrementRequests(int64(generator.Total())) return err } swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - e.options.RateLimiter.Take() - err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) + r.options.RateLimiter.Take() + err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() }(request) - e.options.Progress.IncrementRequests() + r.options.Progress.IncrementRequests() } swg.Wait() return requestErr } -// executeRaceRequest executes race condition request for a URL -func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := e.newGenerator() +// executeRaceRequest executes turbo http request for a URL +func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := r.newGenerator() // need to extract the target from the url URL, err := url.Parse(reqURL) @@ -106,11 +106,11 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output pipeOptions := rawhttp.DefaultPipelineOptions pipeOptions.Host = URL.Host pipeOptions.MaxConnections = 1 - if e.PipelineConcurrentConnections > 0 { - pipeOptions.MaxConnections = e.PipelineConcurrentConnections + if r.PipelineConcurrentConnections > 0 { + pipeOptions.MaxConnections = r.PipelineConcurrentConnections } - if e.PipelineRequestsPerConnection > 0 { - pipeOptions.MaxPendingRequests = e.PipelineRequestsPerConnection + if r.PipelineRequestsPerConnection > 0 { + pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection } pipeclient := rawhttp.NewPipelineClient(pipeOptions) @@ -130,7 +130,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Total())) + r.options.Progress.DecrementRequests(int64(generator.Total())) return err } request.pipelinedClient = pipeclient @@ -139,14 +139,14 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output go func(httpRequest *generatedRequest) { defer swg.Done() - err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) + err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() }(request) - e.options.Progress.IncrementRequests() + r.options.Progress.IncrementRequests() } swg.Wait() return requestErr @@ -352,12 +352,12 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam const two = 2 // setCustomHeaders sets the custom headers for generated request -func (e *Request) setCustomHeaders(r *generatedRequest) { - for k, v := range e.customHeaders { - if r.rawRequest != nil { - r.rawRequest.Headers[k] = v +func (r *Request) setCustomHeaders(req *generatedRequest) { + for k, v := range r.customHeaders { + if req.rawRequest != nil { + req.rawRequest.Headers[k] = v } else { - r.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v)) + req.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v)) } } } diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go new file mode 100644 index 000000000..d02cfda64 --- /dev/null +++ b/v2/pkg/protocols/http/request_test.go @@ -0,0 +1 @@ +package http diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 545f6649c..223beb633 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -34,7 +34,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res if len(template.Executers) == 1 { mainErr = err } else { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err) + gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } continue } @@ -76,7 +76,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res if len(template.Executers) == 1 { mainErr = err } else { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err) + gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } continue } @@ -84,7 +84,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res if len(template.Executers) == 1 { mainErr = executionErr } else { - gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", executionErr) + gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, executionErr) } } } diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 61f85103e..394b6e9b1 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -14,11 +14,10 @@ func TestWorkflowsSimple(t *testing.T) { progress, _ := progress.NewProgress(false, false, 0) workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true}}}, - }, - options: &protocols.ExecuterOptions{ - Progress: progress, - }} + {Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }}, + }} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -30,14 +29,17 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - firstInput = input - }}}}, - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - secondInput = input - }}}}, - }, - options: &protocols.ExecuterOptions{Progress: progress}} + {Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }}, + {Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }}, + }} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -52,16 +54,16 @@ func TestWorkflowsSubtemplates(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - firstInput = input - }}}, - Subtemplates: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - secondInput = input - }}}, - }}}, - }, - options: &protocols.ExecuterOptions{Progress: progress}} + {Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }}}}, + }} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -76,16 +78,16 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: false, executeHook: func(input string) { - firstInput = input - }}}, - Subtemplates: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - secondInput = input - }}}, - }}}, - }, - options: &protocols.ExecuterOptions{Progress: progress}} + {Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: false, executeHook: func(input string) { + firstInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }}}}, + }} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -100,24 +102,21 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - firstInput = input - }, outputs: []*output.InternalWrappedEvent{ - {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, - Extracts: map[string][]string{}, - }}, - }}}, - Matchers: []*Matcher{ - {Name: "tomcat", Subtemplates: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - secondInput = input - }}}, - }}}, - }, - }, - }, - options: &protocols.ExecuterOptions{Progress: progress}} + {Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }, outputs: []*output.InternalWrappedEvent{ + {OperatorsResult: &operators.Result{ + Matches: map[string]struct{}{"tomcat": {}}, + Extracts: map[string][]string{}, + }}, + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }}}}}}, + }} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -132,24 +131,21 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - firstInput = input - }, outputs: []*output.InternalWrappedEvent{ - {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, - Extracts: map[string][]string{}, - }}, - }}}, - Matchers: []*Matcher{ - {Name: "apache", Subtemplates: []*WorkflowTemplate{ - {Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { - secondInput = input - }}}}, + {Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }, outputs: []*output.InternalWrappedEvent{ + {OperatorsResult: &operators.Result{ + Matches: map[string]struct{}{"tomcat": {}}, + Extracts: map[string][]string{}, }}, - }, - }, - }, - options: &protocols.ExecuterOptions{Progress: progress}} + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + Executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}, Options: &protocols.ExecuterOptions{Progress: progress}}, + }}}}}}, + }} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow")