External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-20 23:15:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Misc

Browse Source
This commit is contained in:
Ice3man543 2021-02-04 22:00:09 +05:30
parent 4d9d53ca60
commit 9402501842
5 changed files with 96 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/internal/runner/templates.go
View File

@ -31,8 +31,6 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri
workflowCount++ workflowCount++
} }
sev := strings.ToLower(types.ToString(t.Info["severity"])) sev := strings.ToLower(types.ToString(t.Info["severity"]))
fmt.Printf("info: %+v\n", t.Info)
if !filterBySeverity || hasMatchingSeverity(sev, severities) { if !filterBySeverity || hasMatchingSeverity(sev, severities) {
parsedTemplates[t.ID] = t parsedTemplates[t.ID] = t
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev)) gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))

56
v2/pkg/protocols/http/request.go
View File

@ -27,10 +27,10 @@ import (
const defaultMaxWorkers = 150 const defaultMaxWorkers = 150
// executeRaceRequest executes race condition request for a URL // executeRaceRequest executes race condition request for a URL
func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (r *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator() generator := r.newGenerator()
maxWorkers := e.RaceNumberRequests maxWorkers := r.RaceNumberRequests
swg := sizedwaitgroup.New(maxWorkers) swg := sizedwaitgroup.New(maxWorkers)
var requestErr error var requestErr error
@ -40,10 +40,10 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp
if err != nil { if err != nil {
return err return err
} }
for i := 0; i < e.RaceNumberRequests; i++ { for i := 0; i < r.RaceNumberRequests; i++ {
swg.Add() swg.Add()
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock() mutex.Lock()
if err != nil { if err != nil {
requestErr = multierr.Append(requestErr, err) requestErr = multierr.Append(requestErr, err)
@ -56,12 +56,12 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp
return requestErr return requestErr
} }
// executeRaceRequest executes race condition request for a URL // executeRaceRequest executes parallel requests for a template
func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator() generator := r.newGenerator()
// Workers that keeps enqueuing new requests // Workers that keeps enqueuing new requests
maxWorkers := e.Threads maxWorkers := r.Threads
swg := sizedwaitgroup.New(maxWorkers) swg := sizedwaitgroup.New(maxWorkers)
var requestErr error var requestErr error
@ -72,30 +72,30 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out
break break
} }
if err != nil { if err != nil {
e.options.Progress.DecrementRequests(int64(generator.Total())) r.options.Progress.DecrementRequests(int64(generator.Total()))
return err return err
} }
swg.Add() swg.Add()
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
defer swg.Done() defer swg.Done()
e.options.RateLimiter.Take() r.options.RateLimiter.Take()
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock() mutex.Lock()
if err != nil { if err != nil {
requestErr = multierr.Append(requestErr, err) requestErr = multierr.Append(requestErr, err)
} }
mutex.Unlock() mutex.Unlock()
}(request) }(request)
e.options.Progress.IncrementRequests() r.options.Progress.IncrementRequests()
} }
swg.Wait() swg.Wait()
return requestErr return requestErr
} }
// executeRaceRequest executes race condition request for a URL // executeRaceRequest executes turbo http request for a URL
func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator() generator := r.newGenerator()
// need to extract the target from the url // need to extract the target from the url
URL, err := url.Parse(reqURL) URL, err := url.Parse(reqURL)
@ -106,11 +106,11 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
pipeOptions := rawhttp.DefaultPipelineOptions pipeOptions := rawhttp.DefaultPipelineOptions
pipeOptions.Host = URL.Host pipeOptions.Host = URL.Host
pipeOptions.MaxConnections = 1 pipeOptions.MaxConnections = 1
if e.PipelineConcurrentConnections > 0 { if r.PipelineConcurrentConnections > 0 {
pipeOptions.MaxConnections = e.PipelineConcurrentConnections pipeOptions.MaxConnections = r.PipelineConcurrentConnections
} }
if e.PipelineRequestsPerConnection > 0 { if r.PipelineRequestsPerConnection > 0 {
pipeOptions.MaxPendingRequests = e.PipelineRequestsPerConnection pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection
} }
pipeclient := rawhttp.NewPipelineClient(pipeOptions) pipeclient := rawhttp.NewPipelineClient(pipeOptions)
@ -130,7 +130,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
break break
} }
if err != nil { if err != nil {
e.options.Progress.DecrementRequests(int64(generator.Total())) r.options.Progress.DecrementRequests(int64(generator.Total()))
return err return err
} }
request.pipelinedClient = pipeclient request.pipelinedClient = pipeclient
@ -139,14 +139,14 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
defer swg.Done() defer swg.Done()
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback) err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock() mutex.Lock()
if err != nil { if err != nil {
requestErr = multierr.Append(requestErr, err) requestErr = multierr.Append(requestErr, err)
} }
mutex.Unlock() mutex.Unlock()
}(request) }(request)
e.options.Progress.IncrementRequests() r.options.Progress.IncrementRequests()
} }
swg.Wait() swg.Wait()
return requestErr return requestErr
@ -352,12 +352,12 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
const two = 2 const two = 2
// setCustomHeaders sets the custom headers for generated request // setCustomHeaders sets the custom headers for generated request
func (e *Request) setCustomHeaders(r *generatedRequest) { func (r *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range e.customHeaders { for k, v := range r.customHeaders {
if r.rawRequest != nil { if req.rawRequest != nil {
r.rawRequest.Headers[k] = v req.rawRequest.Headers[k] = v
} else { } else {
r.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v)) req.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
} }
} }
} }

1
v2/pkg/protocols/http/request_test.go Normal file
View File

@ -0,0 +1 @@
package http

6
v2/pkg/workflows/execute.go
View File

@ -34,7 +34,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
if len(template.Executers) == 1 { if len(template.Executers) == 1 {
mainErr = err mainErr = err
} else { } else {
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err) gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
} }
continue continue
} }
@ -76,7 +76,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
if len(template.Executers) == 1 { if len(template.Executers) == 1 {
mainErr = err mainErr = err
} else { } else {
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err) gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
} }
continue continue
} }
@ -84,7 +84,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
if len(template.Executers) == 1 { if len(template.Executers) == 1 {
mainErr = executionErr mainErr = executionErr
} else { } else {
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", executionErr) gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, executionErr)
} }
} }
} }

132
v2/pkg/workflows/execute_test.go
View File

@ -14,11 +14,10 @@ func TestWorkflowsSimple(t *testing.T) {
progress, _ := progress.NewProgress(false, false, 0) progress, _ := progress.NewProgress(false, false, 0)
workflow := &Workflow{Workflows: []*WorkflowTemplate{ workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true}}}, {Executers: []*ProtocolExecuterPair{{
}, Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progress}},
options: &protocols.ExecuterOptions{ }},
Progress: progress, }}
}}
matched, err := workflow.RunWorkflow("https://test.com") matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow") require.Nil(t, err, "could not run workflow")
@ -30,14 +29,17 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{ workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { {Executers: []*ProtocolExecuterPair{{
firstInput = input Executer: &mockExecuter{result: true, executeHook: func(input string) {
}}}}, firstInput = input
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { }}, Options: &protocols.ExecuterOptions{Progress: progress}},
secondInput = input }},
}}}}, {Executers: []*ProtocolExecuterPair{{
}, Executer: &mockExecuter{result: true, executeHook: func(input string) {
options: &protocols.ExecuterOptions{Progress: progress}} secondInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}},
}}
matched, err := workflow.RunWorkflow("https://test.com") matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow") require.Nil(t, err, "could not run workflow")
@ -52,16 +54,16 @@ func TestWorkflowsSubtemplates(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{ workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { {Executers: []*ProtocolExecuterPair{{
firstInput = input Executer: &mockExecuter{result: true, executeHook: func(input string) {
}}}, firstInput = input
Subtemplates: []*WorkflowTemplate{ }}, Options: &protocols.ExecuterOptions{Progress: progress}},
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
secondInput = input Executer: &mockExecuter{result: true, executeHook: func(input string) {
}}}, secondInput = input
}}}, }}, Options: &protocols.ExecuterOptions{Progress: progress}},
}, }}}},
options: &protocols.ExecuterOptions{Progress: progress}} }}
matched, err := workflow.RunWorkflow("https://test.com") matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow") require.Nil(t, err, "could not run workflow")
@ -76,16 +78,16 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{ workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: false, executeHook: func(input string) { {Executers: []*ProtocolExecuterPair{{
firstInput = input Executer: &mockExecuter{result: false, executeHook: func(input string) {
}}}, firstInput = input
Subtemplates: []*WorkflowTemplate{ }}, Options: &protocols.ExecuterOptions{Progress: progress}},
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
secondInput = input Executer: &mockExecuter{result: true, executeHook: func(input string) {
}}}, secondInput = input
}}}, }}, Options: &protocols.ExecuterOptions{Progress: progress}},
}, }}}},
options: &protocols.ExecuterOptions{Progress: progress}} }}
matched, err := workflow.RunWorkflow("https://test.com") matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow") require.Nil(t, err, "could not run workflow")
@ -100,24 +102,21 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{ workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { {Executers: []*ProtocolExecuterPair{{
firstInput = input Executer: &mockExecuter{result: true, executeHook: func(input string) {
}, outputs: []*output.InternalWrappedEvent{ firstInput = input
{OperatorsResult: &operators.Result{ }, outputs: []*output.InternalWrappedEvent{
Matches: map[string]struct{}{"tomcat": {}}, {OperatorsResult: &operators.Result{
Extracts: map[string][]string{}, Matches: map[string]struct{}{"tomcat": {}},
}}, Extracts: map[string][]string{},
}}}, }},
Matchers: []*Matcher{ }}, Options: &protocols.ExecuterOptions{Progress: progress}},
{Name: "tomcat", Subtemplates: []*WorkflowTemplate{ }, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input secondInput = input
}}}, }}, Options: &protocols.ExecuterOptions{Progress: progress}},
}}}, }}}}}},
}, }}
},
},
options: &protocols.ExecuterOptions{Progress: progress}}
matched, err := workflow.RunWorkflow("https://test.com") matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow") require.Nil(t, err, "could not run workflow")
@ -132,24 +131,21 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
var firstInput, secondInput string var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{ workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) { {Executers: []*ProtocolExecuterPair{{
firstInput = input Executer: &mockExecuter{result: true, executeHook: func(input string) {
}, outputs: []*output.InternalWrappedEvent{ firstInput = input
{OperatorsResult: &operators.Result{ }, outputs: []*output.InternalWrappedEvent{
Matches: map[string]struct{}{"tomcat": {}}, {OperatorsResult: &operators.Result{
Extracts: map[string][]string{}, Matches: map[string]struct{}{"tomcat": {}},
}}, Extracts: map[string][]string{},
}}},
Matchers: []*Matcher{
{Name: "apache", Subtemplates: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}}},
}}, }},
}, }}, Options: &protocols.ExecuterOptions{Progress: progress}},
}, }, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
}, Executer: &mockExecuter{result: true, executeHook: func(input string) {
options: &protocols.ExecuterOptions{Progress: progress}} secondInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}}}}}},
}}
matched, err := workflow.RunWorkflow("https://test.com") matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow") require.Nil(t, err, "could not run workflow")