diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go new file mode 100644 index 000000000..2bdb4a969 --- /dev/null +++ b/v2/pkg/engine/engine.go @@ -0,0 +1,9 @@ +package engine + +// Engine is an engine for running Nuclei Templates/Workflows. +// +// The engine contains multiple thread pools which allow using different +// concurrency values per protocol executed. This was something which was +// missing from the previous versions of nuclei. +type Engine struct { +} diff --git a/v2/pkg/engine/engine_test.go b/v2/pkg/engine/engine_test.go new file mode 100644 index 000000000..00a22ef62 --- /dev/null +++ b/v2/pkg/engine/engine_test.go @@ -0,0 +1 @@ +package engine diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index bf4e79b18..31446c9fc 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "strings" "github.com/pkg/errors" @@ -69,11 +70,6 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplateInfo = template.Info options.TemplatePath = filePath - // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 { - return nil, fmt.Errorf("no requests defined for %s", template.ID) - } - // Compile the workflow request if len(template.Workflows) > 0 { compiled := &template.Workflow @@ -83,56 +79,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.CompiledWorkflow.Options = &options } - // Compile the requests found - requests := []protocols.Request{} - if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsDNS { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) + if err := template.compileProtocolRequests(options); err != nil { + return nil, err } - if len(template.RequestsHTTP) > 0 { - if options.Options.OfflineHTTP { - operatorsList := []*operators.Operators{} - mainLoop: - for _, req := range template.RequestsHTTP { - for _, path := range req.Path { - if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { - break mainLoop - } - } - operatorsList = append(operatorsList, &req.Operators) - } - if len(operatorsList) > 0 { - options.Operators = operatorsList - template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) - } - } else { - for _, req := range template.RequestsHTTP { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - } - if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsFile { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsNetwork { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless { - for _, req := range template.RequestsHeadless { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } if template.Executer != nil { if err := template.Executer.Compile(); err != nil { return nil, errors.Wrap(err, "could not compile request") @@ -151,14 +101,95 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute } // parseSelfContainedRequests parses the self contained template requests. -func (t *Template) parseSelfContainedRequests() { - if !t.SelfContained { +func (template *Template) parseSelfContainedRequests() { + if !template.SelfContained { return } - for _, request := range t.RequestsHTTP { + for _, request := range template.RequestsHTTP { request.SelfContained = true } - for _, request := range t.RequestsNetwork { + for _, request := range template.RequestsNetwork { request.SelfContained = true } } + +// Requests returns the total request count for the template +func (template *Template) Requests() int { + return len(template.RequestsDNS) + + len(template.RequestsHTTP) + + len(template.RequestsFile) + + len(template.RequestsNetwork) + + len(template.RequestsHeadless) + + len(template.Workflows) +} + +// compileProtocolRequests compiles all the protocol requests for the template +func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error { + templateRequests := template.Requests() + + if templateRequests == 0 { + return fmt.Errorf("no requests defined for %s", template.ID) + } + + if template.Options.Options.OfflineHTTP { + template.compileOfflineHTTPRequest(options) + return nil + } + + var requests []protocols.Request + switch { + case len(template.RequestsDNS) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + + case len(template.RequestsFile) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + + case len(template.RequestsNetwork) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + + case len(template.RequestsHTTP) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + + case len(template.RequestsHeadless) > 0 && options.Options.Headless: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + } + template.Executer = executer.NewExecuter(requests, &options) + return nil +} + +func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request { + switch reflect.TypeOf(requests).Kind() { + case reflect.Slice: + s := reflect.ValueOf(requests) + + requestSlice := make([]protocols.Request, s.Len()) + for i := 0; i < s.Len(); i++ { + value := s.Index(i) + valueInterface := value.Interface() + requestSlice[i] = valueInterface.(protocols.Request) + } + return requestSlice + } + return nil +} + +// compileOfflineHTTPRequest iterates all requests if offline http mode is +// specified and collects all matchers for all the base request templates +// (those with URL {{BaseURL}} and it's slash variation.) +func (template *Template) compileOfflineHTTPRequest(options protocols.ExecuterOptions) { + operatorsList := []*operators.Operators{} + +mainLoop: + for _, req := range template.RequestsHTTP { + for _, path := range req.Path { + if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { + break mainLoop + } + } + operatorsList = append(operatorsList, &req.Operators) + } + if len(operatorsList) > 0 { + options.Operators = operatorsList + template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) + } +}