From 14bb1b7b21a48d5bf012a3225367c252016cca03 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Thu, 14 Oct 2021 23:30:37 +0200 Subject: [PATCH] Implement `-template-url` and `-workflow-url` for retrieving lists of templates/workflows to run. --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/runner.go | 2 + v2/pkg/catalog/loader/loader.go | 22 ++++-- v2/pkg/catalog/loader/remote_loader.go | 98 ++++++++++++++++++++++++++ v2/pkg/types/types.go | 4 ++ 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 v2/pkg/catalog/loader/remote_loader.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 83b13fc6b..81e701fc6 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -56,7 +56,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"), flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index e87903f14..06abc70ae 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -337,7 +337,9 @@ func (r *Runner) RunEnumeration() error { loaderConfig := loader.Config{ Templates: r.options.Templates, + TemplateURLs: r.options.TemplateURLs, Workflows: r.options.Workflows, + WorkflowURLs: r.options.WorkflowURLs, ExcludeTemplates: r.options.ExcludedTemplates, Tags: r.options.Tags, ExcludeTags: r.options.ExcludeTags, diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4ae..6d914ae5c 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -15,7 +15,9 @@ import ( // Config contains the configuration options for the loader type Config struct { Templates []string + TemplateURLs []string Workflows []string + WorkflowURLs []string ExcludeTemplates []string IncludeTemplates []string @@ -37,6 +39,7 @@ type Store struct { pathFilter *filter.PathFilter config *Config finalTemplates []string + finalWorkflows []string templates []*templates.Template workflows []*templates.Template @@ -61,13 +64,24 @@ func New(config *Config) (*Store, error) { IncludedTemplates: config.IncludeTemplates, ExcludedTemplates: config.ExcludeTemplates, }, config.Catalog), + finalTemplates: config.Templates, + finalWorkflows: config.Workflows, + } + + if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 { + remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) + if err != nil { + return store, err + } + store.finalTemplates = append(store.finalTemplates, remoteTemplates...) + store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...) } // Handle a case with no templates or workflows, where we use base directory - if len(config.Templates) == 0 && len(config.Workflows) == 0 { - config.Templates = append(config.Templates, config.TemplatesDirectory) + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 { + store.finalTemplates = []string{config.TemplatesDirectory} } - store.finalTemplates = append(store.finalTemplates, config.Templates...) + return store, nil } @@ -90,7 +104,7 @@ func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) { // the complete compiled templates for a nuclei execution configuration. func (store *Store) Load() { store.templates = store.LoadTemplates(store.finalTemplates) - store.workflows = store.LoadWorkflows(store.config.Workflows) + store.workflows = store.LoadWorkflows(store.finalWorkflows) } // ValidateTemplates takes a list of templates and validates them diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go new file mode 100644 index 000000000..ec47ed51b --- /dev/null +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -0,0 +1,98 @@ +package loader + +import ( + "bufio" + "fmt" + "github.com/pkg/errors" + "net/http" + "strings" +) + +type ContentType string + +const ( + Template ContentType = "Template" + Workflow ContentType = "Workflow" +) + +type RemoteContentError struct { + Content []string + Type ContentType + Error error +} + +func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) { + remoteContentErrorChannel := make(chan RemoteContentError) + + for _, templateURL := range templateURLs { + go getRemoteContent(templateURL, remoteContentErrorChannel, Template) + } + for _, workflowURL := range workflowURLs { + go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow) + } + + var remoteTemplateList []string + var remoteWorkFlowList []string + var err error + for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ { + remoteContentError := <-remoteContentErrorChannel + if remoteContentError.Error != nil { + if err != nil { + err = errors.New(remoteContentError.Error.Error() + ": " + err.Error()) + } else { + err = remoteContentError.Error + } + } else { + if remoteContentError.Type == Template { + remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...) + } else if remoteContentError.Type == Workflow { + remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...) + } + } + } + + return remoteTemplateList, remoteWorkFlowList, err +} + +func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) { + response, err := http.Get(URL) + if err != nil { + w <- RemoteContentError{ + Error: err, + } + return + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode > 299 { + w <- RemoteContentError{ + Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode), + } + return + } + + if err != nil { + + } + + scanner := bufio.NewScanner(response.Body) + var templateList []string + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + templateList = append(templateList, text) + } + + if err := scanner.Err(); err != nil { + w <- RemoteContentError{ + Error: errors.Wrap(err, "get \"%s\""), + } + return + } + + w <- RemoteContentError{ + Content: templateList, + Type: contentType, + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dd81c3e4a..b5762a861 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -15,8 +15,12 @@ type Options struct { ExcludeTags goflags.NormalizedStringSlice // Workflows specifies any workflows to run by nuclei Workflows goflags.StringSlice + // WorkflowURLs specifies URLs to a list of workflows to use + WorkflowURLs goflags.StringSlice // Templates specifies the template/templates to use Templates goflags.StringSlice + // TemplateURLs specifies URLs to a list of templates to use + TemplateURLs goflags.StringSlice // ExcludedTemplates specifies the template/templates to exclude ExcludedTemplates goflags.StringSlice // CustomHeaders is the list of custom global headers to send with each request.