diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d46129579..42e445775 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -24,6 +24,10 @@ var ( ) func main() { + if err := runner.ConfigureOptions(); err != nil { + gologger.Fatal().Msgf("Could not initialize options: %s\n", err) + } + readConfig() runner.ParseOptions(options) @@ -84,29 +88,29 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "templates", "Templates", - flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", []string{}, "URL containing list of templates to run"), + flagSet.FileNormalizedStringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), + flagSet.FileNormalizedStringSliceVarP(&options.TemplateURLs, "template-url", "tu", []string{}, "URL containing list of templates to run"), flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), - flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", []string{}, "URL containing list of workflows to run"), + flagSet.FileNormalizedStringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), + flagSet.FileNormalizedStringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", []string{}, "URL containing list of workflows to run"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"), ) createGroup(flagSet, "filters", "Filtering", - flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), - flagSet.NormalizedStringSliceVarP(&options.IncludeTags, "include-tags", "itags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list - flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), - flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"), - flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), + flagSet.FileNormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), + flagSet.FileNormalizedStringSliceVarP(&options.IncludeTags, "include-tags", "itags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list + flagSet.FileNormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), + flagSet.FileNormalizedStringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"), + flagSet.FileNormalizedStringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.Protocols, "type", "pt", fmt.Sprintf("protocol types to be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), flagSet.VarP(&options.ExcludeProtocols, "exclude-type", "ept", fmt.Sprintf("protocol types to not be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), - flagSet.NormalizedStringSliceVarP(&options.Authors, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), - flagSet.NormalizedStringSliceVarP(&options.IncludeIds, "template-id", "id", []string{}, "List of template IDs to run (comma-separated, file)"), - flagSet.NormalizedStringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", []string{}, "List of template IDs to exclude (comma-separated, file)"), + flagSet.FileNormalizedStringSliceVarP(&options.Authors, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), + flagSet.FileNormalizedStringSliceVarP(&options.IncludeIds, "template-id", "id", []string{}, "List of template IDs to run (comma-separated, file)"), + flagSet.FileNormalizedStringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", []string{}, "List of template IDs to exclude (comma-separated, file)"), ) createGroup(flagSet, "output", "Output", diff --git a/v2/go.mod b/v2/go.mod index 84167ad32..ca4a57c43 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -30,7 +30,7 @@ require ( github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 - github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0 + github.com/projectdiscovery/goflags v0.0.8-0.20220208065736-e1d58bce8ce5 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/interactsh v1.0.1-0.20220131074403-ca8bb8f87cd0 diff --git a/v2/go.sum b/v2/go.sum index 618f3d26f..9d2199c4c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -424,9 +424,12 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e h1:RJJuYyuwskYtzZi2gziy6SE/b7saWEzyskaA252E0VY= github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= +github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= -github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0 h1:KtCp/dCsxXNdT8m0yyWc/4ou4YaKWVakAr3G03TjQCk= -github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20220208064206-3e5adbf0b79e h1:34CL3AUFnZlV2HiraTiOq0mVo0GVMqsllhkhmYXfVcQ= +github.com/projectdiscovery/goflags v0.0.8-0.20220208064206-3e5adbf0b79e/go.mod h1:37KhVbVLllyuIAgpXGqcvE/hsFEwJ+ctEUSHawjhsBY= +github.com/projectdiscovery/goflags v0.0.8-0.20220208065736-e1d58bce8ce5 h1:IoDOKD+ZWctt0yGMwgGSCjWmSAaaMds7J9Tbxy6zv+A= +github.com/projectdiscovery/goflags v0.0.8-0.20220208065736-e1d58bce8ce5/go.mod h1:37KhVbVLllyuIAgpXGqcvE/hsFEwJ+ctEUSHawjhsBY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index b95813033..9352dd032 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -13,15 +13,22 @@ import ( "github.com/go-playground/validator/v10" "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) +func ConfigureOptions() error { + goflags.DefaultFileNormalizedStringSliceOptions.IsFromFile = func(s string) bool { + return !isTemplate(s) + } + return nil +} + // ParseOptions parses the command line flags provided by a user func ParseOptions(options *types.Options) { // Check if stdin pipe was given @@ -117,37 +124,9 @@ func validateOptions(options *types.Options) error { validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile}) } - // expand include/exclude templates id filenames - if includeIds, err := processIdsFiltering(options.IncludeIds); err != nil { - return err - } else { - options.IncludeIds = includeIds - } - if excludeIds, err := processIdsFiltering(options.ExcludeIds); err != nil { - return err - } else { - options.ExcludeIds = excludeIds - } - return nil } -func processIdsFiltering(ids []string) ([]string, error) { - var finalIds []string - for _, id := range ids { - if fileutil.FileExists(id) { - fileIds, err := utils.LoadFile(id) - if err != nil { - return nil, err - } - finalIds = append(finalIds, fileIds...) - } else { - finalIds = append(finalIds, id) - } - } - return finalIds, nil -} - // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 8302ccad9..333805119 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -456,7 +456,7 @@ func (r *Runner) readNewTemplatesFile() ([]string, error) { if text == "" { continue } - if isNewTemplate(text) { + if isTemplate(text) { templatesList = append(templatesList, text) } } @@ -483,7 +483,7 @@ func (r *Runner) countNewTemplates() int { continue } - if isNewTemplate(text) { + if isTemplate(text) { count++ } @@ -491,7 +491,7 @@ func (r *Runner) countNewTemplates() int { return count } -func isNewTemplate(filename string) bool { +func isTemplate(filename string) bool { return stringsutil.EqualFoldAny(filepath.Ext(filename), templates.TemplateExtension) } diff --git a/v2/pkg/model/types/severity/severities.go b/v2/pkg/model/types/severity/severities.go index 91ffa1ceb..9a4fccb89 100644 --- a/v2/pkg/model/types/severity/severities.go +++ b/v2/pkg/model/types/severity/severities.go @@ -12,7 +12,7 @@ import ( type Severities []Severity func (severities *Severities) Set(values string) error { - inputSeverities, err := goflags.ToNormalizedStringSlice(values) + inputSeverities, err := goflags.ToFileNormalizedStringSlice(values) if err != nil { return err } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index cc0c8b93d..9d588920e 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -123,7 +123,7 @@ func (holder TypeHolder) MarshalYAML() (interface{}, error) { type ProtocolTypes []ProtocolType func (protocolTypes *ProtocolTypes) Set(values string) error { - inputTypes, err := goflags.ToNormalizedStringSlice(values) + inputTypes, err := goflags.ToFileNormalizedStringSlice(values) if err != nil { return err } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 2be9af680..a56bdc970 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -12,21 +12,21 @@ type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths // can be specified with -l flag and -tags can be used in combination with // the -l flag. - Tags goflags.NormalizedStringSlice + Tags goflags.FileNormalizedStringSlice // ExcludeTags is the list of tags to exclude - ExcludeTags goflags.NormalizedStringSlice + ExcludeTags goflags.FileNormalizedStringSlice // Workflows specifies any workflows to run by nuclei - Workflows goflags.StringSlice + Workflows goflags.FileNormalizedStringSlice // WorkflowURLs specifies URLs to a list of workflows to use - WorkflowURLs goflags.StringSlice + WorkflowURLs goflags.FileNormalizedStringSlice // Templates specifies the template/templates to use - Templates goflags.StringSlice + Templates goflags.FileNormalizedStringSlice // TemplateURLs specifies URLs to a list of templates to use - TemplateURLs goflags.StringSlice + TemplateURLs goflags.FileNormalizedStringSlice // RemoteTemplates specifies list of allowed URLs to load remote templates from RemoteTemplateDomainList goflags.StringSlice // ExcludedTemplates specifies the template/templates to exclude - ExcludedTemplates goflags.StringSlice + ExcludedTemplates goflags.FileNormalizedStringSlice // CustomHeaders is the list of custom global headers to send with each request. CustomHeaders goflags.StringSlice // Vars is the list of custom global vars @@ -38,19 +38,19 @@ type Options struct { // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities // Authors filters templates based on their author and only run the matching ones. - Authors goflags.NormalizedStringSlice + Authors goflags.FileNormalizedStringSlice // Protocols contains the protocols to be allowed executed Protocols types.ProtocolTypes // ExcludeProtocols contains protocols to not be executed ExcludeProtocols types.ProtocolTypes // IncludeTags includes specified tags to be run even while being in denylist - IncludeTags goflags.NormalizedStringSlice + IncludeTags goflags.FileNormalizedStringSlice // IncludeTemplates includes specified templates to be run even while being in denylist - IncludeTemplates goflags.StringSlice + IncludeTemplates goflags.FileNormalizedStringSlice // IncludeIds includes specified ids to be run even while being in denylist - IncludeIds goflags.NormalizedStringSlice + IncludeIds goflags.FileNormalizedStringSlice // ExcludeIds contains templates ids to not be executed - ExcludeIds goflags.NormalizedStringSlice + ExcludeIds goflags.FileNormalizedStringSlice InternalResolversList []string // normalized from resolvers flag as well as file provided. // ProjectPath allows nuclei to use a user defined project folder diff --git a/v2/pkg/utils/utils.go b/v2/pkg/utils/utils.go index e7a1413f5..f5d30b2f3 100644 --- a/v2/pkg/utils/utils.go +++ b/v2/pkg/utils/utils.go @@ -7,8 +7,6 @@ import ( "net/url" "os" "strings" - - "github.com/projectdiscovery/fileutil" ) func IsBlank(value string) bool { @@ -30,18 +28,6 @@ func UnwrapError(err error) error { return err } -func LoadFile(filename string) ([]string, error) { - var items []string - readfileChan, err := fileutil.ReadFile(filename) - if err != nil { - return nil, err - } - for includeIdLine := range readfileChan { - items = append(items, includeIdLine) - } - return items, nil -} - // IsURL tests a string to determine if it is a well-structured url or not. func IsURL(input string) bool { _, err := url.ParseRequestURI(input)