External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 22:05:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added workflows flag + new templates flag to run newly added ones

Browse Source
This commit is contained in:
Ice3man543 2021-03-05 12:08:31 +05:30
parent 8cf1471e25
commit 31ce4b12cd
9 changed files with 93 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
v2/cmd/integration-test/workflow.go
View File

@ -28,7 +28,7 @@ func (h *workflowBasic) Execute(filePath string) error {
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
@ -50,7 +50,7 @@ func (h *workflowConditionMatched) Execute(filePath string) error {
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
@ -72,7 +72,7 @@ func (h *workflowConditionUnmatch) Execute(filePath string) error {
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
@ -94,7 +94,7 @@ func (h *workflowMatcherName) Execute(filePath string) error {
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}

3
v2/cmd/nuclei/main.go
View File

@ -41,6 +41,7 @@ based on templates offering massive extensibility and ease of use.`)
set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on")
set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei")
set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
@ -81,10 +82,10 @@ based on templates offering massive extensibility and ease of use.`)
set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support")
set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen")
set.BoolVarP(&options.Workflows, "workflows", "w", false, "Only run workflow templates with nuclei")
set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line")
set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
set.BoolVar(&options.NewTemplates, "new-templates", false, "Only run newly added templates")
_ = set.Parse()
if cfgFile != "" {

2
v2/internal/runner/options.go
View File

@ -82,7 +82,7 @@ func validateOptions(options *types.Options) error {
if !options.TemplateList {
// Check if a list of templates was provided and it exists
if len(options.Templates) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
if len(options.Templates) == 0 && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
return errors.New("no template/templates provided")
}
}

3
v2/internal/runner/processor.go
View File

@ -9,9 +9,6 @@ import (
// processTemplateWithList process a template on the URL list
func (r *Runner) processTemplateWithList(template *templates.Template) bool {
if r.options.Workflows {
return false
}
results := &atomic.Bool{}
wg := sizedwaitgroup.New(r.options.BulkSize)
r.hostMap.Scan(func(k, _ []byte) error {

41
v2/internal/runner/runner.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"fmt"
"os"
"path"
"strings"
"github.com/logrusorgru/aurora"
@ -82,7 +83,7 @@ func New(options *types.Options) (*Runner, error) {
os.Exit(0)
}
if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
os.Exit(0)
}
if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
@ -200,6 +201,13 @@ func (r *Runner) RunEnumeration() {
if len(r.options.Templates) == 0 && len(r.options.Tags) > 0 {
r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
}
if r.options.NewTemplates {
templates, err := r.readNewTemplatesFile()
if err != nil {
gologger.Warning().Msgf("Could not get newly added templates: %s\n", err)
}
r.options.Templates = append(r.options.Templates, templates...)
}
includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates)
excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates)
// defaults to all templates
@ -224,7 +232,10 @@ func (r *Runner) RunEnumeration() {
// pre-parse all the templates, apply filters
finalTemplates := []*templates.Template{}
availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity)
workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
var unclusteredRequests int64 = 0
for _, template := range availableTemplates {
@ -264,6 +275,9 @@ func (r *Runner) RunEnumeration() {
finalTemplates = append(finalTemplates, cluster...)
}
}
for _, workflows := range availableWorkflows {
finalTemplates = append(finalTemplates, workflows)
}
var totalRequests int64 = 0
for _, t := range finalTemplates {
@ -275,7 +289,7 @@ func (r *Runner) RunEnumeration() {
if totalRequests < unclusteredRequests {
gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
}
templateCount := originalTemplatesCount
templateCount := originalTemplatesCount + len(availableWorkflows)
// 0 matches means no templates were found in directory
if templateCount == 0 {
@ -325,3 +339,24 @@ func (r *Runner) RunEnumeration() {
r.browser.Close()
}
}
// readNewTemplatesFile reads newly added templates from directory if it exists
func (r *Runner) readNewTemplatesFile() ([]string, error) {
additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
file, err := os.Open(additionsFile)
if err != nil {
return nil, err
}
defer file.Close()
templates := []string{}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
text := scanner.Text()
if text == "" {
continue
}
templates = append(templates, text)
}
return templates, nil
}

7
v2/internal/runner/templates.go
View File

@ -14,7 +14,7 @@ import (
// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
// by severity, along with a flag indicating if workflows are present.
func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string) (parsedTemplates map[string]*templates.Template, workflowCount int) {
func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) {
filterBySeverity := len(severities) > 0
gologger.Info().Msgf("Loading templates...")
@ -26,9 +26,12 @@ func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string) (pars
gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
continue
}
if len(t.Workflows) == 0 && r.options.Workflows {
if len(t.Workflows) == 0 && workflows {
continue // don't print if user only wants to run workflows
}
if len(t.Workflows) > 0 && !workflows {
continue // don't print workflow if user only wants to run templates
}
if len(t.Workflows) > 0 {
workflowCount++
}

17
v2/internal/runner/update.go
View File

@ -7,7 +7,6 @@ import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
@ -22,6 +21,7 @@ import (
"github.com/blang/semver"
"github.com/google/go-github/v32/github"
"github.com/olekukonko/tablewriter"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
)
@ -221,6 +221,21 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU
r.printUpdateChangelog(results, version)
checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum")
err = writeTemplatesChecksum(checksumFile, results.checksums)
if err != nil {
return nil, errors.Wrap(err, "could not write checksum")
}
// Write the additions to a cached file for new runs.
additionsFile := path.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
buffer := &bytes.Buffer{}
for _, addition := range results.additions {
buffer.WriteString(addition)
buffer.WriteString("\n")
}
err = ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm)
if err != nil {
return nil, errors.Wrap(err, "could not write new additions file")
}
return results, err
}

23
v2/internal/testutils/integration.go
View File

@ -30,6 +30,29 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) (
return parts, nil
}
// RunNucleiWorkflowAndGetResults returns a list of results for a workflow
func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei", "-w", template, "-target", url)
if debug {
cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug")
cmd.Stderr = os.Stderr
}
cmd.Args = append(cmd.Args, extra...)
data, err := cmd.Output()
if err != nil {
return nil, err
}
parts := []string{}
items := strings.Split(string(data), "\n")
for _, i := range items {
if i != "" {
parts = append(parts, i)
}
}
return parts, nil
}
// TestCase is a single integration test case
type TestCase interface {
// Execute executes a test case and returns any errors if occurred

6
v2/pkg/types/types.go
View File

@ -8,6 +8,8 @@ type Options struct {
// can be specified with -l flag and -tags can be used in combination with
// the -l flag.
Tags goflags.StringSlice
// Workflows specifies any workflows to run by nuclei
Workflows goflags.StringSlice
// Templates specifies the template/templates to use
Templates goflags.StringSlice
// ExcludedTemplates specifies the template/templates to exclude
@ -65,8 +67,6 @@ type Options struct {
Headless bool
// ShowBrowser specifies whether the show the browser in headless mode
ShowBrowser bool
// Workflows specifies if only to execute workflows (no normal templates will be run)
Workflows bool
// SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack.
SystemResolvers bool
// RandomAgent generates random User-Agent
@ -107,4 +107,6 @@ type Options struct {
NoMeta bool
// Project is used to avoid sending same HTTP request multiple times
Project bool
// NewTemplates only runs newly added templates from the repository
NewTemplates bool
}