diff --git a/integration_tests/workflow/basic.yaml b/integration_tests/workflow/basic.yaml index efbeefa47..f49193b18 100644 --- a/integration_tests/workflow/basic.yaml +++ b/integration_tests/workflow/basic.yaml @@ -6,5 +6,5 @@ info: severity: info workflows: - - template: match-1.yaml - - template: match-2.yaml \ No newline at end of file + - template: workflow/match-1.yaml + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/workflow/condition-matched.yaml b/integration_tests/workflow/condition-matched.yaml new file mode 100644 index 000000000..8b0a65732 --- /dev/null +++ b/integration_tests/workflow/condition-matched.yaml @@ -0,0 +1,11 @@ +id: condition-matched-workflow + +info: + name: Condition Matched Workflow + author: pdteam + severity: info + +workflows: + - template: workflow/match-1.yaml + subtemplates: + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/workflow/condition-unmatched.yaml b/integration_tests/workflow/condition-unmatched.yaml new file mode 100644 index 000000000..dec9e1a7d --- /dev/null +++ b/integration_tests/workflow/condition-unmatched.yaml @@ -0,0 +1,11 @@ +id: condition-unmatched-workflow + +info: + name: Condition UnMatched Workflow + author: pdteam + severity: info + +workflows: + - template: workflow/nomatch-1.yaml + subtemplates: + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/workflow/match-1.yaml b/integration_tests/workflow/match-1.yaml new file mode 100644 index 000000000..c7e07e8cf --- /dev/null +++ b/integration_tests/workflow/match-1.yaml @@ -0,0 +1,15 @@ +id: basic-get + +info: + name: Basic GET Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "This is test matcher text" \ No newline at end of file diff --git a/integration_tests/workflow/match-2.yaml b/integration_tests/workflow/match-2.yaml new file mode 100644 index 000000000..01e9d4b6f --- /dev/null +++ b/integration_tests/workflow/match-2.yaml @@ -0,0 +1,16 @@ +id: basic-get-another + +info: + name: Basic Another GET Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + name: test-matcher + words: + - "This is test matcher text" \ No newline at end of file diff --git a/integration_tests/workflow/matcher-name.yaml b/integration_tests/workflow/matcher-name.yaml new file mode 100644 index 000000000..f09ddd6ba --- /dev/null +++ b/integration_tests/workflow/matcher-name.yaml @@ -0,0 +1,13 @@ +id: matcher-name-workflow + +info: + name: Matcher Name Workflow + author: pdteam + severity: info + +workflows: + - template: workflow/match-2.yaml + matchers: + - name: test-matcher + subtemplates: + - template: workflow/match-1.yaml \ No newline at end of file diff --git a/integration_tests/workflow/nomatch-1.yaml b/integration_tests/workflow/nomatch-1.yaml new file mode 100644 index 000000000..71f126421 --- /dev/null +++ b/integration_tests/workflow/nomatch-1.yaml @@ -0,0 +1,15 @@ +id: basic-get-nomatch + +info: + name: Basic GET Request NoMatch + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "Random" \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 2ce62c274..6b9e161c9 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -20,9 +20,10 @@ func main() { failed := aurora.Red("[✘]").String() protocolTests := map[string]map[string]testutils.TestCase{ - "http": httpTestcases, - "network": networkTestcases, - "dns": dnsTestCases, + "http": httpTestcases, + "network": networkTestcases, + "dns": dnsTestCases, + "workflow": workflowTestcases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go new file mode 100644 index 000000000..f260ceb1e --- /dev/null +++ b/v2/cmd/integration-test/workflow.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" +) + +var workflowTestcases = map[string]testutils.TestCase{ + "workflow/basic.yaml": &workflowBasic{}, + "workflow/condition-matched.yaml": &workflowConditionMatched{}, + "workflow/condition-unmatched.yaml": &workflowConditionUnmatch{}, + "workflow/matcher-name.yaml": &workflowMatcherName{}, +} + +type workflowBasic struct{} + +// Executes executes a test case and returns an error if occurred +func (h *workflowBasic) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + httpDebugRequestDump(r) + fmt.Fprintf(w, "This is test matcher text") + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 2 { + return errIncorrectResultsCount(results) + } + return nil +} + +type workflowConditionMatched struct{} + +// Executes executes a test case and returns an error if occurred +func (h *workflowConditionMatched) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + httpDebugRequestDump(r) + fmt.Fprintf(w, "This is test matcher text") + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type workflowConditionUnmatch struct{} + +// Executes executes a test case and returns an error if occurred +func (h *workflowConditionUnmatch) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + httpDebugRequestDump(r) + fmt.Fprintf(w, "This is test matcher text") + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 0 { + return errIncorrectResultsCount(results) + } + return nil +} + +type workflowMatcherName struct{} + +// Executes executes a test case and returns an error if occurred +func (h *workflowMatcherName) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + httpDebugRequestDump(r) + fmt.Fprintf(w, "This is test matcher text") + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/go.sum b/v2/go.sum index db89ed9fc..4d6251fa2 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -551,6 +551,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 7e2d9195e..d98782a6b 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -71,7 +71,7 @@ func (r *Runner) updateTemplates() error { } gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) - err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) + _, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) if err != nil { return err } @@ -128,7 +128,7 @@ func (r *Runner) updateTemplates() error { r.templatesConfig.CurrentVersion = version.String() gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) - err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) + _, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) if err != nil { return err } @@ -181,48 +181,71 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor } // downloadReleaseAndUnzip downloads and unzips the release in a directory -func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) error { +func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { - return fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err) + return nil, fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err) } res, err := http.DefaultClient.Do(req) if err != nil { - return fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err) + return nil, fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { - return fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode) + return nil, fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode) } buf, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("failed to create buffer for zip file: %s", err) + return nil, fmt.Errorf("failed to create buffer for zip file: %s", err) } reader := bytes.NewReader(buf) z, err := zip.NewReader(reader, reader.Size()) if err != nil { - return fmt.Errorf("failed to uncompress zip file: %s", err) + return nil, fmt.Errorf("failed to uncompress zip file: %s", err) } // Create the template folder if it doesn't exists err = os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm) if err != nil { - return fmt.Errorf("failed to create template base folder: %s", err) + return nil, fmt.Errorf("failed to create template base folder: %s", err) + } + + results, err := r.compareAndWriteTemplates(z) + if err != nil { + return nil, fmt.Errorf("failed to write templates: %s", err) + } + + r.printUpdateChangelog(results, version) + checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum") + err = writeTemplatesChecksum(checksumFile, results.checksums) + return results, err +} + +type templateUpdateResults struct { + additions []string + deletions []string + modifications []string + totalCount int + checksums map[string]string +} + +// compareAndWriteTemplates compares and returns the stats of a template +// update operations. +func (r *Runner) compareAndWriteTemplates(z *zip.Reader) (*templateUpdateResults, error) { + results := &templateUpdateResults{ + checksums: make(map[string]string), } - totalCount := 0 - additions, deletions, modifications := []string{}, []string{}, []string{} // We use file-checksums that are md5 hashes to store the list of files->hashes // that have been downloaded previously. // If the path isn't found in new update after being read from the previous checksum, // it is removed. This allows us fine-grained control over the download process // as well as solves a long problem with nuclei-template updates. checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum") - previousChecksum := readPreviousTemplatesChecksum(checksumFile) - checksums := make(map[string]string) + previousChecksum, _ := readPreviousTemplatesChecksum(checksumFile) for _, file := range z.File { directory, name := filepath.Split(file.Name) if name == "" { @@ -234,11 +257,11 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU if (!strings.EqualFold(name, ".nuclei-ignore") && strings.HasPrefix(name, ".")) || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { continue } - totalCount++ + results.totalCount++ templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath) - err = os.MkdirAll(templateDirectory, os.ModePerm) + err := os.MkdirAll(templateDirectory, os.ModePerm) if err != nil { - return fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) + return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) } templatePath := path.Join(templateDirectory, name) @@ -250,13 +273,13 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) if err != nil { f.Close() - return fmt.Errorf("could not create uncompressed file: %s", err) + return nil, fmt.Errorf("could not create uncompressed file: %s", err) } reader, err := file.Open() if err != nil { f.Close() - return fmt.Errorf("could not open archive to extract file: %s", err) + return nil, fmt.Errorf("could not open archive to extract file: %s", err) } hasher := md5.New() @@ -264,40 +287,41 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU _, err = io.Copy(f, io.TeeReader(reader, hasher)) if err != nil { f.Close() - return fmt.Errorf("could not write template file: %s", err) + return nil, fmt.Errorf("could not write template file: %s", err) } f.Close() + oldChecksum, checksumOK := previousChecksum[templatePath] + + checksum := hex.EncodeToString(hasher.Sum(nil)) if isAddition { - additions = append(additions, path.Join(finalPath, name)) - } else { - modifications = append(modifications, path.Join(finalPath, name)) + results.additions = append(results.additions, path.Join(finalPath, name)) + } else if checksumOK && oldChecksum[0] != checksum { + results.modifications = append(results.modifications, path.Join(finalPath, name)) } - checksums[templatePath] = hex.EncodeToString(hasher.Sum(nil)) + results.checksums[templatePath] = checksum } // If we don't find a previous file in new download and it hasn't been // changed on the disk, delete it. for k, v := range previousChecksum { - _, ok := checksums[k] + _, ok := results.checksums[k] if !ok && v[0] == v[1] { os.Remove(k) - deletions = append(deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/")) + results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/")) } } - - r.printUpdateChangelog(additions, modifications, deletions, version, totalCount) - return writeTemplatesChecksum(checksumFile, checksums) + return results, nil } // readPreviousTemplatesChecksum reads the previous checksum file from the disk. // // It reads two checksums, the first checksum is what we expect and the second is // the actual checksum of the file on disk currently. -func readPreviousTemplatesChecksum(file string) map[string][2]string { +func readPreviousTemplatesChecksum(file string) (map[string][2]string, error) { f, err := os.Open(file) if err != nil { - return nil + return nil, err } defer f.Close() scanner := bufio.NewScanner(f) @@ -316,20 +340,19 @@ func readPreviousTemplatesChecksum(file string) map[string][2]string { f, err := os.Open(parts[0]) if err != nil { - continue + return nil, err } hasher := md5.New() if _, err := io.Copy(hasher, f); err != nil { - f.Close() - continue + return nil, err } f.Close() values[1] = hex.EncodeToString(hasher.Sum(nil)) checksum[parts[0]] = values } - return checksum + return checksum, nil } // writeTemplatesChecksum writes the nuclei-templates checksum data to disk. @@ -340,44 +363,36 @@ func writeTemplatesChecksum(file string, checksum map[string]string) error { } defer f.Close() + builder := &strings.Builder{} for k, v := range checksum { - _, _ = f.WriteString(k) - _, _ = f.WriteString(",") - _, _ = f.WriteString(v) - _, _ = f.WriteString("\n") + builder.WriteString(k) + builder.WriteString(",") + builder.WriteString(v) + builder.WriteString("\n") + + if _, checksumErr := f.WriteString(builder.String()); checksumErr != nil { + return err + } + builder.Reset() } return nil } -func (r *Runner) printUpdateChangelog(additions, modifications, deletions []string, version string, totalCount int) { - if len(additions) > 0 { - gologger.Print().Msgf("\nNew additions: \n\n") +func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) { + if len(results.additions) > 0 { + gologger.Print().Msgf("\nNewly added templates: \n\n") - for _, addition := range additions { + for _, addition := range results.additions { gologger.Print().Msgf("%s", addition) } } - if len(modifications) > 0 { - gologger.Print().Msgf("\nModifications: \n\n") - - for _, modification := range modifications { - gologger.Print().Msgf("%s", modification) - } - } - if len(deletions) > 0 { - gologger.Print().Msgf("\nDeletions: \n\n") - - for _, deletion := range deletions { - gologger.Print().Msgf("%s", deletion) - } - } gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version) data := [][]string{ - {strconv.Itoa(totalCount), strconv.Itoa(len(additions)), strconv.Itoa(len(modifications)), strconv.Itoa(len(deletions))}, + {strconv.Itoa(results.totalCount), strconv.Itoa(len(results.additions)), strconv.Itoa(len(results.deletions))}, } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Total", "New", "Modifications", "Deletions"}) + table.SetHeader([]string{"Total", "Added", "Removed"}) for _, v := range data { table.Append(v) } diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go new file mode 100644 index 000000000..8ee4e9478 --- /dev/null +++ b/v2/internal/runner/update_test.go @@ -0,0 +1,156 @@ +package runner + +import ( + "archive/zip" + "context" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/stretchr/testify/require" +) + +func TestDownloadReleaseAndUnzipAddition(t *testing.T) { + gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) + + baseTemplates, err := ioutil.TempDir("", "old-temp-*") + require.Nil(t, err, "could not create temp directory") + defer os.RemoveAll(baseTemplates) + + err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777) + require.Nil(t, err, "could not create write base file") + + err = zipFromDirectory("base.zip", baseTemplates) + require.Nil(t, err, "could not create zip from directory") + defer os.Remove("base.zip") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "base.zip") + })) + defer ts.Close() + + templatesDirectory, err := ioutil.TempDir("", "template-*") + require.Nil(t, err, "could not create temp directory") + defer os.RemoveAll(templatesDirectory) + + r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}} + + results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL) + require.Nil(t, err, "could not download release and unzip") + require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition") + + newTempDir, err := ioutil.TempDir("", "new-tmp-*") + require.Nil(t, err, "could not create temp directory") + defer os.RemoveAll(newTempDir) + + err = ioutil.WriteFile(path.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777) + require.Nil(t, err, "could not create base file") + err = ioutil.WriteFile(path.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777) + require.Nil(t, err, "could not create new file") + + err = zipFromDirectory("new.zip", newTempDir) + require.Nil(t, err, "could not create new zip from directory") + defer os.Remove("new.zip") + + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "new.zip") + })) + defer ts2.Close() + + results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL) + require.Nil(t, err, "could not download release and unzip") + + require.Equal(t, "new.yaml", results.additions[0], "could not get correct new addition") +} + +func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { + gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) + + baseTemplates, err := ioutil.TempDir("", "old-temp-*") + require.Nil(t, err, "could not create temp directory") + defer os.RemoveAll(baseTemplates) + + err = ioutil.WriteFile(path.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777) + require.Nil(t, err, "could not create write base file") + + err = zipFromDirectory("base.zip", baseTemplates) + require.Nil(t, err, "could not create zip from directory") + defer os.Remove("base.zip") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "base.zip") + })) + defer ts.Close() + + templatesDirectory, err := ioutil.TempDir("", "template-*") + require.Nil(t, err, "could not create temp directory") + defer os.RemoveAll(templatesDirectory) + + r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}} + + results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL) + require.Nil(t, err, "could not download release and unzip") + require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition") + + newTempDir, err := ioutil.TempDir("", "new-tmp-*") + require.Nil(t, err, "could not create temp directory") + defer os.RemoveAll(newTempDir) + + err = zipFromDirectory("new.zip", newTempDir) + require.Nil(t, err, "could not create new zip from directory") + defer os.Remove("new.zip") + + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "new.zip") + })) + defer ts2.Close() + + results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL) + require.Nil(t, err, "could not download release and unzip") + + require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions") +} + +func zipFromDirectory(zipPath, directory string) error { + file, err := os.Create(zipPath) + if err != nil { + return err + } + defer file.Close() + + w := zip.NewWriter(file) + defer w.Close() + + walker := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + f, err := w.Create(strings.TrimPrefix(path, directory)) + if err != nil { + return err + } + _, err = io.Copy(f, file) + if err != nil { + return err + } + return nil + } + return filepath.Walk(directory, walker) +} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 8569c491e..01b2d6097 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -2,6 +2,7 @@ package testutils import ( "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -118,3 +119,9 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco } return executerOpts } + +// NoopWriter is a NooP gologger writer. +type NoopWriter struct{} + +// Write writes the data to an output writer. +func (n *NoopWriter) Write(data []byte, level levels.Level) {} diff --git a/v2/pkg/catalog/ignore_test.go b/v2/pkg/catalog/ignore_test.go index cb2fb37b1..362ef7f65 100644 --- a/v2/pkg/catalog/ignore_test.go +++ b/v2/pkg/catalog/ignore_test.go @@ -5,18 +5,12 @@ import ( "testing" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/stretchr/testify/require" ) -type noopWriter struct{} - -// Write writes the data to an output writer. -func (n *noopWriter) Write(data []byte, level levels.Level) {} - func TestIgnoreFilesIgnore(t *testing.T) { - writer := &noopWriter{} - gologger.DefaultLogger.SetWriter(writer) + gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) c := &Catalog{ ignoreFiles: []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}, diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 70d1edcc4..3356875bb 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -30,13 +30,26 @@ func (w *Workflow) RunWorkflow(input string) bool { // in a recursive manner running all subtemplates and matchers. func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup) error { var firstMatched bool - + var err error var mainErr error + if len(template.Matchers) == 0 { for _, executer := range template.Executers { executer.Options.Progress.AddToTotal(int64(executer.Executer.Requests())) - matched, err := executer.Executer.Execute(input) + // Don't print results with subtemplates, only print results on template. + if len(template.Subtemplates) > 0 { + err = executer.Executer.ExecuteWithResults(input, func(result *output.InternalWrappedEvent) { + if result.OperatorsResult == nil { + return + } + if len(result.Results) > 0 { + firstMatched = true + } + }) + } else { + firstMatched, err = executer.Executer.Execute(input) + } if err != nil { if len(template.Executers) == 1 { mainErr = err @@ -45,13 +58,11 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } continue } - if matched { - firstMatched = matched - results.CAS(false, matched) - } } } - + if len(template.Subtemplates) == 0 { + results.CAS(false, firstMatched) + } if len(template.Matchers) > 0 { for _, executer := range template.Executers { executer.Options.Progress.AddToTotal(int64(executer.Executer.Requests())) @@ -95,7 +106,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for _, subtemplate := range template.Subtemplates { swg.Add() - func(template *WorkflowTemplate) { + go func(template *WorkflowTemplate) { err := w.runWorkflowStep(template, input, results, swg) if err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 37e7953bb..966d98e3b 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -56,6 +56,8 @@ func TestWorkflowsSubtemplates(t *testing.T) { {Executers: []*ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input + }, outputs: []*output.InternalWrappedEvent{ + {OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) {