diff --git a/integration_tests/http/save-extractor-values-to-file.yaml b/integration_tests/http/save-extractor-values-to-file.yaml new file mode 100644 index 000000000..64bf63f40 --- /dev/null +++ b/integration_tests/http/save-extractor-values-to-file.yaml @@ -0,0 +1,18 @@ +id: save-extractor-values-to-file + +info: + name: save extractor values to file + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + + extractors: + - type: regex + part: body + regex: + - '[0-9]+' + to: output.txt \ No newline at end of file diff --git a/integration_tests/http/self-contained-with-params.yaml b/integration_tests/http/self-contained-with-params.yaml new file mode 100644 index 000000000..73665c772 --- /dev/null +++ b/integration_tests/http/self-contained-with-params.yaml @@ -0,0 +1,18 @@ +id: self-contained-with-params + +info: + name: self contained with params + author: pd-team + severity: info + +self-contained: true +requests: + - raw: + - | + GET http://127.0.0.1:5431/?something=here&key=value HTTP/1.1 + Host: {{Hostname}} + + matchers: + - type: word + words: + - This is self-contained response \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 6a1997b1f..21e718a28 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "errors" "fmt" @@ -19,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" logutil "github.com/projectdiscovery/utils/log" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" @@ -48,6 +50,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, "http/self-contained.yaml": &httpRequestSelfContained{}, + "http/self-contained-with-params.yaml": &httpRequestSelfContainedWithParams{}, "http/self-contained-file-input.yaml": &httpRequestSelfContainedFileInput{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, @@ -69,6 +72,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-without-scheme.yaml": &httpGetWithoutScheme{}, "http/cl-body-without-header.yaml": &httpCLBodyWithoutHeader{}, "http/cl-body-with-header.yaml": &httpCLBodyWithHeader{}, + "http/save-extractor-values-to-file.yaml": &httpSaveExtractorValuesToFile{}, } type httpInteractshRequest struct{} @@ -855,6 +859,42 @@ func (h *httpRequestSelfContained) Execute(filePath string) error { return expectResultsCount(results, 1) } +// testcase to check duplicated values in params +type httpRequestSelfContainedWithParams struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error { + router := httprouter.New() + var err error + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + params := r.URL.Query() + // we intentionally use params["test"] instead of params.Get("test") to test the case where + // there are multiple parameters with the same name + if !reflect.DeepEqual(params["something"], []string{"here"}) { + err = errorutil.WrapfWithNil(err, "expected %v, got %v", []string{"here"}, params["something"]) + } + if !reflect.DeepEqual(params["key"], []string{"value"}) { + err = errorutil.WrapfWithNil(err, "expected %v, got %v", []string{"key"}, params["value"]) + } + _, _ = w.Write([]byte("This is self-contained response")) + }) + server := &http.Server{ + Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), + Handler: router, + } + go func() { + _ = server.ListenAndServe() + }() + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + if err != nil { + return err + } + + return expectResultsCount(results, 1) +} + type httpRequestSelfContainedFileInput struct{} func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error { @@ -1262,3 +1302,31 @@ func (h *httpCLBodyWithHeader) Execute(filePath string) error { } return expectResultsCount(got, 1) } + +type httpSaveExtractorValuesToFile struct{} + +func (h *httpSaveExtractorValuesToFile) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + var buff bytes.Buffer + for i := 0; i < 10; i++ { + buff.WriteString(fmt.Sprintf(`"value": %v`+"\n", i)) + } + _, _ = w.Write(buff.Bytes()) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + + // remove output.txt file if exists + if !fileutil.FileExists("output.txt") { + return fmt.Errorf("extractor output file output.txt file does not exist") + } else { + _ = os.Remove("output.txt") + } + return expectResultsCount(results, 1) +} diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index d5c948ef2..5dd827bd3 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -3,9 +3,10 @@ package extractors import ( "encoding/json" "fmt" + "strings" + "github.com/antchfx/htmlquery" "github.com/antchfx/xmlquery" - "strings" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -29,6 +30,7 @@ func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { } } } + e.SaveToFile(results) return results } @@ -56,6 +58,7 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} results[itemString] = struct{}{} } } + e.SaveToFile(results) return results } @@ -93,6 +96,7 @@ func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} { } } } + e.SaveToFile(results) return results } @@ -123,6 +127,7 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} { } } } + e.SaveToFile(results) return results } @@ -159,6 +164,7 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { } } } + e.SaveToFile(results) return results } @@ -179,6 +185,6 @@ func (e *Extractor) ExtractDSL(data map[string]interface{}) map[string]struct{} } } } - + e.SaveToFile(results) return results } diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 7071a0bda..f531c0131 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -1,10 +1,14 @@ package extractors import ( + "os" + "path/filepath" "regexp" "github.com/Knetic/govaluate" "github.com/itchyny/gojq" + "github.com/projectdiscovery/gologger" + fileutil "github.com/projectdiscovery/utils/file" ) // Extractor is used to extract part of response using a regex. @@ -113,4 +117,36 @@ type Extractor struct { // - false // - true CaseInsensitive bool `yaml:"case-insensitive,omitempty" json:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` + // description: | + // ToFile (to) saves extracted requests to file and if file is present values are appended to file. + ToFile string `yaml:"to,omitempty" json:"to,omitempty" jsonschema:"title=save extracted values to file,description=save extracted values to file"` +} + +// SaveToFile saves extracted values to file if `to` is present and valid +func (e *Extractor) SaveToFile(data map[string]struct{}) { + if e.ToFile == "" { + return + } + + if !fileutil.FileExists(e.ToFile) { + baseDir := filepath.Dir(e.ToFile) + if baseDir != "." && !fileutil.FolderExists(baseDir) { + if err := fileutil.CreateFolder(baseDir); err != nil { + gologger.Error().Msgf("extractor: could not create folder %s: %s\n", baseDir, err) + return + } + } + } + file, err := os.OpenFile(e.ToFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + gologger.Error().Msgf("extractor: could not open file %s: %s\n", e.ToFile, err) + return + } + defer file.Close() + for k := range data { + if _, err = file.WriteString(k + "\n"); err != nil { + gologger.Error().Msgf("extractor: could not write to file %s: %s\n", e.ToFile, err) + return + } + } } diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 8f33ce6e0..6ef593329 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -170,6 +170,11 @@ func (p *StatsTicker) makePrintCallback() func(stats clistats.StatisticsClient) requests, okRequests := stats.GetCounter("requests") total, okTotal := stats.GetCounter("total") + // If input is not given, total is 0 which cause percentage overflow + if total == 0 { + total = requests + } + if okRequests && okTotal && duration > 0 && !p.cloud { builder.WriteString(" | RPS: ") builder.WriteString(clistats.String(uint64(float64(requests) / duration.Seconds()))) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index d1e064283..74ffd4a8d 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -255,6 +255,11 @@ func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urluti // finalVars = contains all variables including generator and protocol specific variables // generatorValues = contains variables used in fuzzing or other generator specific values func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest string, baseURL *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) { + // Unlike other requests parsedURL/ InputURL in self contained templates is extracted from raw request itself h + // and variables are supposed to be given from command line and not from inputURL + // ence this cause issues like duplicated params/paths. + // TODO: implement a generic raw request parser in rawhttp library (without variables and stuff) + baseURL.Params = nil // this fixes issue of duplicated params in self contained templates but not a appropriate fix rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe) if err != nil { return nil, err