duplicated params in self contained requests (#3608)

* fix duplicated params in self-contained+ export extracted values to file

* add integration tests + fix percentage overflow in pb

* fix integration test template id

* integration test: validate if file exists

integration_tests/http/save-extractor-values-to-file.yaml

@@ -0,0 +1,18 @@
+id: save-extractor-values-to-file
+
+info:
+  name: save extractor values to file
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    extractors:
+      - type: regex
+        part: body
+        regex:
+          - '[0-9]+'
+        to: output.txt

integration_tests/http/self-contained-with-params.yaml

@@ -0,0 +1,18 @@
+id: self-contained-with-params
+
+info:
+  name: self contained with params
+  author: pd-team
+  severity: info
+
+self-contained: true
+requests:
+  - raw:
+      - |
+        GET http://127.0.0.1:5431/?something=here&key=value HTTP/1.1
+        Host: {{Hostname}}
+
+    matchers:
+      - type: word
+        words:
+          - This is self-contained response

v2/cmd/integration-test/http.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -19,6 +20,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
 	"github.com/projectdiscovery/retryablehttp-go"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	fileutil "github.com/projectdiscovery/utils/file"
 	logutil "github.com/projectdiscovery/utils/log"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
@@ -48,6 +50,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/request-condition.yaml":                   &httpRequestCondition{},
 	"http/request-condition-new.yaml":               &httpRequestCondition{},
 	"http/self-contained.yaml":                      &httpRequestSelfContained{},
+	"http/self-contained-with-params.yaml":          &httpRequestSelfContainedWithParams{},
 	"http/self-contained-file-input.yaml":           &httpRequestSelfContainedFileInput{},
 	"http/get-case-insensitive.yaml":                &httpGetCaseInsensitive{},
 	"http/get.yaml,http/get-case-insensitive.yaml":  &httpGetCaseInsensitiveCluster{},
@@ -69,6 +72,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/get-without-scheme.yaml":                  &httpGetWithoutScheme{},
 	"http/cl-body-without-header.yaml":              &httpCLBodyWithoutHeader{},
 	"http/cl-body-with-header.yaml":                 &httpCLBodyWithHeader{},
+	"http/save-extractor-values-to-file.yaml":       &httpSaveExtractorValuesToFile{},
 }
 
 type httpInteractshRequest struct{}
@@ -855,6 +859,42 @@ func (h *httpRequestSelfContained) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+// testcase to check duplicated values in params
+type httpRequestSelfContainedWithParams struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error {
+	router := httprouter.New()
+	var err error
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		params := r.URL.Query()
+		// we intentionally use params["test"] instead of params.Get("test") to test the case where
+		// there are multiple parameters with the same name
+		if !reflect.DeepEqual(params["something"], []string{"here"}) {
+			err = errorutil.WrapfWithNil(err, "expected %v, got %v", []string{"here"}, params["something"])
+		}
+		if !reflect.DeepEqual(params["key"], []string{"value"}) {
+			err = errorutil.WrapfWithNil(err, "expected %v, got %v", []string{"key"}, params["value"])
+		}
+		_, _ = w.Write([]byte("This is self-contained response"))
+	})
+	server := &http.Server{
+		Addr:    fmt.Sprintf("localhost:%d", defaultStaticPort),
+		Handler: router,
+	}
+	go func() {
+		_ = server.ListenAndServe()
+	}()
+	defer server.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
 type httpRequestSelfContainedFileInput struct{}
 
 func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error {
@@ -1262,3 +1302,31 @@ func (h *httpCLBodyWithHeader) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 1)
 }
+
+type httpSaveExtractorValuesToFile struct{}
+
+func (h *httpSaveExtractorValuesToFile) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		var buff bytes.Buffer
+		for i := 0; i < 10; i++ {
+			buff.WriteString(fmt.Sprintf(`"value": %v`+"\n", i))
+		}
+		_, _ = w.Write(buff.Bytes())
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	// remove output.txt file if exists
+	if !fileutil.FileExists("output.txt") {
+		return fmt.Errorf("extractor output file output.txt file does not exist")
+	} else {
+		_ = os.Remove("output.txt")
+	}
+	return expectResultsCount(results, 1)
+}

v2/pkg/operators/extractors/extract.go

@@ -3,9 +3,10 @@ package extractors
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
+
 	"github.com/antchfx/htmlquery"
 	"github.com/antchfx/xmlquery"
-	"strings"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
@@ -29,6 +30,7 @@ func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
 			}
 		}
 	}
+	e.SaveToFile(results)
 	return results
 }
 
@@ -56,6 +58,7 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
 			results[itemString] = struct{}{}
 		}
 	}
+	e.SaveToFile(results)
 	return results
 }
 
@@ -93,6 +96,7 @@ func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
 			}
 		}
 	}
+	e.SaveToFile(results)
 	return results
 }
 
@@ -123,6 +127,7 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} {
 			}
 		}
 	}
+	e.SaveToFile(results)
 	return results
 }
 
@@ -159,6 +164,7 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
 			}
 		}
 	}
+	e.SaveToFile(results)
 	return results
 }
 
@@ -179,6 +185,6 @@ func (e *Extractor) ExtractDSL(data map[string]interface{}) map[string]struct{}
 			}
 		}
 	}
-
+	e.SaveToFile(results)
 	return results
 }

v2/pkg/operators/extractors/extractors.go

@@ -1,10 +1,14 @@
 package extractors
 
 import (
+	"os"
+	"path/filepath"
 	"regexp"
 
 	"github.com/Knetic/govaluate"
 	"github.com/itchyny/gojq"
+	"github.com/projectdiscovery/gologger"
+	fileutil "github.com/projectdiscovery/utils/file"
 )
 
 // Extractor is used to extract part of response using a regex.
@@ -113,4 +117,36 @@ type Extractor struct {
 	//   - false
 	//   - true
 	CaseInsensitive bool `yaml:"case-insensitive,omitempty" json:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"`
+	// description: |
+	//  ToFile (to) saves extracted requests to file and if file is present values are appended to file.
+	ToFile string `yaml:"to,omitempty" json:"to,omitempty" jsonschema:"title=save extracted values to file,description=save extracted values to file"`
+}
+
+// SaveToFile saves extracted values to file if `to` is present and valid
+func (e *Extractor) SaveToFile(data map[string]struct{}) {
+	if e.ToFile == "" {
+		return
+	}
+
+	if !fileutil.FileExists(e.ToFile) {
+		baseDir := filepath.Dir(e.ToFile)
+		if baseDir != "." && !fileutil.FolderExists(baseDir) {
+			if err := fileutil.CreateFolder(baseDir); err != nil {
+				gologger.Error().Msgf("extractor: could not create folder %s: %s\n", baseDir, err)
+				return
+			}
+		}
+	}
+	file, err := os.OpenFile(e.ToFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		gologger.Error().Msgf("extractor: could not open file %s: %s\n", e.ToFile, err)
+		return
+	}
+	defer file.Close()
+	for k := range data {
+		if _, err = file.WriteString(k + "\n"); err != nil {
+			gologger.Error().Msgf("extractor: could not write to file %s: %s\n", e.ToFile, err)
+			return
+		}
+	}
 }

v2/pkg/progress/progress.go

@@ -170,6 +170,11 @@ func (p *StatsTicker) makePrintCallback() func(stats clistats.StatisticsClient)
 		requests, okRequests := stats.GetCounter("requests")
 		total, okTotal := stats.GetCounter("total")
 
+		// If input is not given, total is 0 which cause percentage overflow
+		if total == 0 {
+			total = requests
+		}
+
 		if okRequests && okTotal && duration > 0 && !p.cloud {
 			builder.WriteString(" | RPS: ")
 			builder.WriteString(clistats.String(uint64(float64(requests) / duration.Seconds())))

v2/pkg/protocols/http/build_request.go

@@ -255,6 +255,11 @@ func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urluti
 // finalVars = contains all variables including generator and protocol specific variables
 // generatorValues = contains variables used in fuzzing or other generator specific values
 func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest string, baseURL *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
+	// Unlike other requests parsedURL/ InputURL in self contained templates is extracted from raw request itself h
+	// and variables are supposed to be given from command line and not from inputURL
+	// ence this cause issues like duplicated params/paths.
+	// TODO: implement a generic raw request parser in rawhttp library (without variables and stuff)
+	baseURL.Params = nil // this fixes issue of duplicated params in self contained templates but not a appropriate fix
 	rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe)
 	if err != nil {
 		return nil, err