diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 79d718fa6..de3f9c15b 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -10,6 +10,7 @@ import ( "net/http/httputil" "os" "reflect" + "regexp" "strconv" "strings" "time" @@ -80,6 +81,8 @@ var httpTestcases = []TestCaseInfo{ {Path: "protocols/http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}}, {Path: "protocols/http/matcher-status.yaml", TestCase: &matcherStatusTest{}}, {Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}}, + {Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}}, + {Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}}, } type httpInteractshRequest struct{} @@ -1475,3 +1478,56 @@ func (h *httpInteractshRequestsWithMCAnd) Execute(filePath string) error { } return expectResultsCount(got, 1) } + +// integration test to check if preprocessor i.e {{randstr}} +// is working correctly +type httpPreprocessor struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpPreprocessor) Execute(filePath string) error { + router := httprouter.New() + re := regexp.MustCompile(`[A-Za-z0-9]{25,}`) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + value := r.URL.RequestURI() + if re.MatchString(value) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "ok") + } else { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, "not ok") + } + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + + return expectResultsCount(results, 1) +} + +type httpMultiRequest struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpMultiRequest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/ping", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "ping") + }) + router.GET("/pong", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "pong") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + + return expectResultsCount(results, 1) +} diff --git a/go.mod b/go.mod index d14904fe9..543ee59de 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/projectdiscovery/hmap v0.0.22 github.com/projectdiscovery/interactsh v1.1.7 github.com/projectdiscovery/rawhttp v0.1.22 - github.com/projectdiscovery/retryabledns v1.0.38 + github.com/projectdiscovery/retryabledns v1.0.39 github.com/projectdiscovery/retryablehttp-go v1.0.31 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 @@ -77,7 +77,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.0.25 + github.com/projectdiscovery/dsl v0.0.26 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.24 github.com/projectdiscovery/gologger v1.1.11 @@ -86,11 +86,11 @@ require ( github.com/projectdiscovery/httpx v1.3.5 github.com/projectdiscovery/mapcidr v1.1.12 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.11 + github.com/projectdiscovery/ratelimit v0.0.12 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766 - github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 + github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/utils v0.0.58 github.com/projectdiscovery/wappalyzergo v0.0.109 github.com/redis/go-redis/v9 v9.1.0 diff --git a/go.sum b/go.sum index c245e50cc..85f411bf1 100644 --- a/go.sum +++ b/go.sum @@ -791,8 +791,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.25 h1:KfllJofzqp6dc6BmZOnlGm7PybThBF/DN1xmlCNJv/4= -github.com/projectdiscovery/dsl v0.0.25/go.mod h1:WZJKFQDIy6CAzkvcMwrRld3qfLbUZD+GLO5m+5wb3Fs= +github.com/projectdiscovery/dsl v0.0.26 h1:zSirh38+KIaBtak2Vi8LVpN5Aj1ZuwpARnQ69Ja5fSs= +github.com/projectdiscovery/dsl v0.0.26/go.mod h1:XCsu0FtORqnjRElNTYrewzTw4LPlBtPu/j4T2Hel2UU= github.com/projectdiscovery/fastdialer v0.0.40 h1:gXKJv32xyXDpNXM1zzaY7IGYfsZPPOp1V8CDjgKrNX0= github.com/projectdiscovery/fastdialer v0.0.40/go.mod h1:51m88QxZiJ06Df6WCFeauY9EjfbalLFKeqvDmXovwTI= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= @@ -819,14 +819,14 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= -github.com/projectdiscovery/ratelimit v0.0.11 h1:QPIIo8ACGd2YayOQ/EdQOP3hd+X+9afSbC34wa3eiyI= -github.com/projectdiscovery/ratelimit v0.0.11/go.mod h1:wEGPNnZyUalJX63y7rZeeRZgRd4HR+FpBc9BsQxpYY0= +github.com/projectdiscovery/ratelimit v0.0.12 h1:Hwn9D1PHDz4O4ndzFZCttvTqdtxjBJrifi7n4oa4/Og= +github.com/projectdiscovery/ratelimit v0.0.12/go.mod h1:mA4775qZGakULYWDI0yswMntTWKQmEbFbb199uuGdzs= github.com/projectdiscovery/rawhttp v0.1.22 h1:tJiAUjgM7xtDadiLqpF76w/ddJxGCSFu5FfTFwKqIhk= github.com/projectdiscovery/rawhttp v0.1.22/go.mod h1:zZl88fEut97zuZco8gHaEQEOuYKN7F089/REo3C+J3Y= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.38 h1:PR5pM0702/0upq30R0k/AnXhcAFqHGC6ShGof8rmT+w= -github.com/projectdiscovery/retryabledns v1.0.38/go.mod h1:MtqDQkV1brf0prql8RtyDrmd1Y39MTGIupzeiFOXRuo= +github.com/projectdiscovery/retryabledns v1.0.39 h1:hvh/fNfKAB+JdOtLtJLo1/MO2nihTE/OWanGnIQp8Ds= +github.com/projectdiscovery/retryabledns v1.0.39/go.mod h1:HaHABQBdiL1KoOQ99XqMRDC0RhkzkGfiV3QnUhYUeRg= github.com/projectdiscovery/retryablehttp-go v1.0.31 h1:Z66QM4FkAyIEBje4acu3hdUPaE3NSFYNa54hR69ZlPo= github.com/projectdiscovery/retryablehttp-go v1.0.31/go.mod h1:pFBFbxnb7fupJbl99n9M0GkoUZCtbkRZM3xfmTSWjqE= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= @@ -835,8 +835,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766 h1:wa2wak7RAPA9QfCKZYXVvJCggbrIptc4ZkPjEvCKAKo= github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766/go.mod h1:bFATagikCvdPOsmaN1h5VQSbZjTW8bCQ6bjoQEePUq8= -github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= -github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= +github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A= +github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/utils v0.0.58 h1:kk2AkSO84QZc9rDRI8jWA2Iia4uzb4sUcfh4h0xA20I= github.com/projectdiscovery/utils v0.0.58/go.mod h1:rsR5Kzjrb+/Yp7JSnEblLk4LfU4zH5Z7wQn8RzaGSdY= github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k= diff --git a/integration_tests/protocols/http/http-preprocessor.yaml b/integration_tests/protocols/http/http-preprocessor.yaml new file mode 100644 index 000000000..b856f3ccb --- /dev/null +++ b/integration_tests/protocols/http/http-preprocessor.yaml @@ -0,0 +1,17 @@ +id: http-preprocessor + +info: + name: Test Http Preprocessor + author: pdteam + severity: info + +requests: + - raw: + - | + GET /?test={{randstr}} HTTP/1.1 + Host: {{Hostname}} + + matchers: + - type: status + status: + - 200 \ No newline at end of file diff --git a/integration_tests/protocols/http/multi-request.yaml b/integration_tests/protocols/http/multi-request.yaml new file mode 100644 index 000000000..4ede5e37e --- /dev/null +++ b/integration_tests/protocols/http/multi-request.yaml @@ -0,0 +1,26 @@ +id: http-multi-request + +info: + name: http multi request template + author: pdteam + severity: info + description: template with multiple http request with combined logic + reference: https://example-reference-link + +# requestURI is reflected back as response body here +http: + - raw: + - | + GET /ping HTTP/1.1 + Host: {{Hostname}} + + - | + GET /pong HTTP/1.1 + Host: {{Hostname}} + + matchers: + - type: dsl + dsl: + - 'body_1 == "ping"' + - 'body_2 == "pong"' + condition: and \ No newline at end of file diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index db73f2864..56895fc2c 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.0.0` + Version = `v3.0.1` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/core/inputs/hybrid/hmap_test.go b/pkg/core/inputs/hybrid/hmap_test.go index fee92f469..3dbbdb734 100644 --- a/pkg/core/inputs/hybrid/hmap_test.go +++ b/pkg/core/inputs/hybrid/hmap_test.go @@ -180,6 +180,10 @@ func Test_expandASNInputValue(t *testing.T) { got = append(got, metainput.Input) return nil }) + if len(got) == 0 { + // asnmap server is down + t.SkipNow() + } // read the expected IPs from the file fileContent, err := os.ReadFile(tt.expectedOutputFile) require.Nil(t, err, "could not read the expectedOutputFile file") diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 9131ba4a2..aac2bad71 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -793,7 +793,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if request.NeedsRequestCondition() { for k, v := range outputEvent { key := fmt.Sprintf("%s_%d", k, requestCount) - if previousEvent[key] != nil { + if previousEvent != nil { previousEvent[key] = v } finalEvent[key] = v diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 3c6a9694f..5f5e63456 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -272,7 +272,7 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option // persist verified status value and then // expand all preprocessor and reparse template - // === signature verification befoer preprocessors === + // === signature verification before preprocessors === template, err := parseTemplate(data, options) if err != nil { return nil, err diff --git a/pkg/templates/preprocessors.go b/pkg/templates/preprocessors.go index f730ec098..d7e5864d6 100644 --- a/pkg/templates/preprocessors.go +++ b/pkg/templates/preprocessors.go @@ -17,7 +17,9 @@ type Preprocessor interface { var ( preprocessorRegex = regexp.MustCompile(`{{([a-z0-9_]+)}}`) - defaultPreprocessors = []Preprocessor{} + defaultPreprocessors = []Preprocessor{ + &randStrPreprocessor{}, + } ) func getPreprocessors(preprocessor Preprocessor) []Preprocessor {