Misc sdk changes (#6018)

* feat: misc sdk changes to parser

* misc

* feat: fixed failing tests

* fix lint error + update yamldoc-go

* return 0 exit code if integration test re-run passes

* exclude tech / wordpress template from test

---------

Co-authored-by: Ice3man <nizamulrana@gmail.com>

Makefile

@@ -137,5 +137,5 @@ dsl-docs:
 template-validate: build
 template-validate:
 	./bin/nuclei -ut
-	./bin/nuclei -validate
+	./bin/nuclei -validate -et http/technologies
-	./bin/nuclei -validate -w workflows
+	./bin/nuclei -validate -w workflows -et http/technologies

cmd/integration-test/integration-test.go

@@ -115,6 +115,10 @@ func main() {
 				fmt.Println("::group::Failed integration tests in debug mode")
 				_ = runTests(failedTestTemplatePaths)
 				fmt.Println("::endgroup::")
+			} else {
+				fmt.Println("::group::All tests passed")
+				fmt.Println("::endgroup::")
+				os.Exit(0)
 			}
 		}
 

cmd/integration-test/ssl.go

@@ -123,7 +123,11 @@ func (h *sslWithVars) Execute(filePath string) error {
 type sslMultiReq struct{}
 
 func (h *sslMultiReq) Execute(filePath string) error {
-	ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) {
+	//nolint:staticcheck // SSLv3 is intentionally used for testing purposes
+	ts := testutils.NewTCPServer(&tls.Config{
+		MinVersion: tls.VersionSSL30,
+		MaxVersion: tls.VersionTLS11,
+	}, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 		data := make([]byte, 4)
 		if _, err := conn.Read(data); err != nil {

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/projectdiscovery/rawhttp v0.1.84
 	github.com/projectdiscovery/retryabledns v1.0.94
 	github.com/projectdiscovery/retryablehttp-go v1.0.97
-	github.com/projectdiscovery/yamldoc-go v1.0.4
+	github.com/projectdiscovery/yamldoc-go v1.0.5
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
 	github.com/segmentio/ksuid v1.0.4

go.sum

@@ -927,8 +927,8 @@ github.com/projectdiscovery/utils v0.4.8 h1:/Xd38fP8xc6kifZayjrhcYALenJrjO3sHO7l
 github.com/projectdiscovery/utils v0.4.8/go.mod h1:S314NzLcXVCbLbwYCoorAJYcnZEwv7Uhw2d3aF5fJ4s=
 github.com/projectdiscovery/wappalyzergo v0.2.13 h1:yrzmcT/Zq+H+qpbAjHKKFVVQA7uei+sAYlmS2HLpkGI=
 github.com/projectdiscovery/wappalyzergo v0.2.13/go.mod h1:3vtvQCSYpU+Ilk0qy09WYT9BH0Stut5Qon7KJJ78GKw=
-github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
+github.com/projectdiscovery/yamldoc-go v1.0.5 h1:On5D9F5Qe59Tek9SX05uSbjahiRTvDqUSIoG/gDuRd4=
-github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
+github.com/projectdiscovery/yamldoc-go v1.0.5/go.mod h1:R5lWrNzP+7Oyn77NDVPnBsxx2/FyQZBBkIAaSaCQFxw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=

pkg/protocols/offlinehttp/request.go

@@ -1,7 +1,9 @@
 package offlinehttp
 
 import (
+	"fmt"
 	"io"
+	"net/http"
 	"net/http/httputil"
 	"os"
 
@@ -36,7 +38,7 @@ var RawInputMode = false
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	if RawInputMode {
-		return request.executeRawInput(input.MetaInput.Input, input, callback)
+		return request.executeRawInput(input.MetaInput.Input, "", input, callback)
 	}
 
 	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
@@ -74,7 +76,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 			}
 			dataStr := conversion.String(buffer)
 
-			if err := request.executeRawInput(dataStr, input, callback); err != nil {
+			if err := request.executeRawInput(dataStr, data, input, callback); err != nil {
 				gologger.Error().Msgf("Could not execute raw input %s: %s\n", data, err)
 				return
 			}
@@ -90,7 +92,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	return nil
 }
 
-func (request *Request) executeRawInput(data string, input *contextargs.Context, callback protocols.OutputEventCallback) error {
+func (request *Request) executeRawInput(data, inputString string, input *contextargs.Context, callback protocols.OutputEventCallback) error {
 	resp, err := readResponseFromString(data)
 	if err != nil {
 		return errors.Wrap(err, "could not read raw response")
@@ -111,8 +113,12 @@ func (request *Request) executeRawInput(data string, input *contextargs.Context,
 	if err != nil {
 		return errors.Wrap(err, "could not read raw http response body")
 	}
+	reqURL := inputString
+	if inputString == "" {
+		reqURL = getURLFromRequest(resp.Request)
+	}
 
-	outputEvent := request.responseToDSLMap(resp, data, data, data, conversion.String(dumpedResponse), conversion.String(body), utils.HeadersToString(resp.Header), 0, nil)
+	outputEvent := request.responseToDSLMap(resp, data, reqURL, data, conversion.String(dumpedResponse), conversion.String(body), utils.HeadersToString(resp.Header), 0, nil)
 	// add response fields to template context and merge templatectx variables to output event
 	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), outputEvent)
 	if request.options.HasTemplateCtx(input.MetaInput) {
@@ -124,3 +130,10 @@ func (request *Request) executeRawInput(data string, input *contextargs.Context,
 	callback(event)
 	return nil
 }
+
+func getURLFromRequest(req *http.Request) string {
+	if req.URL.Scheme == "" {
+		req.URL.Scheme = "https"
+	}
+	return fmt.Sprintf("%s://%s%s", req.URL.Scheme, req.Host, req.URL.Path)
+}

pkg/templates/parser.go

@@ -33,6 +33,13 @@ func NewParser() *Parser {
 	return p
 }
 
+func NewParserWithParsedCache(cache *Cache) *Parser {
+	return &Parser{
+		parsedTemplatesCache:   cache,
+		compiledTemplatesCache: NewCache(),
+	}
+}
+
 // Cache returns the parsed templates cache
 func (p *Parser) Cache() *Cache {
 	return p.parsedTemplatesCache