Adding SNI override via request annotations (#1970)

* Adding SNI override via request annotations

* adding cli flag priority

integration_tests/http/get-override-sni.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-http-example
+
+info:
+  name: Test RAW GET Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        @tls-sni:request.host
+        GET / HTTP/1.1
+        Host: test
+
+    matchers:
+      - type: word
+        words:
+          - "test-ok"

v2/cmd/integration-test/http.go

@@ -48,6 +48,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/stop-at-first-match.yaml":                 &httpStopAtFirstMatch{},
 	"http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{},
 	"http/variables.yaml":                           &httpVariables{},
+	"http/get-override-sni.yaml":                    &httpSniAnnotation{},
 	"http/get-sni.yaml":                             &customCLISNI{},
 }
 
@@ -833,3 +834,25 @@ func (h *customCLISNI) Execute(filePath string) error {
 	}
 	return expectResultsCount(results, 1)
 }
+
+type httpSniAnnotation struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpSniAnnotation) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		if r.TLS.ServerName == "test" {
+			_, _ = w.Write([]byte("test-ok"))
+		} else {
+			_, _ = w.Write([]byte("test-ko"))
+		}
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}

v2/go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
 	github.com/projectdiscovery/cryptoutil v1.0.0
-	github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47
+	github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83
 	github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
 	github.com/projectdiscovery/fileutil v0.0.0-20220427234316-40b2541a84b8
 	github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323

v2/go.sum

@@ -437,6 +437,8 @@ github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQ
 github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
 github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47 h1:TUsZiwez9uFmph1hlTsiH7rdB+wi4524+lMuV2z6FaM=
 github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47/go.mod h1:GbQvP1ezGlQn0af3lVcl08b5eRQu960T7A9pwazybSo=
+github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83 h1:1hzvl0lsWpvQ8nn1s9YMyBjO13/Z+f/T4W2jroOohfo=
+github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI=
 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew=
 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ=
 github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=

v2/pkg/protocols/common/protocolstate/state.go

@@ -21,6 +21,7 @@ func Init(options *types.Options) error {
 	}
 	opts.WithDialerHistory = true
 	opts.WithZTLS = options.ZTLS
+	opts.SNIName = options.SNI
 	dialer, err := fastdialer.NewDialer(opts)
 	if err != nil {
 		return errors.Wrap(err, "could not create dialer")

v2/pkg/protocols/headless/engine/http_client.go

@@ -41,6 +41,7 @@ func newHttpClient(options *types.Options) (*http.Client, error) {
 
 	transport := &http.Transport{
 		DialContext:         dialer.Dial,
+		DialTLSContext:      dialer.DialTLS,
 		MaxIdleConns:        500,
 		MaxIdleConnsPerHost: 500,
 		MaxConnsPerHost:     500,

v2/pkg/protocols/http/build_request.go

@@ -290,7 +290,10 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest
 		return nil, err
 	}
 
-	parseAnnotations(rawRequest, req)
+	if reqWithAnnotations, hasAnnotations := parseAnnotations(rawRequest, req); hasAnnotations {
+		req = reqWithAnnotations
+		request = request.WithContext(req.Context())
+	}
 
 	return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil
 }

v2/pkg/protocols/http/httpclientpool/clientpool.go

@@ -191,6 +191,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 
 	transport := &http.Transport{
 		DialContext:         Dialer.Dial,
+		DialTLSContext:      Dialer.DialTLS,
 		MaxIdleConns:        maxIdleConns,
 		MaxIdleConnsPerHost: maxIdleConnsPerHost,
 		MaxConnsPerHost:     maxConnsPerHost,

v2/pkg/protocols/http/request_annotations.go

@@ -1,22 +1,33 @@
 package http
 
 import (
+	"context"
 	"net"
 	"net/http"
 	"regexp"
 	"strings"
 
+	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/iputil"
 	"github.com/projectdiscovery/stringsutil"
 	"github.com/projectdiscovery/urlutil"
 )
 
+var (
 	// @Host:target overrides the input target with the annotated one (similar to self-contained requests)
-var reHostAnnotation = regexp.MustCompile(`(?m)^@Host:\s*(.+)\s*$`)
+	reHostAnnotation = regexp.MustCompile(`(?m)^@Host:\s*(.+)\s*$`)
+	// @tls-sni:target overrides the input target with the annotated one
+	// special values:
+	// request.host: takes the value from the host header
+	// target: overiddes with the specific value
+	reSniAnnotation = regexp.MustCompile(`(?m)^@tls-sni:\s*(.+)\s*$`)
+)
 
 // parseAnnotations and override requests settings
-func parseAnnotations(rawRequest string, request *http.Request) {
+func parseAnnotations(rawRequest string, request *http.Request) (*http.Request, bool) {
 	// parse request for known ovverride annotations
+	var modified bool
+	// @Host:target
 	if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
 		value := strings.TrimSpace(hosts[1])
 		// handle scheme
@@ -39,7 +50,25 @@ func parseAnnotations(rawRequest string, request *http.Request) {
 			}
 			request.URL.Host = hostPort
 		}
+		modified = true
 	}
+
+	// @tls-sni:target
+	if hosts := reSniAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {
+		value := strings.TrimSpace(hosts[1])
+		value = stringsutil.TrimPrefixAny(value, "http://", "https://")
+		if idxForwardSlash := strings.Index(value, "/"); idxForwardSlash >= 0 {
+			value = value[:idxForwardSlash]
+		}
+
+		if stringsutil.EqualFoldAny(value, "request.host") {
+			value = request.Host
+		}
+		ctx := context.WithValue(request.Context(), fastdialer.SniName, value)
+		request = request.Clone(ctx)
+		modified = true
+	}
+	return request, modified
 }
 
 func isHostPort(value string) bool {

v2/pkg/reporting/exporters/es/elasticsearch.go

@@ -63,6 +63,7 @@ func New(option *Options) (*Exporter, error) {
 				MaxIdleConns:        10,
 				MaxIdleConnsPerHost: 10,
 				DialContext:         protocolstate.Dialer.Dial,
+				DialTLSContext:      protocolstate.Dialer.DialTLS,
 				TLSClientConfig:     &tls.Config{InsecureSkipVerify: option.SSLVerification},
 			},
 		}