Improving clientpool with client certificates (#3851)

* Improving clientpool with client certificates

* adding test case

* Revert "Merge branch 'dev' into issue-3800-client-cert"

This reverts commit 7f057d742f4b9bda8e83b2052e29617b86b6776d, reversing
changes made to 7297cebcf8bb0f88961b644fc2ac7c040df8ffd9.

* Revert "Revert "Merge branch 'dev' into issue-3800-client-cert""

This reverts commit 2053a248a0cdc2002e0b4b4faa3472cf11c29760.

* go fmt

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

integration_tests/generic/auth/certificate/assets/client.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDETCCAfkCFFSLOinkkPWOfwuzMHF0B9EZIIghMA0GCSqGSIb3DQEBCwUAMEUx
+CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MzA2WhcNMjMwNzIxMDA0
+MzA2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
+CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAp187MX/sPGlTl8WklyTSh8+kYURy2zWmp2FglMvo058NanCD
+rjYJvS2FNM2ZwRZH52qBr6rpqIsa8QDFjnlFJK5y4FNr104ZjDr1YXDfX5ftQeeX
+wCavFRDdH/s43kGyNjNeSH78EhqmC3z7LxCJT5k8rXRAyC02uBRqQTrRHfNuVvGI
+JZ59jxESzRJt89kWuYymkAj16LHffMvcq6HLwr/KG8IyrJJRj5KcDzYorFonTPe9
+rwBlAbU3LpQ4ZlEP1mQA2PdCg3t85pz3n+57Iw839bWrwbjfijXHY3yjbqfEAaqa
+md08nByTg8TZlt9UzjWgj0K1DXDMDDkJKqi39QIDAQABMA0GCSqGSIb3DQEBCwUA
+A4IBAQAH0MhznadrMFuY2ZYA69FbsvOygMctv8qZW1HrHS0X13IXeW+8uxfb5+gk
+yKFgXNMFueyd5PoN9vyC1t9AOBPnI56gaYm/MbDtwEqGo8C+9fjJasY23J90p54t
+G6vxcXwo33HVpWBeRBkVF/SePeCn+MKk0jd/JgJS0T0s1Ih6wkn84/83hDk4M2M2
+/yhc3wuNYdf/WB8QAfJAc2YpIfkMOoxGPTsxvREiZrPUyGiWa507hrHcQU0GV8qC
+KcnS7UUCT9TtJvQIKHwW68XjBudWpaILBj1TS8hOGseOJydJqbk8wyMTE6fgc1Ss
+KfTrfa0HOHIkAU/TfE22Zqfw4z70
+-----END CERTIFICATE-----

integration_tests/generic/auth/certificate/assets/client.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCnXzsxf+w8aVOX
+xaSXJNKHz6RhRHLbNaanYWCUy+jTnw1qcIOuNgm9LYU0zZnBFkfnaoGvqumoixrx
+AMWOeUUkrnLgU2vXThmMOvVhcN9fl+1B55fAJq8VEN0f+zjeQbI2M15IfvwSGqYL
+fPsvEIlPmTytdEDILTa4FGpBOtEd825W8Yglnn2PERLNEm3z2Ra5jKaQCPXosd98
+y9yrocvCv8obwjKsklGPkpwPNiisWidM972vAGUBtTculDhmUQ/WZADY90KDe3zm
+nPef7nsjDzf1tavBuN+KNcdjfKNup8QBqpqZ3TycHJODxNmW31TONaCPQrUNcMwM
+OQkqqLf1AgMBAAECggEAEZiXdorGYUuJeElVFnbOk2ynEXrKwHURgkNgjgQqBCWS
+mYAet/ACchsZCAYdhgk7of62h6tmSUvmlzPHkUT6mfKlLSRYEBir4uxH4+ij8z7b
+uLQKZi8q9QIC3VviDKvHep9H6ENBaP3YOxj2p2oLpYysrmesb98hA5VR3m26knVb
+sfYwLdsrqQ5Tiuzm1Mdaca2HEYq0iXoZqjQHi31h33rw4HFBKcGX6w6PJOon+i2m
+eSSaCAJMYFkoS6NafJYwQDuwIwp+IyrYdI/vaAR6s2ufJ4doIjJC7YuPO5jpLpJc
+IBBl5e7pu4/rlwJCDARfBWuXFbjtoMAvmM1MojdNOQKBgQDYCIl97ef19LmeYfwk
+RW2xhQTVCWwwBBhSl06iB9SeFkzCCRb+AFjhVhcwe4xIWh9GXWZawZC2XSaoTXte
+hxcZRFbnpjcpZ6sYKiP4fB/GXGqYDUWovxu8gmXKrXtfwqJMXklfFE7WblGgK5gG
+l7OfbuczaRaIQHoIQYzfmeCwmwKBgQDGVhCSNFGiRaDG7k2VfXElaIyL7m7FlsXf
+EptolOeoGv5GvVr3CB5TvUqXN3haLZvUbBKRpWrDbP1n1i+77VICOLtq1qf6SEog
+1p2PAccGhXXvrL7LJLUr9Hk831D4fSX9TqVzdxwfMdGFepYoS4vm8fkGVCuaytDa
+fniJl8TarwKBgQCpTreCrAsY5bz7dcuIGamIcLmCxKm3T95IDDEiJ4ToiI2LnFga
+pOcDYtc1tf4RTiAoo1ZuVjk10vdS+7ZuNO1Tbg216rxchNTAUXZzbcPxT8hydiRb
+xbrVGFTybNe+CunrdBGIpH/M6hSqtL+mmwm5L8+eqQNxsSZyhf0D2LMRdQKBgD8J
+CXk+MZfOY1v2Tygs1zIZeVnb7M7VrYvJYSUq9jliYuBevDN5HBJnPfazhYe7qSQp
+OPmbRkRYNm2zEDa9JWxZVY+OK5MLOKwZKbhSy0uSTTpgf78WqpIOwB2NqDFhrRpF
+zaXV/FUZw0qV/HVQFWXQD+JoC/fFb/2RZoPsfX83AoGBAM4VPDUa1I16MuuW+FNZ
+cVENZK/qsXFMvm3xutezvMSgmCxVSnXy9GP8QbqkfMrDJd3v/HnwrC4ORTlU7rim
+AWSvC6CYO1c2RantleA46T90uWW8kP24TK1yWOrRGKuaQYvfokiiLVExq1nA1iSR
+/QPLg6vEoPMOLhB7BQBpsFkq
+-----END PRIVATE KEY-----

integration_tests/generic/auth/certificate/assets/server.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDETCCAfkCFHA1RpGfOY5p/vQmeMQ1oRFqH+CGMA0GCSqGSIb3DQEBCwUAMEUx
+CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MjQ2WhcNMjMwNzIxMDA0
+MjQ2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
+CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEA3VdrKR5hmZ+vyvg6NB2dOL5vEIQ/9DevivnKWqX5mserYLMj
+Wq0knVfogewZnrDe+zVC3kOogBQvYk8Z53kTY9qpJT85dMCuW4xDx0JU+cWHul9a
+pzF+bvws4paCWIcsGONyocPAx5g07LbPU9civC80QkQqELo1zYiRU1bX8vRJJqbN
+TW2mzl9MN3AnCAYTwq8WhVG/1QR3LPQhPR68/1LWrFefQaEWaXT2s+Xv7K7NDXro
+WSba4SgKdFd6fyUVMVr/ioT1KT45TP5jbRrW5JJUTdpkiXaIucrZg39f6F5gTZGA
+U7bNROUMkqrJJngN9+Hp+YH1GpkKgu9EKA30EQIDAQABMA0GCSqGSIb3DQEBCwUA
+A4IBAQAw91bxiAi7DIVsKL3k4B0I+50ZKq9VMVNE3YCTPygpfuRiGQvlITZ5I8I5
+3Ok2wWltgKx6EnicHIlLg42yRj7j3mdgOLMFMrUCfJmdogwnS+k6veG3G1RHUs9r
+ATfX49u/hEX2pe7Rvx2VYVIugwrQESgQ21iaf6uUMsrq6W8eYZ31as1nJKpqIGbu
+W1fZMSi0RIUJP+mpVBE82IW+gJRi3uKU4HKPqyrU3dviBFdBxb3lNbh34/vdNkIw
+4H2CfBxEvdwLYAhWDerlm4wWCmjkMiHfBHPBhhOICTkR25a7NFy27h/UDHjVC/6m
+fGshVSBtxVPJP7kcvZ1scIctvFZZ
+-----END CERTIFICATE-----

integration_tests/generic/auth/certificate/http-get.yaml

@@ -0,0 +1,15 @@
+id: basic-get-with-cert
+
+info:
+  name: Basic GET with Cert
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "Hello"

v2/cmd/integration-test/generic.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+
+	"github.com/julienschmidt/httprouter"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+)
+
+var genericTestcases = map[string]testutils.TestCase{
+	"generic/auth/certificate/http-get.yaml": &clientCertificate{},
+}
+
+var (
+	serverCRT = `-----BEGIN CERTIFICATE-----
+MIIDETCCAfkCFHA1RpGfOY5p/vQmeMQ1oRFqH+CGMA0GCSqGSIb3DQEBCwUAMEUx
+CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNjIxMDA0MjQ2WhcNMjMwNzIxMDA0
+MjQ2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
+CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEA3VdrKR5hmZ+vyvg6NB2dOL5vEIQ/9DevivnKWqX5mserYLMj
+Wq0knVfogewZnrDe+zVC3kOogBQvYk8Z53kTY9qpJT85dMCuW4xDx0JU+cWHul9a
+pzF+bvws4paCWIcsGONyocPAx5g07LbPU9civC80QkQqELo1zYiRU1bX8vRJJqbN
+TW2mzl9MN3AnCAYTwq8WhVG/1QR3LPQhPR68/1LWrFefQaEWaXT2s+Xv7K7NDXro
+WSba4SgKdFd6fyUVMVr/ioT1KT45TP5jbRrW5JJUTdpkiXaIucrZg39f6F5gTZGA
+U7bNROUMkqrJJngN9+Hp+YH1GpkKgu9EKA30EQIDAQABMA0GCSqGSIb3DQEBCwUA
+A4IBAQAw91bxiAi7DIVsKL3k4B0I+50ZKq9VMVNE3YCTPygpfuRiGQvlITZ5I8I5
+3Ok2wWltgKx6EnicHIlLg42yRj7j3mdgOLMFMrUCfJmdogwnS+k6veG3G1RHUs9r
+ATfX49u/hEX2pe7Rvx2VYVIugwrQESgQ21iaf6uUMsrq6W8eYZ31as1nJKpqIGbu
+W1fZMSi0RIUJP+mpVBE82IW+gJRi3uKU4HKPqyrU3dviBFdBxb3lNbh34/vdNkIw
+4H2CfBxEvdwLYAhWDerlm4wWCmjkMiHfBHPBhhOICTkR25a7NFy27h/UDHjVC/6m
+fGshVSBtxVPJP7kcvZ1scIctvFZZ
+-----END CERTIFICATE-----
+`
+	serverKey = `-----BEGIN PRIVATE KEY-----
+MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDdV2spHmGZn6/K
++Do0HZ04vm8QhD/0N6+K+cpapfmax6tgsyNarSSdV+iB7BmesN77NULeQ6iAFC9i
+TxnneRNj2qklPzl0wK5bjEPHQlT5xYe6X1qnMX5u/CziloJYhywY43Khw8DHmDTs
+ts9T1yK8LzRCRCoQujXNiJFTVtfy9Ekmps1NbabOX0w3cCcIBhPCrxaFUb/VBHcs
+9CE9Hrz/UtasV59BoRZpdPaz5e/srs0NeuhZJtrhKAp0V3p/JRUxWv+KhPUpPjlM
+/mNtGtbkklRN2mSJdoi5ytmDf1/oXmBNkYBTts1E5QySqskmeA334en5gfUamQqC
+70QoDfQRAgMBAAECggEBALtPsHMSr9vW5Giq2m6iJRwRJGJg2NJukZLVwuYlkW7n
+zGNAFgo1fkfdTfks+Z1u5rTGJPl9XkpNSrAyaqSVtNALCptnvtLMAIGe2Pj2bH0X
+Kb6R1WCqJOn9ZGq4nkQW2D2Ttb2psCn458jvB9NWu6FvfRUbJFIVk1SFXx6c3pFN
+kPCUudAiscaldUDCiz4FccKGXdRjq6HIeeWqvdErteb6JPTs9QXCHfBql9Esl4rK
+SHt9RmAFNY+CLExHiFPBR15hHZRtiVkAVrgnPg1CPGAyVG0hGXj7YMMWpAyfFWpn
+8gWVt7XJ4UX2knUwfU8p8dWe6qwf+AMrhravYJyccoUCgYEA8Ts0kHFnLga8Ewao
+nyDQs5uYGG0PWkbXqnFVYnMeSbXzyC4ouInIk/eOQABCxdjy3NF9QuYvVLpfLJ+9
+a97q1Vyg6lZ4PPuK8ZcPrHFSNNaj4eWNTOMo/Qdzz4bfflTsv8vjeeMxsqb6woXV
++E23UKCPlQPf86jugZVdaMtvZKsCgYEA6uR7glji70pVoG/f3soX1vllmVTtiLnh
+zYMmwPyTRDvoGgg/nGK+GCq//Xyn8D900hbX8KKqGX7ca5FGk5pOpW/QE9uLcuWK
+xcy8KAc05k1u4VaS5loWKnPGWreIpj3RbCfbPs5X/jBC+fPIA4Q8Qor5ZGdqVBvW
+IKejnNqasjMCgYEAqltPUbpkTWLAKweGyWnZOR3mmUlbkDt7Toje7bmyaAew82t1
+omzbU3N958DHZwVA7aSbu0TnpARB9jeRA77XRHo3wYXzP828X8R4cyVMEriJ35vG
+38eESLyckrAC4SqETyZjrM4/aJT3fawaYVIw5SWegHPOEjr4xFaBMuKH9iUCgYEA
+wFpC2kc374UMAcobpjIQu7aYAKyPqDuwMb+I6NjtMB9uvoKqtMIXsWqwtkBytkcA
+v1p9k01hxmcg0eWxygW/CbM6zkgnNfvLXJeALbdZFo+qkVV4DrMPG8ybToalnJ1a
+9hrda91GKZ4T+uQrktWjE0sDV7loVWBGRY+CaFyL+gkCgYEA3Z0j8VOLJnAKdCDp
+3N74460pykwJ2suEYSJG6glXfU3fZ5VwAYjimxgD0S2VU4qK8PYBfa/oFH2vRX5p
+11dWQWbfBdREO70UmJD4Pr6g3q9AF6DXLXb7dVm4y+hX065Xshk8oIuITVyO/XVK
+wWqBD5GScI+Q7PLMes7aqtsDDJI=
+-----END PRIVATE KEY-----
+`
+)
+
+type clientCertificate struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *clientCertificate) Execute(filePath string) error {
+	router := httprouter.New()
+
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		if len(r.TLS.PeerCertificates) == 0 {
+			http.Error(w, "Client certificate required", http.StatusForbidden)
+			return
+		}
+
+		fmt.Fprintf(w, "Hello, %s!\n", r.TLS.PeerCertificates[0].Subject)
+	})
+
+	_ = os.WriteFile("server.crt", []byte(serverCRT), os.ModePerm)
+	_ = os.WriteFile("server.key", []byte(serverKey), os.ModePerm)
+	defer os.Remove("server.crt")
+	defer os.Remove("server.key")
+
+	serverCert, _ := tls.LoadX509KeyPair("server.crt", "server.key")
+
+	certPool := x509.NewCertPool()
+	caCert, _ := os.ReadFile("server.crt")
+	certPool.AppendCertsFromPEM(caCert)
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{serverCert},
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		ClientCAs:    certPool,
+	}
+
+	ts := httptest.NewUnstartedServer(router)
+
+	ts.TLS = tlsConfig
+
+	ts.StartTLS()
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug,
+		"-ca", "generic/auth/certificate/assets/server.crt",
+		"-cc", "generic/auth/certificate/assets/client.crt",
+		"-ck", "generic/auth/certificate/assets/client.key")
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}

v2/cmd/integration-test/integration-test.go

@@ -39,6 +39,7 @@ var (
 		"offlineHttp":     offlineHttpTestcases,
 		"customConfigDir": customConfigDirTestCases,
 		"fuzzing":         fuzzingTestCases,
+		"generic":         genericTestcases,
 	}
 
 	// For debug purposes

v2/internal/runner/options.go

@@ -22,6 +22,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	fileutil "github.com/projectdiscovery/utils/file"
+	"github.com/projectdiscovery/utils/generic"
 	logutil "github.com/projectdiscovery/utils/log"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -128,11 +129,11 @@ func validateOptions(options *types.Options) error {
 	}
 
 	// Verify if any of the client certificate options were set since it requires all three to work properly
-	if len(options.ClientCertFile) > 0 || len(options.ClientKeyFile) > 0 || len(options.ClientCAFile) > 0 {
-		if len(options.ClientCertFile) == 0 || len(options.ClientKeyFile) == 0 || len(options.ClientCAFile) == 0 {
+	if options.HasClientCertificates() {
+		if generic.EqualsAny("", options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile) {
 			return errors.New("if a client certification option is provided, then all three must be provided")
 		}
-		validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile})
+		validateCertificatePaths(options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile)
 	}
 	// Verify AWS secrets are passed if a S3 template bucket is passed
 	if options.AwsBucketName != "" && options.UpdateTemplates {
@@ -334,9 +335,9 @@ func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPat
 	}
 }
 
-func validateCertificatePaths(certificatePaths []string) {
+func validateCertificatePaths(certificatePaths ...string) {
 	for _, certificatePath := range certificatePaths {
-		if _, err := os.Stat(certificatePath); os.IsNotExist(err) {
+		if !fileutil.FileExists(certificatePath) {
 			// The provided path to the PEM certificate does not exist for the client authentication. As this is
 			// required for successful authentication, log and return an error
 			gologger.Fatal().Msgf("The given path (%s) to the certificate does not exist!", certificatePath)

v2/pkg/parsers/parser.go

@@ -80,7 +80,6 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, template *template
 	match, err := tagFilter.Match(template, extraTags)
 
 	if err == filter.ErrExcluded {
-
 		return false, filter.ErrExcluded
 	}
 

v2/pkg/protocols/http/httpclientpool/clientpool.go

@@ -229,6 +229,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		ForceAttemptHTTP2: options.ForceAttemptHTTP2,
 		DialContext:       Dialer.Dial,
 		DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			if options.HasClientCertificates() {
+				return Dialer.DialTLSWithConfig(ctx, network, addr, tlsConfig)
+			}
 			if options.TlsImpersonate {
 				return Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil)
 			}

v2/pkg/types/types.go

@@ -379,6 +379,11 @@ func (options *Options) ShouldFollowHTTPRedirects() bool {
 	return options.FollowRedirects || options.FollowHostRedirects
 }
 
+// HasClientCertificates determines if any client certificate was specified
+func (options *Options) HasClientCertificates() bool {
+	return options.ClientCertFile != "" || options.ClientCAFile != "" || options.ClientKeyFile != ""
+}
+
 // DefaultOptions returns default options for nuclei
 func DefaultOptions() *Options {
 	return &Options{