scanallip handle edge cases (#3080)

* bug fix:remove port during dns resolution

* scanallip fix edge cases

* add scanallips testcases

* workflow fix

* removing pull cmd

* Auto Generate Syntax Docs + JSONSchema [Sat Dec 24 13:29:21 UTC 2022] 🤖

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
Co-authored-by: GitHub Action <action@github.com>

.github/workflows/publish-docs.yaml

@@ -12,6 +12,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+        with: 
+          fetch-depth: 0
 
       - name: "Set up Go"
         uses: actions/setup-go@v3
@@ -36,7 +38,6 @@ jobs:
         run: |
           git config --local user.email "action@github.com"
           git config --local user.name "GitHub Action"
-          git pull
           git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
           git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
 

SYNTAX-REFERENCE.md

@@ -3519,6 +3519,19 @@ description: |
 
 <div class="dd">
 
+<code>stop-at-first-match</code>  <i>bool</i>
+
+</div>
+<div class="dt">
+
+StopAtFirstMatch stops the execution of the requests and template as soon as a match is found.
+
+</div>
+
+<hr />
+
+<div class="dd">
+
 <code>matchers</code>  <i>[]<a href="#matchersmatcher">matchers.Matcher</a></i>
 
 </div>

nuclei-jsonschema.json

@@ -576,6 +576,11 @@
           "title": "custom user agent for the headless request",
           "description": "Custom user agent for the headless request"
         },
+        "stop-at-first-match": {
+          "type": "boolean",
+          "title": "stop at first match",
+          "description": "Stop the execution after a match is found"
+        },
         "matchers": {
           "items": {
             "$ref": "#/definitions/matchers.Matcher"

v2/pkg/core/inputs/hybrid/hmap.go

@@ -5,7 +5,7 @@ package hybrid
 import (
 	"bufio"
 	"io"
-	"net/url"
+	"net"
 	"os"
 	"strings"
 	"sync"
@@ -22,6 +22,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 	fileutil "github.com/projectdiscovery/utils/file"
 	iputil "github.com/projectdiscovery/utils/ip"
 	readerutil "github.com/projectdiscovery/utils/reader"
@@ -169,39 +170,49 @@ func (i *Input) Set(value string) {
 	if URL == "" {
 		return
 	}
-	// actual hostname
-	var host string
 	// parse hostname if url is given
-	parsedURL, err := url.Parse(value)
+	host := utils.ParseHostname(value)
-	if err == nil && parsedURL.Host != "" {
+	if host == "" {
-		host = parsedURL.Host
+		// not a valid url hence scanallips is skipped
+		gologger.Debug().Msgf("scanAllIps: failed to parse hostname of %v falling back to default", value)
+		i.setItem(&contextargs.MetaInput{Input: value})
+		return
 	} else {
-		parsedURL = nil
+		// case when hostname contains port
-		host = value
+		hostwithoutport, _, erx := net.SplitHostPort(host)
+		if erx == nil && hostwithoutport != "" {
+			// given host contains port
+			host = hostwithoutport
+		}
 	}
 
 	if i.ipOptions.ScanAllIPs {
 		// scan all ips
 		dnsData, err := protocolstate.Dialer.GetDNSData(host)
-		if err == nil && (len(dnsData.A)+len(dnsData.AAAA)) > 0 {
+		if err == nil {
-			var ips []string
+			if (len(dnsData.A) + len(dnsData.AAAA)) > 0 {
-			if i.ipOptions.IPV4 {
+				var ips []string
-				ips = append(ips, dnsData.A...)
+				if i.ipOptions.IPV4 {
-			}
+					ips = append(ips, dnsData.A...)
-			if i.ipOptions.IPV6 {
-				ips = append(ips, dnsData.AAAA...)
-			}
-			for _, ip := range ips {
-				if ip == "" {
-					continue
 				}
-				metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+				if i.ipOptions.IPV6 {
-				i.setItem(metaInput)
+					ips = append(ips, dnsData.AAAA...)
+				}
+				for _, ip := range ips {
+					if ip == "" {
+						continue
+					}
+					metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+					i.setItem(metaInput)
+				}
+				return
+			} else {
+				gologger.Debug().Msgf("scanAllIps: no ip's found reverting to default")
 			}
-			return
+		} else {
+			// failed to scanallips falling back to defaults
+			gologger.Debug().Msgf("scanAllIps: dns resolution failed: %v", err)
 		}
-		// failed to scanallips falling back to defaults
-		gologger.Error().Msgf("failed to scan all ips reverting to default %v", err)
 	}
 
 	ips := []string{}
@@ -212,7 +223,7 @@ func (i *Input) Set(value string) {
 			// pick/ prefer 1st
 			ips = append(ips, dnsData.AAAA[0])
 		} else {
-			gologger.Warning().Msgf("target does not have ipv6 address falling back to ipv4 %s\n", err)
+			gologger.Warning().Msgf("target does not have ipv6 address falling back to ipv4 %v\n", err)
 		}
 	}
 	if i.ipOptions.IPV4 {

v2/pkg/core/inputs/hybrid/hmap_test.go

@@ -51,7 +51,7 @@ func Test_expandCIDRInputValue(t *testing.T) {
 
 type mockDnsHandler struct{}
 
-func (this *mockDnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
+func (m *mockDnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 	msg := dns.Msg{}
 	msg.SetReply(r)
 	switch r.Question[0].Qtype {
@@ -85,18 +85,14 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
 	defaultOpts := types.DefaultOptions()
 	defaultOpts.InternalResolversList = []string{"127.0.0.1:61234"}
 	_ = protocolstate.Init(defaultOpts)
-	tests := []struct {
+	type testcase struct {
 		hostname string
 		ipv4     bool
 		ipv6     bool
 		expected []string
-	}{
+	}
+	tests := []testcase{
 		{
-			hostname: "scanme.sh",
-			ipv4:     true,
-			ipv6:     true,
-			expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
-		}, {
 			hostname: "scanme.sh",
 			ipv4:     true,
 			expected: []string{"128.199.158.128"},
@@ -104,12 +100,27 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
 			hostname: "scanme.sh",
 			ipv6:     true,
 			expected: []string{"2400:6180:0:d0::91:1001"},
-		}, {
+		},
-			hostname: "http://scanme.sh",
+	}
+	// add extra edge cases
+	urls := []string{
+		"https://scanme.sh/",
+		"http://scanme.sh",
+		"https://scanme.sh:443/",
+		"https://scanme.sh:443/somepath",
+		"http://scanme.sh:80/?with=param",
+		"scanme.sh/home",
+		"scanme.sh",
+	}
+	resolvedIps := []string{"128.199.158.128", "2400:6180:0:d0::91:1001"}
+
+	for _, v := range urls {
+		tests = append(tests, testcase{
+			hostname: v,
 			ipv4:     true,
 			ipv6:     true,
-			expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
+			expected: resolvedIps,
-		},
+		})
 	}
 	for _, tt := range tests {
 		hm, err := hybrid.New(hybrid.DefaultDiskOptions)
@@ -134,7 +145,7 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
 			got = append(got, metainput.CustomIP)
 			return nil
 		})
-		require.ElementsMatch(t, tt.expected, got, "could not get correct ips")
+		require.ElementsMatchf(t, tt.expected, got, "could not get correct ips for hostname %v", tt.hostname)
 		input.Close()
 	}
 }

v2/pkg/templates/templates_doc.go

@@ -1542,7 +1542,7 @@ func init() {
 			Value: "Headless response received from client (default)",
 		},
 	}
-	HEADLESSRequestDoc.Fields = make([]encoder.Doc, 9)
+	HEADLESSRequestDoc.Fields = make([]encoder.Doc, 10)
 	HEADLESSRequestDoc.Fields[0].Name = "id"
 	HEADLESSRequestDoc.Fields[0].Type = "string"
 	HEADLESSRequestDoc.Fields[0].Note = ""
@@ -1573,22 +1573,27 @@ func init() {
 	HEADLESSRequestDoc.Fields[5].Note = ""
 	HEADLESSRequestDoc.Fields[5].Description = "description: |\n 	 If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request."
 	HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = " description: |"
-	HEADLESSRequestDoc.Fields[6].Name = "matchers"
+	HEADLESSRequestDoc.Fields[6].Name = "stop-at-first-match"
-	HEADLESSRequestDoc.Fields[6].Type = "[]matchers.Matcher"
+	HEADLESSRequestDoc.Fields[6].Type = "bool"
 	HEADLESSRequestDoc.Fields[6].Note = ""
-	HEADLESSRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+	HEADLESSRequestDoc.Fields[6].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
-	HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+	HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
-	HEADLESSRequestDoc.Fields[7].Name = "extractors"
+	HEADLESSRequestDoc.Fields[7].Name = "matchers"
-	HEADLESSRequestDoc.Fields[7].Type = "[]extractors.Extractor"
+	HEADLESSRequestDoc.Fields[7].Type = "[]matchers.Matcher"
 	HEADLESSRequestDoc.Fields[7].Note = ""
-	HEADLESSRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+	HEADLESSRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
-	HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+	HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
-	HEADLESSRequestDoc.Fields[8].Name = "matchers-condition"
+	HEADLESSRequestDoc.Fields[8].Name = "extractors"
-	HEADLESSRequestDoc.Fields[8].Type = "string"
+	HEADLESSRequestDoc.Fields[8].Type = "[]extractors.Extractor"
 	HEADLESSRequestDoc.Fields[8].Note = ""
-	HEADLESSRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
-	HEADLESSRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
-	HEADLESSRequestDoc.Fields[8].Values = []string{
+	HEADLESSRequestDoc.Fields[9].Name = "matchers-condition"
+	HEADLESSRequestDoc.Fields[9].Type = "string"
+	HEADLESSRequestDoc.Fields[9].Note = ""
+	HEADLESSRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[9].Values = []string{
 		"and",
 		"or",
 	}

v2/pkg/utils/utils.go

@@ -77,3 +77,24 @@ func StringSliceContains(slice []string, item string) bool {
 	}
 	return false
 }
+
+// ParseHostname returns hostname
+func ParseHostname(inputURL string) string {
+	/*
+		currently if URL is scanme.sh/path or scanme.sh:443 i.e without protocol then
+		url.Parse considers this as valid url but fails to parse hostname
+		this can be handled by adding schema
+	*/
+	input, err := url.Parse(inputURL)
+	if err != nil {
+		return ""
+	}
+	if input.Host == "" {
+		newinput, err := url.Parse("https://" + inputURL)
+		if err != nil {
+			return ""
+		}
+		return newinput.Host
+	}
+	return input.Host
+}

v2/pkg/utils/utils_test.go

@@ -19,3 +19,24 @@ func TestUnwrapError(t *testing.T) {
 	errThree := fmt.Errorf("error with error: %w", errTwo)
 	require.Equal(t, errOne, UnwrapError(errThree))
 }
+
+func TestParseURL(t *testing.T) {
+	testcases := []struct {
+		URL      string
+		Hostname string
+	}{
+		{"https://scanme.sh:443", "scanme.sh:443"},
+		{"http://scanme.sh/path", "scanme.sh"},
+		{"scanme.sh:443/path", "scanme.sh:443"},
+		{"scanme.sh/path", "scanme.sh"},
+	}
+	for _, v := range testcases {
+		urlx := ParseHostname(v.URL)
+		if urlx == "" {
+			t.Errorf("failed to hostname of url %v", v)
+		}
+		if urlx != v.Hostname {
+			t.Errorf("hostname mismatch expected scanme.sh got %v", urlx)
+		}
+	}
+}