File protocol tests + misc fixes

v2/pkg/protocols/dns/operators_test.go

@@ -199,21 +199,17 @@ func TestDNSMakeResult(t *testing.T) {
 		Recursion: false,
 		Name:      "{{FQDN}}",
 		Operators: operators.Operators{
-			Matchers: []*matchers.Matcher{
+			Matchers: []*matchers.Matcher{{
-				&matchers.Matcher{
+				Name:  "test",
-					Name:  "test",
+				Part:  "raw",
-					Part:  "raw",
+				Type:  "word",
-					Type:  "word",
+				Words: []string{"1.1.1.1"},
-					Words: []string{"1.1.1.1"},
+			}},
-				},
+			Extractors: []*extractors.Extractor{{
-			},
+				Part:  "raw",
-			Extractors: []*extractors.Extractor{
+				Type:  "regex",
-				&extractors.Extractor{
+				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
-					Part:  "raw",
+			}},
-					Type:  "regex",
-					Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
-				},
-			},
 		},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{

v2/pkg/protocols/dns/request_test.go

@@ -24,21 +24,17 @@ func TestDNSExecuteWithResults(t *testing.T) {
 		Recursion: false,
 		Name:      "{{FQDN}}",
 		Operators: operators.Operators{
-			Matchers: []*matchers.Matcher{
+			Matchers: []*matchers.Matcher{{
-				&matchers.Matcher{
+				Name:  "test",
-					Name:  "test",
+				Part:  "raw",
-					Part:  "raw",
+				Type:  "word",
-					Type:  "word",
+				Words: []string{"93.184.216.34"},
-					Words: []string{"93.184.216.34"},
+			}},
-				},
+			Extractors: []*extractors.Extractor{{
-			},
+				Part:  "raw",
-			Extractors: []*extractors.Extractor{
+				Type:  "regex",
-				&extractors.Extractor{
+				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
-					Part:  "raw",
+			}},
-					Type:  "regex",
-					Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
-				},
-			},
 		},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{

v2/pkg/protocols/file/file_test.go

@@ -0,0 +1,33 @@
+package file
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileCompile(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-file"
+	request := &Request{
+		ID:                 templateID,
+		MaxSize:            1024,
+		NoRecursive:        false,
+		Extensions:         []string{"*"},
+		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist:  []string{".go"},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile file request")
+
+	require.Contains(t, request.extensionDenylist, ".go", "could not get .go in denylist")
+	require.NotContains(t, request.extensions, ".go", "could get .go in allowlist")
+	require.True(t, request.allExtensions, "could not get correct allExtensions")
+}

v2/pkg/protocols/file/find.go

@@ -87,6 +87,9 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru
 			return godirwalk.SkipNode
 		},
 		Callback: func(path string, d *godirwalk.Dirent) error {
+			if d.IsDir() {
+				return nil
+			}
 			if !r.validatePath(path) {
 				return nil
 			}

v2/pkg/protocols/file/find_test.go

@@ -0,0 +1,64 @@
+package file
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindInputPaths(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-file"
+	request := &Request{
+		ID:                 templateID,
+		MaxSize:            1024,
+		NoRecursive:        false,
+		Extensions:         []string{"*"},
+		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist:  []string{".go"},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile file request")
+
+	tempDir, err := ioutil.TempDir("", "test-*")
+	require.Nil(t, err, "could not create temporary directory")
+	defer os.RemoveAll(tempDir)
+
+	files := map[string]string{
+		"test.go":           "TEST",
+		"config.yaml":       "TEST",
+		"final.yaml":        "TEST",
+		"image_ignored.png": "TEST",
+		"test.js":           "TEST",
+	}
+	for k, v := range files {
+		err = ioutil.WriteFile(path.Join(tempDir, k), []byte(v), 0777)
+		require.Nil(t, err, "could not write temporary file")
+	}
+	expected := []string{"config.yaml", "final.yaml", "test.js"}
+	got := []string{}
+	err = request.getInputPaths(tempDir+"/*", func(item string) {
+		base := path.Base(item)
+		got = append(got, base)
+	})
+	require.Nil(t, err, "could not get input paths for glob")
+	require.ElementsMatch(t, expected, got, "could not get correct file matches for glob")
+
+	got = []string{}
+	err = request.getInputPaths(tempDir, func(item string) {
+		base := path.Base(item)
+		got = append(got, base)
+	})
+	require.Nil(t, err, "could not get input paths for directory")
+	require.ElementsMatch(t, expected, got, "could not get correct file matches for directory")
+}

v2/pkg/protocols/file/operators.go

@@ -11,7 +11,7 @@ import (
 func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
 	partString := matcher.Part
 	switch partString {
-	case "body", "all", "":
+	case "body", "all", "data", "":
 		partString = "raw"
 	}
 
@@ -38,14 +38,9 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
 
 // Extract performs extracting operation for a extractor on model and returns true or false.
 func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
-	part, ok := data[extractor.Part]
+	partString := extractor.Part
-	if !ok {
-		return nil
-	}
-	partString := part.(string)
-
 	switch partString {
-	case "body", "all", "":
+	case "body", "all", "data", "":
 		partString = "raw"
 	}
 
@@ -66,7 +61,7 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
 
 // responseToDSLMap converts a DNS response to a map for use in DSL matching
 func (r *Request) responseToDSLMap(raw string, host, matched string) output.InternalEvent {
-	data := make(output.InternalEvent, 3)
+	data := make(output.InternalEvent, 5)
 
 	// Some data regarding the request metadata
 	data["host"] = host

v2/pkg/protocols/file/operators_test.go

@@ -0,0 +1,208 @@
+package file
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResponseToDSLMap(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-file"
+	request := &Request{
+		ID:                 templateID,
+		MaxSize:            1024,
+		NoRecursive:        false,
+		Extensions:         []string{"*"},
+		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist:  []string{".go"},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile file request")
+
+	resp := "test-data\r\n"
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
+	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Equal(t, resp, event["raw"], "could not get correct resp")
+}
+
+func TestFileOperatorMatch(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-file"
+	request := &Request{
+		ID:                 templateID,
+		MaxSize:            1024,
+		NoRecursive:        false,
+		Extensions:         []string{"*"},
+		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist:  []string{".go"},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile file request")
+
+	resp := "test-data\r\n1.1.1.1\r\n"
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
+	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Equal(t, resp, event["raw"], "could not get correct resp")
+
+	t.Run("valid", func(t *testing.T) {
+		matcher := &matchers.Matcher{
+			Part:  "raw",
+			Type:  "word",
+			Words: []string{"1.1.1.1"},
+		}
+		err = matcher.CompileMatchers()
+		require.Nil(t, err, "could not compile matcher")
+
+		matched := request.Match(event, matcher)
+		require.True(t, matched, "could not match valid response")
+	})
+
+	t.Run("negative", func(t *testing.T) {
+		matcher := &matchers.Matcher{
+			Part:     "raw",
+			Type:     "word",
+			Negative: true,
+			Words:    []string{"random"},
+		}
+		err := matcher.CompileMatchers()
+		require.Nil(t, err, "could not compile negative matcher")
+
+		matched := request.Match(event, matcher)
+		require.True(t, matched, "could not match valid negative response matcher")
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		matcher := &matchers.Matcher{
+			Part:  "raw",
+			Type:  "word",
+			Words: []string{"random"},
+		}
+		err := matcher.CompileMatchers()
+		require.Nil(t, err, "could not compile matcher")
+
+		matched := request.Match(event, matcher)
+		require.False(t, matched, "could match invalid response matcher")
+	})
+}
+
+func TestFileOperatorExtract(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-file"
+	request := &Request{
+		ID:                 templateID,
+		MaxSize:            1024,
+		NoRecursive:        false,
+		Extensions:         []string{"*"},
+		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist:  []string{".go"},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile file request")
+
+	resp := "test-data\r\n1.1.1.1\r\n"
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
+	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Equal(t, resp, event["raw"], "could not get correct resp")
+
+	t.Run("extract", func(t *testing.T) {
+		extractor := &extractors.Extractor{
+			Part:  "raw",
+			Type:  "regex",
+			Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
+		}
+		err = extractor.CompileExtractors()
+		require.Nil(t, err, "could not compile extractor")
+
+		data := request.Extract(event, extractor)
+		require.Greater(t, len(data), 0, "could not extractor valid response")
+		require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data")
+	})
+
+	t.Run("kval", func(t *testing.T) {
+		extractor := &extractors.Extractor{
+			Type: "kval",
+			KVal: []string{"raw"},
+		}
+		err = extractor.CompileExtractors()
+		require.Nil(t, err, "could not compile kval extractor")
+
+		data := request.Extract(event, extractor)
+		require.Greater(t, len(data), 0, "could not extractor kval valid response")
+		require.Equal(t, map[string]struct{}{resp: {}}, data, "could not extract correct kval data")
+	})
+}
+
+func TestFileMakeResult(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-file"
+	request := &Request{
+		ID:                 templateID,
+		MaxSize:            1024,
+		NoRecursive:        false,
+		Extensions:         []string{"*"},
+		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist:  []string{".go"},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Name:  "test",
+				Part:  "raw",
+				Type:  "word",
+				Words: []string{"1.1.1.1"},
+			}},
+			Extractors: []*extractors.Extractor{{
+				Part:  "raw",
+				Type:  "regex",
+				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
+			}},
+		},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile file request")
+
+	resp := "test-data\r\n1.1.1.1\r\n"
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
+	require.Len(t, event, 5, "could not get correct number of items in dsl map")
+	require.Equal(t, resp, event["raw"], "could not get correct resp")
+
+	finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
+	if request.CompiledOperators != nil {
+		result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
+		if ok && result != nil {
+			finalEvent.OperatorsResult = result
+			finalEvent.Results = request.MakeResultEvent(finalEvent)
+		}
+	}
+	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
+	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
+	require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+}

v2/pkg/protocols/file/request.go

@@ -47,7 +47,6 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int
 				return
 			}
 			dataStr := tostring.UnsafeToString(buffer)
-
 			if r.options.Options.Debug || r.options.Options.DebugRequests {
 				gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data)
 				gologger.Print().Msgf("%s", dataStr)

v2/pkg/protocols/file/request_test.go

@@ -0,0 +1,77 @@
+package file
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileExecuteWithResults(t *testing.T) {
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-file"
+	request := &Request{
+		ID:                 templateID,
+		MaxSize:            1024,
+		NoRecursive:        false,
+		Extensions:         []string{"*"},
+		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist:  []string{".go"},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Name:  "test",
+				Part:  "raw",
+				Type:  "word",
+				Words: []string{"1.1.1.1"},
+			}},
+			Extractors: []*extractors.Extractor{{
+				Part:  "raw",
+				Type:  "regex",
+				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
+			}},
+		},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: map[string]string{"severity": "low", "name": "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile file request")
+
+	tempDir, err := ioutil.TempDir("", "test-*")
+	require.Nil(t, err, "could not create temporary directory")
+	defer os.RemoveAll(tempDir)
+
+	files := map[string]string{
+		"config.yaml": "TEST\r\n1.1.1.1\r\n",
+	}
+	for k, v := range files {
+		err = ioutil.WriteFile(path.Join(tempDir, k), []byte(v), 0777)
+		require.Nil(t, err, "could not write temporary file")
+	}
+
+	var finalEvent *output.InternalWrappedEvent
+	t.Run("valid", func(t *testing.T) {
+		metadata := make(output.InternalEvent)
+		previous := make(output.InternalEvent)
+		err := request.ExecuteWithResults(tempDir, metadata, previous, func(event *output.InternalWrappedEvent) {
+			finalEvent = event
+		})
+		require.Nil(t, err, "could not execute file request")
+	})
+	require.NotNil(t, finalEvent, "could not get event output from request")
+	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
+	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
+	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
+	require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	finalEvent = nil
+}

v2/pkg/protocols/network/network_test.go

@@ -16,7 +16,7 @@ func TestNetworkCompileMake(t *testing.T) {
 		ID:       templateID,
 		Address:  []string{"{{Hostname}}", "{{Hostname}}:8082"},
 		ReadSize: 1024,
-		Inputs:   []*Input{&Input{Data: "test-data"}},
+		Inputs:   []*Input{{Data: "test-data"}},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,

v2/pkg/protocols/network/operators_test.go

@@ -20,7 +20,7 @@ func TestResponseToDSLMap(t *testing.T) {
 		ID:       templateID,
 		Address:  []string{"{{Hostname}}"},
 		ReadSize: 1024,
-		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+		Inputs:   []*Input{{Data: "test-data\r\n"}},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -45,7 +45,7 @@ func TestNetworkOperatorMatch(t *testing.T) {
 		ID:       templateID,
 		Address:  []string{"{{Hostname}}"},
 		ReadSize: 1024,
-		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+		Inputs:   []*Input{{Data: "test-data\r\n"}},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -108,7 +108,7 @@ func TestNetworkOperatorExtract(t *testing.T) {
 		ID:       templateID,
 		Address:  []string{"{{Hostname}}"},
 		ReadSize: 1024,
-		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+		Inputs:   []*Input{{Data: "test-data\r\n"}},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -158,23 +158,19 @@ func TestNetworkMakeResult(t *testing.T) {
 		ID:       templateID,
 		Address:  []string{"{{Hostname}}"},
 		ReadSize: 1024,
-		Inputs:   []*Input{&Input{Data: "test-data\r\n"}},
+		Inputs:   []*Input{{Data: "test-data\r\n"}},
 		Operators: operators.Operators{
-			Matchers: []*matchers.Matcher{
+			Matchers: []*matchers.Matcher{{
-				&matchers.Matcher{
+				Name:  "test",
-					Name:  "test",
+				Part:  "raw",
-					Part:  "raw",
+				Type:  "word",
-					Type:  "word",
+				Words: []string{"STAT "},
-					Words: []string{"STAT "},
+			}},
-				},
+			Extractors: []*extractors.Extractor{{
-			},
+				Part:  "raw",
-			Extractors: []*extractors.Extractor{
+				Type:  "regex",
-				&extractors.Extractor{
+				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
-					Part:  "raw",
+			}},
-					Type:  "regex",
-					Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
-				},
-			},
 		},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{

v2/pkg/protocols/network/request_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestDNSExecuteWithResults(t *testing.T) {
+func TestNetworkExecuteWithResults(t *testing.T) {
 	options := testutils.DefaultOptions
 
 	testutils.Init(options)
@@ -21,23 +21,19 @@ func TestDNSExecuteWithResults(t *testing.T) {
 		ID:       templateID,
 		Address:  []string{"{{Hostname}}:80"},
 		ReadSize: 2048,
-		Inputs:   []*Input{&Input{Data: "GET / HTTP/1.1\r\n\r\n"}},
+		Inputs:   []*Input{{Data: "GET / HTTP/1.1\r\n\r\n"}},
 		Operators: operators.Operators{
-			Matchers: []*matchers.Matcher{
+			Matchers: []*matchers.Matcher{{
-				&matchers.Matcher{
+				Name:  "test",
-					Name:  "test",
+				Part:  "raw",
-					Part:  "raw",
+				Type:  "word",
-					Type:  "word",
+				Words: []string{"400 - Bad Request"},
-					Words: []string{"400 - Bad Request"},
+			}},
-				},
+			Extractors: []*extractors.Extractor{{
-			},
+				Part:  "raw",
-			Extractors: []*extractors.Extractor{
+				Type:  "regex",
-				&extractors.Extractor{
+				Regex: []string{"<h1>.*</h1>"},
-					Part:  "raw",
+			}},
-					Type:  "regex",
-					Regex: []string{"<h1>.*</h1>"},
-				},
-			},
 		},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{