Fixing directory walk error check on windows (#1951)

* Fixing directory walk error check on windows

* moving check to helper package

* replacing godirwalk with standard library

v2/cmd/nuclei/main.go

@@ -235,6 +235,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	}
 	cleanupOldResumeFiles()
 }
+
 func cleanupOldResumeFiles() {
 	root, err := config.GetConfigDir()
 	if err != nil {
@@ -246,6 +247,7 @@ func cleanupOldResumeFiles() {
 	}
 	_ = fileutil.DeleteFilesOlderThan(root, filter)
 }
+
 func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
 	flagSet.SetGroup(groupName, description)
 	for _, currentFlag := range flags {

v2/go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karlseguin/ccache v2.0.3+incompatible
-	github.com/karrick/godirwalk v1.17.0
+	github.com/karrick/godirwalk v1.17.0 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/miekg/dns v1.1.48
 	github.com/olekukonko/tablewriter v0.0.5

v2/internal/runner/templates.go

@@ -1,11 +1,11 @@
 package runner
 
 import (
+	"io/fs"
 	"os"
+	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -40,9 +40,13 @@ func (r *Runner) listAvailableTemplates() {
 		r.templatesConfig.TemplateVersion,
 		r.templatesConfig.TemplatesDirectory,
 	)
-	err := directoryWalker(
+	err := filepath.WalkDir(
 		r.templatesConfig.TemplatesDirectory,
-		func(path string, d *godirwalk.Dirent) error {
+		func(path string, d fs.DirEntry, err error) error {
+			// continue on errors
+			if err != nil {
+				return nil
+			}
 			if d.IsDir() && path != r.templatesConfig.TemplatesDirectory {
 				gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String())
 			} else if strings.HasSuffix(path, ".yaml") {
@@ -56,13 +60,3 @@ func (r *Runner) listAvailableTemplates() {
 		gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err)
 	}
 }
-
-func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
-	return godirwalk.Walk(fsPath, &godirwalk.Options{
-		Callback: callback,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
-			return godirwalk.SkipNode
-		},
-		Unsorted: true,
-	})
-}

v2/pkg/catalog/find.go

@@ -1,11 +1,11 @@
 package catalog
 
 import (
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/gologger"
@@ -136,12 +136,13 @@ func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{})
 // findDirectoryMatches finds matches for templates from a directory
 func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
 	var results []string
-	err := godirwalk.Walk(absPath, &godirwalk.Options{
+	err := filepath.WalkDir(
-		Unsorted: true,
+		absPath,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+		func(path string, d fs.DirEntry, err error) error {
-			return godirwalk.SkipNode
+			// continue on errors
-		},
+			if err != nil {
-		Callback: func(path string, d *godirwalk.Dirent) error {
+				return nil
+			}
 			if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
 				if _, ok := processed[path]; !ok {
 					results = append(results, path)
@@ -150,6 +151,6 @@ func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]stru
 			}
 			return nil
 		},
-	})
+	)
 	return results, err
 }

v2/pkg/protocols/common/compare/compare.go

@@ -1,6 +1,8 @@
 package compare
 
-import "strings"
+import (
+	"strings"
+)
 
 // StringSlice compares two string slices for equality
 func StringSlice(a, b []string) bool {

v2/pkg/protocols/file/find.go

@@ -2,11 +2,11 @@ package file
 
 import (
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/folderutil"
@@ -86,12 +86,13 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
 
 // findDirectoryMatches finds matches for templates from a directory
 func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
-	err := godirwalk.Walk(absPath, &godirwalk.Options{
+	err := filepath.WalkDir(
-		Unsorted: true,
+		absPath,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+		func(path string, d fs.DirEntry, err error) error {
-			return godirwalk.SkipNode
+			// continue on errors
-		},
+			if err != nil {
-		Callback: func(path string, d *godirwalk.Dirent) error {
+				return nil
+			}
 			if d.IsDir() {
 				return nil
 			}
@@ -104,7 +105,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 			}
 			return nil
 		},
-	})
+	)
 	return err
 }
 

v2/pkg/protocols/offlinehttp/find.go

@@ -1,11 +1,11 @@
 package offlinehttp
 
 import (
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 )
 
@@ -80,12 +80,13 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
 
 // findDirectoryMatches finds matches for templates from a directory
 func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
-	err := godirwalk.Walk(absPath, &godirwalk.Options{
+	err := filepath.WalkDir(
-		Unsorted: true,
+		absPath,
-		ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
+		func(p string, d fs.DirEntry, err error) error {
-			return godirwalk.SkipNode
+			// continue on errors
-		},
+			if err != nil {
-		Callback: func(p string, d *godirwalk.Dirent) error {
+				return nil
+			}
 			if d.IsDir() {
 				return nil
 			}
@@ -98,6 +99,6 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
 			}
 			return nil
 		},
-	})
+	)
 	return err
 }