fix relative path issue + remove residual code (#4284)

* fix relative path issue + remove residual code

* use template dir in templateFS

* fix dir relative path issue

* print metrics server address in verbose mode

* add timeout for downloading binary & templates

* update stats & metrics docs

* add template-id loader integration test

.gitignore

@@ -19,9 +19,9 @@ pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
 .gitignore
 pkg/js/devtools/bindgen/cmd/bindgen
 pkg/js/devtools/jsdocgen/jsdocgen
-./bindgen
-./jsdocgen
-./scrapefuncs
 *.DS_Store
 pkg/protocols/headless/engine/.cache
-./nuclei
+/nuclei
+/bindgen
+/jsdocgen
+/scrapefuncs

cmd/integration-test/loader.go

@@ -10,6 +10,7 @@ import (
 	"github.com/julienschmidt/httprouter"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+	errorutil "github.com/projectdiscovery/utils/errors"
 	permissionutil "github.com/projectdiscovery/utils/permission"
 )
 
@@ -20,6 +21,7 @@ var loaderTestcases = []TestCaseInfo{
 	{Path: "loader/nonexistent-template-list.yaml", TestCase: &nonExistentTemplateList{}},
 	{Path: "loader/nonexistent-workflow-list.yaml", TestCase: &nonExistentWorkflowList{}},
 	{Path: "loader/template-list-not-allowed.yaml", TestCase: &remoteTemplateListNotAllowed{}},
+	{Path: "loader/load-template-with-id", TestCase: &loadTemplateWithID{}},
 }
 
 type remoteTemplateList struct{}
@@ -193,3 +195,13 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 
 	return nil
 }
+
+type loadTemplateWithID struct{}
+
+func (h *loadTemplateWithID) Execute(nooop string) error {
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", "scanme.sh", "-id", "self-signed-ssl")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	return expectResultsCount(results, 1)
+}

cmd/nuclei/main.go

@@ -32,6 +32,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/monitor"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
+	updateutils "github.com/projectdiscovery/utils/update"
 )
 
 var (
@@ -158,6 +159,10 @@ func main() {
 }
 
 func readConfig() *goflags.FlagSet {
+
+	// when true updates nuclei binary to latest version
+	var updateNucleiBinary bool
+
 	flagSet := goflags.NewFlagSet()
 	flagSet.CaseSensitive = true
 	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
@@ -342,7 +347,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	)
 
 	flagSet.CreateGroup("update", "Update",
-		flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "up", "update nuclei engine to the latest released version"),
+		flagSet.BoolVarP(&updateNucleiBinary, "update", "up", false, "update nuclei engine to the latest released version"),
 		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"),
 		flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"),
 		flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"),
@@ -412,6 +417,14 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		installer.HideReleaseNotes = false
 	}
 
+	if options.Timeout > 30 {
+		// default github binary/template download timeout is 30 sec
+		updateutils.DownloadUpdateTimeout = time.Duration(options.Timeout) * time.Second
+	}
+	if updateNucleiBinary {
+		runner.NucleiToolUpdateCallback()
+	}
+
 	if options.LeaveDefaultPorts {
 		http.LeaveDefaultPorts = true
 	}

docs/getting-started/running.mdx

@@ -402,10 +402,15 @@ STATISTICS:
    -stats                    display statistics about the running scan
    -sj, -stats-json          write statistics data to an output file in JSONL(ines) format
    -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
-   -m, -metrics              expose nuclei metrics on a port
    -mp, -metrics-port int    port to expose nuclei metrics on (default 9092)
 ```
 
+<Tip>
+  From Nuclei v3.0.0 `-metrics` port has been removed and merged with `-stats`
+  when using `-stats` flag metrics will be by default available at `localhost:9092/metrics`
+  and metrics-port can be configured by `-metrics-port` flag
+</Tip>
+
 ### Rate **Limits**
 
 Nuclei have multiple rate limit controls for multiple factors, including a number of templates to execute in parallel, a number of hosts to be scanned in parallel for each template, and the global number of request / per second you wanted to make/limit using nuclei, here is an example of each flag with description.

pkg/catalog/config/constants.go

@@ -17,7 +17,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.0.1`
+	Version = `v3.0.2-dev`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/catalog/disk/catalog.go

@@ -4,6 +4,8 @@ import (
 	"io"
 	"io/fs"
 	"os"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 )
 
 // DiskCatalog is a template catalog helper implementation based on disk
@@ -19,7 +21,7 @@ func NewCatalog(directory string) *DiskCatalog {
 	if directory != "" {
 		catalog.templatesFS = os.DirFS(directory)
 	} else {
-		catalog.templatesFS = os.DirFS("./")
+		catalog.templatesFS = os.DirFS(config.DefaultConfig.GetTemplateDir())
 	}
 	return catalog
 }

pkg/catalog/disk/path.go

@@ -50,7 +50,7 @@ var errNoValidCombination = errors.New("no valid combination found")
 
 // tryResolve attempts to load locate the target by iterating across all the folders tree
 func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
-	if fileutil.FileExists(fullPath) {
+	if fileutil.FileOrFolderExists(fullPath) {
 		return fullPath, nil
 	}
 	return "", errNoValidCombination

pkg/catalog/loader/loader.go

@@ -159,6 +159,7 @@ func New(config *Config) (*Store, error) {
 		if _, err := urlutil.Parse(v); err == nil {
 			remoteTemplates = append(remoteTemplates, handleTemplatesEditorURLs(v))
 		} else {
+
 			templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file
 		}
 	}
@@ -187,6 +188,7 @@ func New(config *Config) (*Store, error) {
 	if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided {
 		store.finalTemplates = []string{cfg.DefaultConfig.TemplatesDirectory}
 	}
+
 	return store, nil
 }
 

pkg/progress/progress.go

@@ -65,6 +65,8 @@ func NewStatsTicker(duration int, active, outputJSON, cloud bool, port int) (Pro
 	if err != nil {
 		return nil, err
 	}
+	// only print in verbose mode
+	gologger.Verbose().Msgf("Started metrics server at localhost:%v", stats.Options.ListenPort)
 	progress.cloud = cloud
 	progress.active = active
 	progress.stats = stats

pkg/protocols/common/generators/generators.go

@@ -28,16 +28,6 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
 	for name, payload := range payloads {
 		payloadsFinal[name] = payload
 	}
-	for name, payload := range payloads {
-		payloadStr, ok := payload.(string)
-		if ok {
-			final, resolveErr := catalog.ResolvePath(payloadStr, templatePath)
-			if resolveErr != nil {
-				return nil, errors.Wrap(resolveErr, "could not read payload file")
-			}
-			payloadsFinal[name] = final
-		}
-	}
 
 	generator := &PayloadGenerator{catalog: catalog, options: opts}
 	if err := generator.validate(payloadsFinal, templatePath); err != nil {

pkg/protocols/common/generators/validate.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
@@ -25,7 +26,20 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
 			if fileutil.FileExists(payloadType) {
 				continue
 			}
+			// if file already exists in nuclei-templates directory, skip any further checks
+			if fileutil.FileExists(filepath.Join(config.DefaultConfig.GetTemplateDir(), payloadType)) {
+				continue
+			}
 
+			// in below code, we calculate all possible paths from root and try to resolve the payload
+			// at each level of the path. if the payload is found, we break the loop and continue
+			// ex: template-path: /home/user/nuclei-templates/cves/2020/CVE-2020-1234.yaml
+			// then we check if helper file "my-payload.txt" exists at below paths:
+			// 1. /home/user/nuclei-templates/cves/2020/my-payload.txt
+			// 2. /home/user/nuclei-templates/cves/my-payload.txt
+			// 3. /home/user/nuclei-templates/my-payload.txt
+			// 4. /home/user/my-payload.txt
+			// 5. /home/my-payload.txt
 			changed := false
 
 			dir, _ := filepath.Split(templatePath)