Misc changes to update and ignore handling

v2/go.mod

@@ -107,7 +107,7 @@ require (
 	github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 // indirect
 	github.com/projectdiscovery/mapcidr v0.0.6 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
-	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf // indirect
+	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.7 // indirect
 	github.com/tklauser/numcpus v0.2.3 // indirect

v2/go.sum

@@ -363,6 +363,8 @@ github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094006-ee345cd
 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094006-ee345cd25b9b/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf h1:cFUSV7HERosPKdHuPiV348eVxObVMBqNt0ydYVplsI0=
 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ=
+github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
 github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
 github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=

v2/internal/runner/options.go

@@ -36,7 +36,7 @@ func ParseOptions(options *types.Options) {
 		if err != nil {
 			gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
 		}
-		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory)
+		gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory)
 		os.Exit(0)
 	}
 

v2/internal/runner/runner.go

@@ -392,7 +392,7 @@ func (r *Runner) RunEnumeration() error {
 	if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
 		builder.WriteString(" (")
 
-		if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
+		if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
 			builder.WriteString(r.colorizer.Green("latest").String())
 		} else {
 			builder.WriteString(r.colorizer.Red("outdated").String())
@@ -403,7 +403,7 @@ func (r *Runner) RunEnumeration() error {
 	builder.Reset()
 
 	if r.templatesConfig != nil {
-		gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
+		gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr)
 	}
 	if r.interactsh != nil {
 		gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)

v2/internal/runner/templates.go

@@ -70,7 +70,7 @@ func (r *Runner) listAvailableTemplates() {
 
 	gologger.Print().Msgf(
 		"\nListing available v.%s nuclei templates for %s",
-		r.templatesConfig.CurrentVersion,
+		r.templatesConfig.TemplateVersion,
 		r.templatesConfig.TemplatesDirectory,
 	)
 	err := directoryWalker(

v2/internal/runner/update.go

@@ -17,7 +17,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/apex/log"
 	"github.com/blang/semver"
@@ -66,7 +65,7 @@ func (r *Runner) updateTemplates() error {
 			TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
 			NucleiVersion:      config.Version,
 		}
-		if writeErr := config.WriteConfiguration(currentConfig, false); writeErr != nil {
+		if writeErr := config.WriteConfiguration(currentConfig); writeErr != nil {
 			return errors.Wrap(writeErr, "could not write template configuration")
 		}
 		r.templatesConfig = currentConfig
@@ -76,17 +75,7 @@ func (r *Runner) updateTemplates() error {
 		return nil
 	}
 	client.InitNucleiVersion(config.Version)
-	r.fetchLatestVersionsFromGithub() // also fetch latest versions
+	r.fetchLatestVersionsFromGithub(configDir) // also fetch latest versions
-
-	// Check if last checked for nuclei-ignore is more than 1 hours.
-	// and if true, run the check.
-	//
-	// Also at the same time fetch latest version from github to do outdated nuclei
-	// and templates check.
-	checkedIgnore := false
-	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour {
-		checkedIgnore = r.checkNucleiIgnoreFileUpdates(configDir)
-	}
 
 	ctx := context.Background()
 
@@ -95,7 +84,7 @@ func (r *Runner) updateTemplates() error {
 		noTemplatesFound = true
 	}
 
-	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound {
+	if r.templatesConfig.TemplateVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound {
 		gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
 
 		// Use custom location if user has given a template directory
@@ -105,7 +94,7 @@ func (r *Runner) updateTemplates() error {
 		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
 			r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory)
 		}
-		r.fetchLatestVersionsFromGithub() // also fetch latest versions
+		r.fetchLatestVersionsFromGithub(configDir) // also fetch latest versions
 
 		version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
 		if err != nil {
@@ -123,9 +112,9 @@ func (r *Runner) updateTemplates() error {
 		if err != nil {
 			return err
 		}
-		r.templatesConfig.CurrentVersion = version.String()
+		r.templatesConfig.TemplateVersion = version.String()
 
-		err = config.WriteConfiguration(r.templatesConfig, checkedIgnore)
+		err = config.WriteConfiguration(r.templatesConfig)
 		if err != nil {
 			return err
 		}
@@ -134,7 +123,7 @@ func (r *Runner) updateTemplates() error {
 	}
 
 	// Get the configuration currently on disk.
-	verText := r.templatesConfig.CurrentVersion
+	verText := r.templatesConfig.TemplateVersion
 	indices := reVersion.FindStringIndex(verText)
 	if indices == nil {
 		return fmt.Errorf("invalid release found with tag %s", err)
@@ -157,7 +146,7 @@ func (r *Runner) updateTemplates() error {
 		if r.options.UpdateTemplates {
 			gologger.Info().Msgf("No new updates found for nuclei templates")
 		}
-		return config.WriteConfiguration(r.templatesConfig, checkedIgnore)
+		return config.WriteConfiguration(r.templatesConfig)
 	}
 
 	if version.GT(oldVersion) {
@@ -167,7 +156,7 @@ func (r *Runner) updateTemplates() error {
 		if r.options.TemplatesDirectory != "" {
 			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
-		r.templatesConfig.CurrentVersion = version.String()
+		r.templatesConfig.TemplateVersion = version.String()
 
 		gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
 
@@ -179,7 +168,7 @@ func (r *Runner) updateTemplates() error {
 		if err != nil {
 			return err
 		}
-		err = config.WriteConfiguration(r.templatesConfig, checkedIgnore)
+		err = config.WriteConfiguration(r.templatesConfig)
 		if err != nil {
 			return err
 		}
@@ -215,7 +204,7 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
 		_ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644)
 	}
 	if r.templatesConfig != nil {
-		if err := config.WriteConfiguration(r.templatesConfig, true); err != nil {
+		if err := config.WriteConfiguration(r.templatesConfig); err != nil {
 			gologger.Warning().Msgf("Could not get ignore-file from server: %s", err)
 		}
 	}
@@ -473,7 +462,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
 }
 
 // fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
-func (r *Runner) fetchLatestVersionsFromGithub() {
+func (r *Runner) fetchLatestVersionsFromGithub(configDir string) {
 	versions, err := client.GetLatestNucleiTemplatesVersion()
 	if err != nil {
 		gologger.Warning().Msgf("Could not fetch latest releases: %s", err)
@@ -482,6 +471,12 @@ func (r *Runner) fetchLatestVersionsFromGithub() {
 	if r.templatesConfig != nil {
 		r.templatesConfig.NucleiLatestVersion = versions.Nuclei
 		r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates
+
+		// If the fetch has resulted in new version of ignore file, update.
+		if r.templatesConfig.NucleiIgnoreHash == "" || r.templatesConfig.NucleiIgnoreHash != versions.IgnoreHash {
+			r.templatesConfig.NucleiIgnoreHash = versions.IgnoreHash
+			r.checkNucleiIgnoreFileUpdates(configDir)
+		}
 	}
 }
 

v2/pkg/catalog/config/config.go

@@ -3,7 +3,6 @@ package config
 import (
 	"os"
 	"path/filepath"
-	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
@@ -13,10 +12,10 @@ import (
 
 // Config contains the internal nuclei engine configuration
 type Config struct {
-	TemplatesDirectory string    `json:"templates-directory,omitempty"`
+	TemplatesDirectory string `json:"templates-directory,omitempty"`
-	CurrentVersion     string    `json:"current-version,omitempty"`
+	TemplateVersion    string `json:"template-version,omitempty"`
-	NucleiVersion      string    `json:"nuclei-version,omitempty"`
+	NucleiVersion      string `json:"nuclei-version,omitempty"`
-	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
+	NucleiIgnoreHash   string `json:"nuclei-ignore-hash,omitempty"`
 
 	NucleiLatestVersion          string `json:"nuclei-latest-version"`
 	NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
@@ -61,10 +60,7 @@ func ReadConfiguration() (*Config, error) {
 }
 
 // WriteConfiguration writes the updated nuclei configuration to disk
-func WriteConfiguration(config *Config, checkedIgnore bool) error {
+func WriteConfiguration(config *Config) error {
-	if checkedIgnore {
-		config.LastCheckedIgnore = time.Now()
-	}
 	config.NucleiVersion = Version
 
 	templatesConfigFile, err := getConfigDetails()