diff --git a/v2/go.mod b/v2/go.mod index f60abc8d4..414c302bf 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -107,7 +107,7 @@ require ( github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 // indirect github.com/projectdiscovery/mapcidr v0.0.6 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect - github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf // indirect + github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect diff --git a/v2/go.sum b/v2/go.sum index 6d57247e3..85a49b55e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -363,6 +363,8 @@ github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094006-ee345cd github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094006-ee345cd25b9b/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf h1:cFUSV7HERosPKdHuPiV348eVxObVMBqNt0ydYVplsI0= github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index b72e1a7f1..2d2499a37 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -36,7 +36,7 @@ func ParseOptions(options *types.Options) { if err != nil { gologger.Fatal().Msgf("Could not read template configuration: %s\n", err) } - gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory) + gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory) os.Exit(0) } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 0989fe020..b0a6afb77 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -392,7 +392,7 @@ func (r *Runner) RunEnumeration() error { if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic builder.WriteString(" (") - if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion { + if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion { builder.WriteString(r.colorizer.Green("latest").String()) } else { builder.WriteString(r.colorizer.Red("outdated").String()) @@ -403,7 +403,7 @@ func (r *Runner) RunEnumeration() error { builder.Reset() if r.templatesConfig != nil { - gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr) + gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr) } if r.interactsh != nil { gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL) diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 8e5f8d768..61f819ef6 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -70,7 +70,7 @@ func (r *Runner) listAvailableTemplates() { gologger.Print().Msgf( "\nListing available v.%s nuclei templates for %s", - r.templatesConfig.CurrentVersion, + r.templatesConfig.TemplateVersion, r.templatesConfig.TemplatesDirectory, ) err := directoryWalker( diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 29c106e98..c9618847a 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -17,7 +17,6 @@ import ( "runtime" "strconv" "strings" - "time" "github.com/apex/log" "github.com/blang/semver" @@ -66,7 +65,7 @@ func (r *Runner) updateTemplates() error { TemplatesDirectory: filepath.Join(home, "nuclei-templates"), NucleiVersion: config.Version, } - if writeErr := config.WriteConfiguration(currentConfig, false); writeErr != nil { + if writeErr := config.WriteConfiguration(currentConfig); writeErr != nil { return errors.Wrap(writeErr, "could not write template configuration") } r.templatesConfig = currentConfig @@ -76,17 +75,7 @@ func (r *Runner) updateTemplates() error { return nil } client.InitNucleiVersion(config.Version) - r.fetchLatestVersionsFromGithub() // also fetch latest versions - - // Check if last checked for nuclei-ignore is more than 1 hours. - // and if true, run the check. - // - // Also at the same time fetch latest version from github to do outdated nuclei - // and templates check. - checkedIgnore := false - if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour { - checkedIgnore = r.checkNucleiIgnoreFileUpdates(configDir) - } + r.fetchLatestVersionsFromGithub(configDir) // also fetch latest versions ctx := context.Background() @@ -95,7 +84,7 @@ func (r *Runner) updateTemplates() error { noTemplatesFound = true } - if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound { + if r.templatesConfig.TemplateVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound { gologger.Info().Msgf("nuclei-templates are not installed, installing...\n") // Use custom location if user has given a template directory @@ -105,7 +94,7 @@ func (r *Runner) updateTemplates() error { if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") { r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory) } - r.fetchLatestVersionsFromGithub() // also fetch latest versions + r.fetchLatestVersionsFromGithub(configDir) // also fetch latest versions version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) if err != nil { @@ -123,9 +112,9 @@ func (r *Runner) updateTemplates() error { if err != nil { return err } - r.templatesConfig.CurrentVersion = version.String() + r.templatesConfig.TemplateVersion = version.String() - err = config.WriteConfiguration(r.templatesConfig, checkedIgnore) + err = config.WriteConfiguration(r.templatesConfig) if err != nil { return err } @@ -134,7 +123,7 @@ func (r *Runner) updateTemplates() error { } // Get the configuration currently on disk. - verText := r.templatesConfig.CurrentVersion + verText := r.templatesConfig.TemplateVersion indices := reVersion.FindStringIndex(verText) if indices == nil { return fmt.Errorf("invalid release found with tag %s", err) @@ -157,7 +146,7 @@ func (r *Runner) updateTemplates() error { if r.options.UpdateTemplates { gologger.Info().Msgf("No new updates found for nuclei templates") } - return config.WriteConfiguration(r.templatesConfig, checkedIgnore) + return config.WriteConfiguration(r.templatesConfig) } if version.GT(oldVersion) { @@ -167,7 +156,7 @@ func (r *Runner) updateTemplates() error { if r.options.TemplatesDirectory != "" { r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory } - r.templatesConfig.CurrentVersion = version.String() + r.templatesConfig.TemplateVersion = version.String() gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) @@ -179,7 +168,7 @@ func (r *Runner) updateTemplates() error { if err != nil { return err } - err = config.WriteConfiguration(r.templatesConfig, checkedIgnore) + err = config.WriteConfiguration(r.templatesConfig) if err != nil { return err } @@ -215,7 +204,7 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { _ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644) } if r.templatesConfig != nil { - if err := config.WriteConfiguration(r.templatesConfig, true); err != nil { + if err := config.WriteConfiguration(r.templatesConfig); err != nil { gologger.Warning().Msgf("Could not get ignore-file from server: %s", err) } } @@ -473,7 +462,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st } // fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github -func (r *Runner) fetchLatestVersionsFromGithub() { +func (r *Runner) fetchLatestVersionsFromGithub(configDir string) { versions, err := client.GetLatestNucleiTemplatesVersion() if err != nil { gologger.Warning().Msgf("Could not fetch latest releases: %s", err) @@ -482,6 +471,12 @@ func (r *Runner) fetchLatestVersionsFromGithub() { if r.templatesConfig != nil { r.templatesConfig.NucleiLatestVersion = versions.Nuclei r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates + + // If the fetch has resulted in new version of ignore file, update. + if r.templatesConfig.NucleiIgnoreHash == "" || r.templatesConfig.NucleiIgnoreHash != versions.IgnoreHash { + r.templatesConfig.NucleiIgnoreHash = versions.IgnoreHash + r.checkNucleiIgnoreFileUpdates(configDir) + } } } diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index bf615fe46..1c5737545 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -3,7 +3,6 @@ package config import ( "os" "path/filepath" - "time" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" @@ -13,10 +12,10 @@ import ( // Config contains the internal nuclei engine configuration type Config struct { - TemplatesDirectory string `json:"templates-directory,omitempty"` - CurrentVersion string `json:"current-version,omitempty"` - NucleiVersion string `json:"nuclei-version,omitempty"` - LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"` + TemplatesDirectory string `json:"templates-directory,omitempty"` + TemplateVersion string `json:"template-version,omitempty"` + NucleiVersion string `json:"nuclei-version,omitempty"` + NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` NucleiLatestVersion string `json:"nuclei-latest-version"` NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"` @@ -61,10 +60,7 @@ func ReadConfiguration() (*Config, error) { } // WriteConfiguration writes the updated nuclei configuration to disk -func WriteConfiguration(config *Config, checkedIgnore bool) error { - if checkedIgnore { - config.LastCheckedIgnore = time.Now() - } +func WriteConfiguration(config *Config) error { config.NucleiVersion = Version templatesConfigFile, err := getConfigDetails()