External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 17:05:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Misc changes to update and ignore handling

Browse Source
This commit is contained in:
Ice3man543 2021-09-15 04:01:40 +05:30
parent 6667f285d3
commit 4d52fb45e4
7 changed files with 30 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/go.mod
View File

@ -107,7 +107,7 @@ require (
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 // indirect
github.com/projectdiscovery/mapcidr v0.0.6 // indirect
github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf // indirect
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.7 // indirect
github.com/tklauser/numcpus v0.2.3 // indirect

2
v2/go.sum
View File

@ -363,6 +363,8 @@ github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094006-ee345cd
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094006-ee345cd25b9b/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf h1:cFUSV7HERosPKdHuPiV348eVxObVMBqNt0ydYVplsI0=
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210913094946-d1ec15db5faf/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ=
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=

2
v2/internal/runner/options.go
View File

@ -36,7 +36,7 @@ func ParseOptions(options *types.Options) {
if err != nil {
gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
}
gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.CurrentVersion, configuration.TemplatesDirectory)
gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory)
os.Exit(0)
}

4
v2/internal/runner/runner.go
View File

@ -392,7 +392,7 @@ func (r *Runner) RunEnumeration() error {
if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
builder.WriteString(" (")
if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
@ -403,7 +403,7 @@ func (r *Runner) RunEnumeration() error {
builder.Reset()
if r.templatesConfig != nil {
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr)
}
if r.interactsh != nil {
gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)

2
v2/internal/runner/templates.go
View File

@ -70,7 +70,7 @@ func (r *Runner) listAvailableTemplates() {
gologger.Print().Msgf(
"\nListing available v.%s nuclei templates for %s",
r.templatesConfig.CurrentVersion,
r.templatesConfig.TemplateVersion,
r.templatesConfig.TemplatesDirectory,
)
err := directoryWalker(

41
v2/internal/runner/update.go
View File

@ -17,7 +17,6 @@ import (
"runtime"
"strconv"
"strings"
"time"
"github.com/apex/log"
"github.com/blang/semver"
@ -66,7 +65,7 @@ func (r *Runner) updateTemplates() error {
TemplatesDirectory: filepath.Join(home, "nuclei-templates"),
NucleiVersion: config.Version,
}
if writeErr := config.WriteConfiguration(currentConfig, false); writeErr != nil {
if writeErr := config.WriteConfiguration(currentConfig); writeErr != nil {
return errors.Wrap(writeErr, "could not write template configuration")
}
r.templatesConfig = currentConfig
@ -76,17 +75,7 @@ func (r *Runner) updateTemplates() error {
return nil
}
client.InitNucleiVersion(config.Version)
r.fetchLatestVersionsFromGithub() // also fetch latest versions
// Check if last checked for nuclei-ignore is more than 1 hours.
// and if true, run the check.
//
// Also at the same time fetch latest version from github to do outdated nuclei
// and templates check.
checkedIgnore := false
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour {
checkedIgnore = r.checkNucleiIgnoreFileUpdates(configDir)
}
r.fetchLatestVersionsFromGithub(configDir) // also fetch latest versions
ctx := context.Background()
@ -95,7 +84,7 @@ func (r *Runner) updateTemplates() error {
noTemplatesFound = true
}
if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound {
if r.templatesConfig.TemplateVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) || noTemplatesFound {
gologger.Info().Msgf("nuclei-templates are not installed, installing...\n")
// Use custom location if user has given a template directory
@ -105,7 +94,7 @@ func (r *Runner) updateTemplates() error {
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory)
}
r.fetchLatestVersionsFromGithub() // also fetch latest versions
r.fetchLatestVersionsFromGithub(configDir) // also fetch latest versions
version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
if err != nil {
@ -123,9 +112,9 @@ func (r *Runner) updateTemplates() error {
if err != nil {
return err
}
r.templatesConfig.CurrentVersion = version.String()
r.templatesConfig.TemplateVersion = version.String()
err = config.WriteConfiguration(r.templatesConfig, checkedIgnore)
err = config.WriteConfiguration(r.templatesConfig)
if err != nil {
return err
}
@ -134,7 +123,7 @@ func (r *Runner) updateTemplates() error {
}
// Get the configuration currently on disk.
verText := r.templatesConfig.CurrentVersion
verText := r.templatesConfig.TemplateVersion
indices := reVersion.FindStringIndex(verText)
if indices == nil {
return fmt.Errorf("invalid release found with tag %s", err)
@ -157,7 +146,7 @@ func (r *Runner) updateTemplates() error {
if r.options.UpdateTemplates {
gologger.Info().Msgf("No new updates found for nuclei templates")
}
return config.WriteConfiguration(r.templatesConfig, checkedIgnore)
return config.WriteConfiguration(r.templatesConfig)
}
if version.GT(oldVersion) {
@ -167,7 +156,7 @@ func (r *Runner) updateTemplates() error {
if r.options.TemplatesDirectory != "" {
r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
}
r.templatesConfig.CurrentVersion = version.String()
r.templatesConfig.TemplateVersion = version.String()
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
@ -179,7 +168,7 @@ func (r *Runner) updateTemplates() error {
if err != nil {
return err
}
err = config.WriteConfiguration(r.templatesConfig, checkedIgnore)
err = config.WriteConfiguration(r.templatesConfig)
if err != nil {
return err
}
@ -215,7 +204,7 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
_ = ioutil.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644)
}
if r.templatesConfig != nil {
if err := config.WriteConfiguration(r.templatesConfig, true); err != nil {
if err := config.WriteConfiguration(r.templatesConfig); err != nil {
gologger.Warning().Msgf("Could not get ignore-file from server: %s", err)
}
}
@ -473,7 +462,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
}
// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
func (r *Runner) fetchLatestVersionsFromGithub() {
func (r *Runner) fetchLatestVersionsFromGithub(configDir string) {
versions, err := client.GetLatestNucleiTemplatesVersion()
if err != nil {
gologger.Warning().Msgf("Could not fetch latest releases: %s", err)
@ -482,6 +471,12 @@ func (r *Runner) fetchLatestVersionsFromGithub() {
if r.templatesConfig != nil {
r.templatesConfig.NucleiLatestVersion = versions.Nuclei
r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates
// If the fetch has resulted in new version of ignore file, update.
if r.templatesConfig.NucleiIgnoreHash == "" || r.templatesConfig.NucleiIgnoreHash != versions.IgnoreHash {
r.templatesConfig.NucleiIgnoreHash = versions.IgnoreHash
r.checkNucleiIgnoreFileUpdates(configDir)
}
}
}

10
v2/pkg/catalog/config/config.go
View File

@ -3,7 +3,6 @@ package config
import (
"os"
"path/filepath"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
@ -14,9 +13,9 @@ import (
// Config contains the internal nuclei engine configuration
type Config struct {
TemplatesDirectory string `json:"templates-directory,omitempty"`
CurrentVersion string `json:"current-version,omitempty"`
TemplateVersion string `json:"template-version,omitempty"`
NucleiVersion string `json:"nuclei-version,omitempty"`
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"`
NucleiLatestVersion string `json:"nuclei-latest-version"`
NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
@ -61,10 +60,7 @@ func ReadConfiguration() (*Config, error) {
}
// WriteConfiguration writes the updated nuclei configuration to disk
func WriteConfiguration(config *Config, checkedIgnore bool) error {
if checkedIgnore {
config.LastCheckedIgnore = time.Now()
}
func WriteConfiguration(config *Config) error {
config.NucleiVersion = Version
templatesConfigFile, err := getConfigDetails()