mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-17 21:25:27 +00:00
Loader rewriter working poc
This commit is contained in:
parent
7669e9781a
commit
dff76e9cd2
@ -81,13 +81,6 @@ func validateOptions(options *types.Options) error {
|
|||||||
return errors.New("both verbose and silent mode specified")
|
return errors.New("both verbose and silent mode specified")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !options.TemplateList {
|
|
||||||
// Check if a list of templates was provided and it exists
|
|
||||||
if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
|
|
||||||
return errors.New("no template/templates provided")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate proxy options if provided
|
// Validate proxy options if provided
|
||||||
err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
|
err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -260,6 +260,17 @@ func (r *Runner) RunEnumeration() {
|
|||||||
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
|
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
|
||||||
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
|
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
|
||||||
|
|
||||||
|
executerOpts := protocols.ExecuterOptions{
|
||||||
|
Output: r.output,
|
||||||
|
Options: r.options,
|
||||||
|
Progress: r.progress,
|
||||||
|
Catalog: r.catalog,
|
||||||
|
IssuesClient: r.issuesClient,
|
||||||
|
RateLimiter: r.ratelimiter,
|
||||||
|
Interactsh: r.interactsh,
|
||||||
|
ProjectFile: r.projectFile,
|
||||||
|
Browser: r.browser,
|
||||||
|
}
|
||||||
loaderConfig := &loader.Config{
|
loaderConfig := &loader.Config{
|
||||||
Templates: r.options.Templates,
|
Templates: r.options.Templates,
|
||||||
Workflows: r.options.Workflows,
|
Workflows: r.options.Workflows,
|
||||||
@ -272,21 +283,60 @@ func (r *Runner) RunEnumeration() {
|
|||||||
IncludeTags: r.options.IncludeTags,
|
IncludeTags: r.options.IncludeTags,
|
||||||
TemplatesDirectory: r.options.TemplatesDirectory,
|
TemplatesDirectory: r.options.TemplatesDirectory,
|
||||||
Catalog: r.catalog,
|
Catalog: r.catalog,
|
||||||
|
ExecutorOptions: executerOpts,
|
||||||
}
|
}
|
||||||
store, err := loader.New(loaderConfig)
|
store, err := loader.New(loaderConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
gologger.Fatal().Msgf("Could not load templates from config: %s\n", err)
|
gologger.Fatal().Msgf("Could not load templates from config: %s\n", err)
|
||||||
}
|
}
|
||||||
|
store.Load()
|
||||||
|
|
||||||
|
builder := &strings.Builder{}
|
||||||
|
if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
|
||||||
|
builder.WriteString(" (")
|
||||||
|
|
||||||
|
if config.Version == r.templatesConfig.NucleiLatestVersion {
|
||||||
|
builder.WriteString(r.colorizer.Green("latest").String())
|
||||||
|
} else {
|
||||||
|
builder.WriteString(r.colorizer.Red("outdated").String())
|
||||||
|
}
|
||||||
|
builder.WriteString(")")
|
||||||
|
}
|
||||||
|
messageStr := builder.String()
|
||||||
|
builder.Reset()
|
||||||
|
|
||||||
|
gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
|
||||||
|
|
||||||
|
if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
|
||||||
|
builder.WriteString(" (")
|
||||||
|
|
||||||
|
if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
|
||||||
|
builder.WriteString(r.colorizer.Green("latest").String())
|
||||||
|
} else {
|
||||||
|
builder.WriteString(r.colorizer.Red("outdated").String())
|
||||||
|
}
|
||||||
|
builder.WriteString(")")
|
||||||
|
}
|
||||||
|
messageStr = builder.String()
|
||||||
|
builder.Reset()
|
||||||
|
|
||||||
|
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
|
||||||
|
|
||||||
|
if r.interactsh != nil {
|
||||||
|
gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
|
||||||
|
}
|
||||||
|
if len(store.Templates()) > 0 {
|
||||||
|
gologger.Info().Msgf("Running Nuclei Templates (%d)", len(store.Templates()))
|
||||||
|
}
|
||||||
|
if len(store.Workflows()) > 0 {
|
||||||
|
gologger.Info().Msgf("Running Nuclei Workflows (%d)", len(store.Workflows()))
|
||||||
|
}
|
||||||
|
|
||||||
// pre-parse all the templates, apply filters
|
// pre-parse all the templates, apply filters
|
||||||
finalTemplates := []*templates.Template{}
|
finalTemplates := []*templates.Template{}
|
||||||
|
|
||||||
workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
|
|
||||||
availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
|
|
||||||
availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
|
|
||||||
|
|
||||||
var unclusteredRequests int64 = 0
|
var unclusteredRequests int64 = 0
|
||||||
for _, template := range availableTemplates {
|
for _, template := range store.Templates() {
|
||||||
// workflows will dynamically adjust the totals while running, as
|
// workflows will dynamically adjust the totals while running, as
|
||||||
// it can't be know in advance which requests will be called
|
// it can't be know in advance which requests will be called
|
||||||
if len(template.Workflows) > 0 {
|
if len(template.Workflows) > 0 {
|
||||||
@ -295,9 +345,21 @@ func (r *Runner) RunEnumeration() {
|
|||||||
unclusteredRequests += int64(template.TotalRequests) * r.inputCount
|
unclusteredRequests += int64(template.TotalRequests) * r.inputCount
|
||||||
}
|
}
|
||||||
|
|
||||||
originalTemplatesCount := len(availableTemplates)
|
if r.options.Verbose {
|
||||||
|
for _, template := range store.Templates() {
|
||||||
|
r.logAvailableTemplate(template.Path)
|
||||||
|
}
|
||||||
|
for _, template := range store.Workflows() {
|
||||||
|
r.logAvailableTemplate(template.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
templatesMap := make(map[string]*templates.Template)
|
||||||
|
for _, v := range store.Templates() {
|
||||||
|
templatesMap[v.ID] = v
|
||||||
|
}
|
||||||
|
originalTemplatesCount := len(store.Templates())
|
||||||
clusterCount := 0
|
clusterCount := 0
|
||||||
clusters := clusterer.Cluster(availableTemplates)
|
clusters := clusterer.Cluster(templatesMap)
|
||||||
for _, cluster := range clusters {
|
for _, cluster := range clusters {
|
||||||
if len(cluster) > 1 && !r.options.OfflineHTTP {
|
if len(cluster) > 1 && !r.options.OfflineHTTP {
|
||||||
executerOpts := protocols.ExecuterOptions{
|
executerOpts := protocols.ExecuterOptions{
|
||||||
@ -324,7 +386,7 @@ func (r *Runner) RunEnumeration() {
|
|||||||
finalTemplates = append(finalTemplates, cluster...)
|
finalTemplates = append(finalTemplates, cluster...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, workflows := range availableWorkflows {
|
for _, workflows := range store.Workflows() {
|
||||||
finalTemplates = append(finalTemplates, workflows)
|
finalTemplates = append(finalTemplates, workflows)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -336,20 +398,16 @@ func (r *Runner) RunEnumeration() {
|
|||||||
totalRequests += int64(t.TotalRequests) * r.inputCount
|
totalRequests += int64(t.TotalRequests) * r.inputCount
|
||||||
}
|
}
|
||||||
if totalRequests < unclusteredRequests {
|
if totalRequests < unclusteredRequests {
|
||||||
gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
|
gologger.Info().Msgf("Reduced %d requests (%d templates clustered)", unclusteredRequests-totalRequests, clusterCount)
|
||||||
}
|
}
|
||||||
templateCount := originalTemplatesCount + len(availableWorkflows)
|
workflowCount := len(store.Workflows())
|
||||||
|
templateCount := originalTemplatesCount + workflowCount
|
||||||
|
|
||||||
// 0 matches means no templates were found in directory
|
// 0 matches means no templates were found in directory
|
||||||
if templateCount == 0 {
|
if templateCount == 0 {
|
||||||
gologger.Fatal().Msgf("Error, no templates were found.\n")
|
gologger.Fatal().Msgf("Error, no templates were found.\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
|
|
||||||
r.colorizer.Bold(templateCount).String(),
|
|
||||||
r.colorizer.Bold(templateCount-workflowCount).String(),
|
|
||||||
r.colorizer.Bold(workflowCount).String())
|
|
||||||
|
|
||||||
results := &atomic.Bool{}
|
results := &atomic.Bool{}
|
||||||
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
|
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
|
||||||
|
|
||||||
|
|||||||
@ -1,36 +1,36 @@
|
|||||||
package runner
|
package runner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/karrick/godirwalk"
|
"github.com/karrick/godirwalk"
|
||||||
"github.com/projectdiscovery/gologger"
|
"github.com/projectdiscovery/gologger"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
// parseTemplateFile returns the parsed template file
|
// parseTemplateFile returns the parsed template file
|
||||||
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
|
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
|
||||||
executerOpts := protocols.ExecuterOptions{
|
f, err := os.Open(file)
|
||||||
Output: r.output,
|
|
||||||
Options: r.options,
|
|
||||||
Progress: r.progress,
|
|
||||||
Catalog: r.catalog,
|
|
||||||
IssuesClient: r.issuesClient,
|
|
||||||
RateLimiter: r.ratelimiter,
|
|
||||||
Interactsh: r.interactsh,
|
|
||||||
ProjectFile: r.projectFile,
|
|
||||||
Browser: r.browser,
|
|
||||||
}
|
|
||||||
template, err := templates.Parse(file, executerOpts)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if template == nil {
|
defer f.Close()
|
||||||
return nil, nil
|
|
||||||
|
data, err := ioutil.ReadAll(f)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
template := &templates.Template{}
|
||||||
|
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
return template, nil
|
return template, nil
|
||||||
}
|
}
|
||||||
@ -52,7 +52,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
|
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
|
||||||
} else {
|
} else {
|
||||||
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
|
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,38 +89,12 @@ func (r *Runner) listAvailableTemplates() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
|
|
||||||
for _, s := range allowedSeverities {
|
|
||||||
finalSeverities := []string{}
|
|
||||||
if strings.Contains(s, ",") {
|
|
||||||
finalSeverities = strings.Split(s, ",")
|
|
||||||
} else {
|
|
||||||
finalSeverities = append(finalSeverities, s)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, sev := range finalSeverities {
|
|
||||||
sev = strings.ToLower(sev)
|
|
||||||
if sev != "" && strings.HasPrefix(templateSeverity, sev) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
|
func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
|
||||||
err := godirwalk.Walk(fsPath, &godirwalk.Options{
|
return godirwalk.Walk(fsPath, &godirwalk.Options{
|
||||||
Callback: callback,
|
Callback: callback,
|
||||||
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
|
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
|
||||||
return godirwalk.SkipNode
|
return godirwalk.SkipNode
|
||||||
},
|
},
|
||||||
Unsorted: true,
|
Unsorted: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
// directory couldn't be walked
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/md5"
|
"crypto/md5"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
@ -14,6 +15,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@ -23,6 +25,7 @@ import (
|
|||||||
"github.com/olekukonko/tablewriter"
|
"github.com/olekukonko/tablewriter"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/projectdiscovery/gologger"
|
"github.com/projectdiscovery/gologger"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -30,6 +33,13 @@ const (
|
|||||||
repoName = "nuclei-templates"
|
repoName = "nuclei-templates"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const nucleiIgnoreFile = ".nuclei-ignore"
|
||||||
|
|
||||||
|
// nucleiConfigFilename is the filename of nuclei configuration file.
|
||||||
|
const nucleiConfigFilename = ".templates-config.json"
|
||||||
|
|
||||||
|
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
|
||||||
|
|
||||||
// updateTemplates checks if the default list of nuclei-templates
|
// updateTemplates checks if the default list of nuclei-templates
|
||||||
// exist in the users home directory, if not the latest revision
|
// exist in the users home directory, if not the latest revision
|
||||||
// is downloaded from github.
|
// is downloaded from github.
|
||||||
@ -46,7 +56,7 @@ func (r *Runner) updateTemplates() error {
|
|||||||
|
|
||||||
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
|
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
|
||||||
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
|
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
|
||||||
config, readErr := readConfiguration()
|
config, readErr := config.ReadConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return readErr
|
return readErr
|
||||||
}
|
}
|
||||||
@ -55,12 +65,12 @@ func (r *Runner) updateTemplates() error {
|
|||||||
|
|
||||||
ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
|
ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
|
||||||
if r.templatesConfig == nil {
|
if r.templatesConfig == nil {
|
||||||
currentConfig := &nucleiConfig{
|
currentConfig := &config.Config{
|
||||||
TemplatesDirectory: path.Join(home, "nuclei-templates"),
|
TemplatesDirectory: path.Join(home, "nuclei-templates"),
|
||||||
IgnoreURL: ignoreURL,
|
IgnoreURL: ignoreURL,
|
||||||
NucleiVersion: Version,
|
NucleiVersion: config.Version,
|
||||||
}
|
}
|
||||||
if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
|
if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
|
||||||
return errors.Wrap(writeErr, "could not write template configuration")
|
return errors.Wrap(writeErr, "could not write template configuration")
|
||||||
}
|
}
|
||||||
r.templatesConfig = currentConfig
|
r.templatesConfig = currentConfig
|
||||||
@ -68,12 +78,19 @@ func (r *Runner) updateTemplates() error {
|
|||||||
|
|
||||||
// Check if last checked for nuclei-ignore is more than 1 hours.
|
// Check if last checked for nuclei-ignore is more than 1 hours.
|
||||||
// and if true, run the check.
|
// and if true, run the check.
|
||||||
|
//
|
||||||
|
// Also at the same time fetch latest version from github to do outdated nuclei
|
||||||
|
// and templates check.
|
||||||
|
checkedIgnore := false
|
||||||
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
|
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
|
||||||
|
r.fetchLatestVersionsFromGithub()
|
||||||
|
|
||||||
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
|
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
|
||||||
ignoreURL = r.templatesConfig.IgnoreURL
|
ignoreURL = r.templatesConfig.IgnoreURL
|
||||||
}
|
}
|
||||||
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
|
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
|
||||||
|
|
||||||
|
checkedIgnore = true
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
|
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
|
||||||
if reqErr == nil {
|
if reqErr == nil {
|
||||||
@ -106,7 +123,7 @@ func (r *Runner) updateTemplates() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Use custom location if user has given a template directory
|
// Use custom location if user has given a template directory
|
||||||
r.templatesConfig = &nucleiConfig{
|
r.templatesConfig = &config.Config{
|
||||||
TemplatesDirectory: path.Join(home, "nuclei-templates"),
|
TemplatesDirectory: path.Join(home, "nuclei-templates"),
|
||||||
}
|
}
|
||||||
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
|
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
|
||||||
@ -120,13 +137,14 @@ func (r *Runner) updateTemplates() error {
|
|||||||
}
|
}
|
||||||
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
|
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
|
||||||
|
|
||||||
|
r.fetchLatestVersionsFromGithub() // also fetch latest versions
|
||||||
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
|
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
r.templatesConfig.CurrentVersion = version.String()
|
r.templatesConfig.CurrentVersion = version.String()
|
||||||
|
|
||||||
err = r.writeConfiguration(r.templatesConfig)
|
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -162,13 +180,13 @@ func (r *Runner) updateTemplates() error {
|
|||||||
|
|
||||||
if version.EQ(oldVersion) {
|
if version.EQ(oldVersion) {
|
||||||
gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
|
gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
|
||||||
return r.writeConfiguration(r.templatesConfig)
|
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
|
||||||
}
|
}
|
||||||
|
|
||||||
if version.GT(oldVersion) {
|
if version.GT(oldVersion) {
|
||||||
if !r.options.UpdateTemplates {
|
if !r.options.UpdateTemplates {
|
||||||
gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
|
gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
|
||||||
return r.writeConfiguration(r.templatesConfig)
|
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.options.TemplatesDirectory != "" {
|
if r.options.TemplatesDirectory != "" {
|
||||||
@ -177,11 +195,12 @@ func (r *Runner) updateTemplates() error {
|
|||||||
r.templatesConfig.CurrentVersion = version.String()
|
r.templatesConfig.CurrentVersion = version.String()
|
||||||
|
|
||||||
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
|
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
|
||||||
|
r.fetchLatestVersionsFromGithub()
|
||||||
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
|
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
err = r.writeConfiguration(r.templatesConfig)
|
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -461,3 +480,55 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
|
|||||||
}
|
}
|
||||||
table.Render()
|
table.Render()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
|
||||||
|
func (r *Runner) fetchLatestVersionsFromGithub() {
|
||||||
|
nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
|
||||||
|
if err != nil {
|
||||||
|
gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
|
||||||
|
}
|
||||||
|
templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
|
||||||
|
if err != nil {
|
||||||
|
gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
|
||||||
|
}
|
||||||
|
if r.templatesConfig != nil {
|
||||||
|
r.templatesConfig.NucleiLatestVersion = nucleiLatest
|
||||||
|
r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type githubTagData struct {
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
|
||||||
|
// githubFetchLatestTagRepo fetches latest tag from github
|
||||||
|
func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
var tags []githubTagData
|
||||||
|
err = json.Unmarshal(body, &tags)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if len(tags) == 0 {
|
||||||
|
return "", fmt.Errorf("no tags found for %s", repo)
|
||||||
|
}
|
||||||
|
return tags[0].Name, nil
|
||||||
|
}
|
||||||
|
|||||||
@ -43,7 +43,6 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
|
|||||||
defer os.RemoveAll(templatesDirectory)
|
defer os.RemoveAll(templatesDirectory)
|
||||||
|
|
||||||
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}}
|
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}}
|
||||||
|
|
||||||
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
|
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
|
||||||
require.Nil(t, err, "could not download release and unzip")
|
require.Nil(t, err, "could not download release and unzip")
|
||||||
require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
|
require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
|
||||||
|
|||||||
@ -3,7 +3,6 @@ package config
|
|||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"regexp"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
@ -19,13 +18,14 @@ type Config struct {
|
|||||||
IgnoreURL string `json:"ignore-url,omitempty"`
|
IgnoreURL string `json:"ignore-url,omitempty"`
|
||||||
NucleiVersion string `json:"nuclei-version,omitempty"`
|
NucleiVersion string `json:"nuclei-version,omitempty"`
|
||||||
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
|
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
|
||||||
|
|
||||||
|
NucleiLatestVersion string `json:"nuclei-latest-version"`
|
||||||
|
NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// nucleiConfigFilename is the filename of nuclei configuration file.
|
// nucleiConfigFilename is the filename of nuclei configuration file.
|
||||||
const nucleiConfigFilename = ".templates-config.json"
|
const nucleiConfigFilename = ".templates-config.json"
|
||||||
|
|
||||||
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
|
|
||||||
|
|
||||||
// Version is the current version of nuclei
|
// Version is the current version of nuclei
|
||||||
const Version = `2.3.8`
|
const Version = `2.3.8`
|
||||||
|
|
||||||
@ -59,12 +59,16 @@ func ReadConfiguration() (*Config, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// WriteConfiguration writes the updated nuclei configuration to disk
|
// WriteConfiguration writes the updated nuclei configuration to disk
|
||||||
func WriteConfiguration(config *Config) error {
|
func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
|
||||||
if config.IgnoreURL == "" {
|
if config.IgnoreURL == "" {
|
||||||
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
|
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
|
||||||
}
|
}
|
||||||
|
if checked {
|
||||||
config.LastChecked = time.Now()
|
config.LastChecked = time.Now()
|
||||||
|
}
|
||||||
|
if checkedIgnore {
|
||||||
config.LastCheckedIgnore = time.Now()
|
config.LastCheckedIgnore = time.Now()
|
||||||
|
}
|
||||||
config.NucleiVersion = Version
|
config.NucleiVersion = Version
|
||||||
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
|
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"github.com/projectdiscovery/gologger"
|
"github.com/projectdiscovery/gologger"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
|
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
@ -28,6 +29,7 @@ type Config struct {
|
|||||||
IncludeTags []string
|
IncludeTags []string
|
||||||
|
|
||||||
Catalog *catalog.Catalog
|
Catalog *catalog.Catalog
|
||||||
|
ExecutorOptions protocols.ExecuterOptions
|
||||||
TemplatesDirectory string
|
TemplatesDirectory string
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -37,6 +39,9 @@ type Store struct {
|
|||||||
config *Config
|
config *Config
|
||||||
finalTemplates []string
|
finalTemplates []string
|
||||||
templateMatched bool
|
templateMatched bool
|
||||||
|
|
||||||
|
templates []*templates.Template
|
||||||
|
workflows []*templates.Template
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new template store based on provided configuration
|
// New creates a new template store based on provided configuration
|
||||||
@ -47,7 +52,6 @@ func New(config *Config) (*Store, error) {
|
|||||||
tagFilter: config.createTagFilter(),
|
tagFilter: config.createTagFilter(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// hasFindByMetadata := config.hasFindByMetadata()
|
|
||||||
// Handle a case with no templates or workflows, where we use base directory
|
// Handle a case with no templates or workflows, where we use base directory
|
||||||
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
|
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
|
||||||
config.Templates = append(config.Templates, config.TemplatesDirectory)
|
config.Templates = append(config.Templates, config.TemplatesDirectory)
|
||||||
@ -55,13 +59,24 @@ func New(config *Config) (*Store, error) {
|
|||||||
store.templateMatched = true
|
store.templateMatched = true
|
||||||
}
|
}
|
||||||
store.finalTemplates = append(store.finalTemplates, config.Templates...)
|
store.finalTemplates = append(store.finalTemplates, config.Templates...)
|
||||||
|
|
||||||
return store, nil
|
return store, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Templates returns all the templates in the store
|
||||||
|
func (s *Store) Templates() []*templates.Template {
|
||||||
|
return s.templates
|
||||||
|
}
|
||||||
|
|
||||||
|
// Workflows returns all the workflows in the store
|
||||||
|
func (s *Store) Workflows() []*templates.Template {
|
||||||
|
return s.workflows
|
||||||
|
}
|
||||||
|
|
||||||
// Load loads all the templates from a store, performs filtering and returns
|
// Load loads all the templates from a store, performs filtering and returns
|
||||||
// the complete compiled templates for a nuclei execution configuration.
|
// the complete compiled templates for a nuclei execution configuration.
|
||||||
func (s *Store) Load() (templates []string, workflows []string) {
|
func (s *Store) Load() {
|
||||||
includedTemplates := s.config.Catalog.GetTemplatesPath(s.config.Templates)
|
includedTemplates := s.config.Catalog.GetTemplatesPath(s.finalTemplates)
|
||||||
includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
|
includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
|
||||||
excludedTemplates := s.config.Catalog.GetTemplatesPath(s.config.ExcludeTemplates)
|
excludedTemplates := s.config.Catalog.GetTemplatesPath(s.config.ExcludeTemplates)
|
||||||
alwaysIncludeTemplates := s.config.Catalog.GetTemplatesPath(s.config.IncludeTemplates)
|
alwaysIncludeTemplates := s.config.Catalog.GetTemplatesPath(s.config.IncludeTemplates)
|
||||||
@ -89,7 +104,12 @@ func (s *Store) Load() (templates []string, workflows []string) {
|
|||||||
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
|
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
|
||||||
}
|
}
|
||||||
if loaded {
|
if loaded {
|
||||||
templates = append(templates, k)
|
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
|
||||||
|
if err != nil {
|
||||||
|
gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
|
||||||
|
} else if parsed != nil {
|
||||||
|
s.templates = append(s.templates, parsed)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -110,10 +130,14 @@ func (s *Store) Load() (templates []string, workflows []string) {
|
|||||||
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
|
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
|
||||||
}
|
}
|
||||||
if loaded {
|
if loaded {
|
||||||
workflows = append(workflows, k)
|
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
|
||||||
|
if err != nil {
|
||||||
|
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
|
||||||
|
} else if parsed != nil {
|
||||||
|
s.workflows = append(s.workflows, parsed)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return templates, workflows
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadTemplateParseMetadata loads a template by parsing metadata and running
|
// loadTemplateParseMetadata loads a template by parsing metadata and running
|
||||||
|
|||||||
@ -189,49 +189,3 @@ func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, o
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// matchTemplateWithTags matches if the template matches a tag
|
|
||||||
func matchTemplateWithTags(tags, severity string, tagsInput []string) error {
|
|
||||||
actualTags := strings.Split(tags, ",")
|
|
||||||
if severity != "" {
|
|
||||||
actualTags = append(actualTags, severity) // also add severity to tag
|
|
||||||
}
|
|
||||||
|
|
||||||
matched := false
|
|
||||||
mainLoop:
|
|
||||||
for _, t := range tagsInput {
|
|
||||||
commaTags := strings.Split(t, ",")
|
|
||||||
for _, tag := range commaTags {
|
|
||||||
tag = strings.TrimSpace(tag)
|
|
||||||
key, value := getKeyValue(tag)
|
|
||||||
|
|
||||||
for _, templTag := range actualTags {
|
|
||||||
templTag = strings.TrimSpace(templTag)
|
|
||||||
tKey, tValue := getKeyValue(templTag)
|
|
||||||
|
|
||||||
if strings.EqualFold(key, tKey) && strings.EqualFold(value, tValue) {
|
|
||||||
matched = true
|
|
||||||
break mainLoop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !matched {
|
|
||||||
return errors.New("could not match template tags with input")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// getKeyValue returns key value pair for a data string
|
|
||||||
func getKeyValue(data string) (key, value string) {
|
|
||||||
if strings.Contains(data, ":") {
|
|
||||||
parts := strings.SplitN(data, ":", 2)
|
|
||||||
if len(parts) == 2 {
|
|
||||||
key, value = parts[0], parts[1]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if value == "" {
|
|
||||||
value = data
|
|
||||||
}
|
|
||||||
return key, value
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,41 +0,0 @@
|
|||||||
package templates
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMatchTemplateWithTags(t *testing.T) {
|
|
||||||
err := matchTemplateWithTags("php,linux,symfony", "", []string{"php"})
|
|
||||||
require.Nil(t, err, "could not get php tag from input slice")
|
|
||||||
|
|
||||||
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"cms:symfony"})
|
|
||||||
require.Nil(t, err, "could not get php tag from input key value")
|
|
||||||
|
|
||||||
err = matchTemplateWithTags("lang:php,os:linux,symfony", "", []string{"cms:symfony"})
|
|
||||||
require.NotNil(t, err, "could get key value tag from input key value")
|
|
||||||
|
|
||||||
err = matchTemplateWithTags("lang:php,os:linux,cms:jira", "", []string{"cms:symfony"})
|
|
||||||
require.NotNil(t, err, "could get key value tag from input key value")
|
|
||||||
|
|
||||||
t.Run("space", func(t *testing.T) {
|
|
||||||
err = matchTemplateWithTags("lang:php, os:linux, cms:symfony", "", []string{"cms:symfony"})
|
|
||||||
require.Nil(t, err, "could get key value tag from input key value with space")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("comma-tags", func(t *testing.T) {
|
|
||||||
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"test,cms:symfony"})
|
|
||||||
require.Nil(t, err, "could get key value tag from input key value with comma")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("severity", func(t *testing.T) {
|
|
||||||
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "low", []string{"low"})
|
|
||||||
require.Nil(t, err, "could get key value tag for severity")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("blank-tags", func(t *testing.T) {
|
|
||||||
err = matchTemplateWithTags("", "low", []string{"jira"})
|
|
||||||
require.NotNil(t, err, "could get value tag for blank severity")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Loading…
x
Reference in New Issue
Block a user