mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-17 19:35:27 +00:00
Loader rewriter working poc
This commit is contained in:
parent
7669e9781a
commit
dff76e9cd2
@ -81,13 +81,6 @@ func validateOptions(options *types.Options) error {
|
||||
return errors.New("both verbose and silent mode specified")
|
||||
}
|
||||
|
||||
if !options.TemplateList {
|
||||
// Check if a list of templates was provided and it exists
|
||||
if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
|
||||
return errors.New("no template/templates provided")
|
||||
}
|
||||
}
|
||||
|
||||
// Validate proxy options if provided
|
||||
err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
|
||||
if err != nil {
|
||||
|
||||
@ -260,6 +260,17 @@ func (r *Runner) RunEnumeration() {
|
||||
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
|
||||
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
|
||||
|
||||
executerOpts := protocols.ExecuterOptions{
|
||||
Output: r.output,
|
||||
Options: r.options,
|
||||
Progress: r.progress,
|
||||
Catalog: r.catalog,
|
||||
IssuesClient: r.issuesClient,
|
||||
RateLimiter: r.ratelimiter,
|
||||
Interactsh: r.interactsh,
|
||||
ProjectFile: r.projectFile,
|
||||
Browser: r.browser,
|
||||
}
|
||||
loaderConfig := &loader.Config{
|
||||
Templates: r.options.Templates,
|
||||
Workflows: r.options.Workflows,
|
||||
@ -272,21 +283,60 @@ func (r *Runner) RunEnumeration() {
|
||||
IncludeTags: r.options.IncludeTags,
|
||||
TemplatesDirectory: r.options.TemplatesDirectory,
|
||||
Catalog: r.catalog,
|
||||
ExecutorOptions: executerOpts,
|
||||
}
|
||||
store, err := loader.New(loaderConfig)
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Could not load templates from config: %s\n", err)
|
||||
}
|
||||
store.Load()
|
||||
|
||||
builder := &strings.Builder{}
|
||||
if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
|
||||
builder.WriteString(" (")
|
||||
|
||||
if config.Version == r.templatesConfig.NucleiLatestVersion {
|
||||
builder.WriteString(r.colorizer.Green("latest").String())
|
||||
} else {
|
||||
builder.WriteString(r.colorizer.Red("outdated").String())
|
||||
}
|
||||
builder.WriteString(")")
|
||||
}
|
||||
messageStr := builder.String()
|
||||
builder.Reset()
|
||||
|
||||
gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
|
||||
|
||||
if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
|
||||
builder.WriteString(" (")
|
||||
|
||||
if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
|
||||
builder.WriteString(r.colorizer.Green("latest").String())
|
||||
} else {
|
||||
builder.WriteString(r.colorizer.Red("outdated").String())
|
||||
}
|
||||
builder.WriteString(")")
|
||||
}
|
||||
messageStr = builder.String()
|
||||
builder.Reset()
|
||||
|
||||
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
|
||||
|
||||
if r.interactsh != nil {
|
||||
gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
|
||||
}
|
||||
if len(store.Templates()) > 0 {
|
||||
gologger.Info().Msgf("Running Nuclei Templates (%d)", len(store.Templates()))
|
||||
}
|
||||
if len(store.Workflows()) > 0 {
|
||||
gologger.Info().Msgf("Running Nuclei Workflows (%d)", len(store.Workflows()))
|
||||
}
|
||||
|
||||
// pre-parse all the templates, apply filters
|
||||
finalTemplates := []*templates.Template{}
|
||||
|
||||
workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
|
||||
availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
|
||||
availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
|
||||
|
||||
var unclusteredRequests int64 = 0
|
||||
for _, template := range availableTemplates {
|
||||
for _, template := range store.Templates() {
|
||||
// workflows will dynamically adjust the totals while running, as
|
||||
// it can't be know in advance which requests will be called
|
||||
if len(template.Workflows) > 0 {
|
||||
@ -295,9 +345,21 @@ func (r *Runner) RunEnumeration() {
|
||||
unclusteredRequests += int64(template.TotalRequests) * r.inputCount
|
||||
}
|
||||
|
||||
originalTemplatesCount := len(availableTemplates)
|
||||
if r.options.Verbose {
|
||||
for _, template := range store.Templates() {
|
||||
r.logAvailableTemplate(template.Path)
|
||||
}
|
||||
for _, template := range store.Workflows() {
|
||||
r.logAvailableTemplate(template.Path)
|
||||
}
|
||||
}
|
||||
templatesMap := make(map[string]*templates.Template)
|
||||
for _, v := range store.Templates() {
|
||||
templatesMap[v.ID] = v
|
||||
}
|
||||
originalTemplatesCount := len(store.Templates())
|
||||
clusterCount := 0
|
||||
clusters := clusterer.Cluster(availableTemplates)
|
||||
clusters := clusterer.Cluster(templatesMap)
|
||||
for _, cluster := range clusters {
|
||||
if len(cluster) > 1 && !r.options.OfflineHTTP {
|
||||
executerOpts := protocols.ExecuterOptions{
|
||||
@ -324,7 +386,7 @@ func (r *Runner) RunEnumeration() {
|
||||
finalTemplates = append(finalTemplates, cluster...)
|
||||
}
|
||||
}
|
||||
for _, workflows := range availableWorkflows {
|
||||
for _, workflows := range store.Workflows() {
|
||||
finalTemplates = append(finalTemplates, workflows)
|
||||
}
|
||||
|
||||
@ -336,20 +398,16 @@ func (r *Runner) RunEnumeration() {
|
||||
totalRequests += int64(t.TotalRequests) * r.inputCount
|
||||
}
|
||||
if totalRequests < unclusteredRequests {
|
||||
gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
|
||||
gologger.Info().Msgf("Reduced %d requests (%d templates clustered)", unclusteredRequests-totalRequests, clusterCount)
|
||||
}
|
||||
templateCount := originalTemplatesCount + len(availableWorkflows)
|
||||
workflowCount := len(store.Workflows())
|
||||
templateCount := originalTemplatesCount + workflowCount
|
||||
|
||||
// 0 matches means no templates were found in directory
|
||||
if templateCount == 0 {
|
||||
gologger.Fatal().Msgf("Error, no templates were found.\n")
|
||||
}
|
||||
|
||||
gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
|
||||
r.colorizer.Bold(templateCount).String(),
|
||||
r.colorizer.Bold(templateCount-workflowCount).String(),
|
||||
r.colorizer.Bold(workflowCount).String())
|
||||
|
||||
results := &atomic.Bool{}
|
||||
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
|
||||
|
||||
|
||||
@ -1,36 +1,36 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/karrick/godirwalk"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// parseTemplateFile returns the parsed template file
|
||||
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
|
||||
executerOpts := protocols.ExecuterOptions{
|
||||
Output: r.output,
|
||||
Options: r.options,
|
||||
Progress: r.progress,
|
||||
Catalog: r.catalog,
|
||||
IssuesClient: r.issuesClient,
|
||||
RateLimiter: r.ratelimiter,
|
||||
Interactsh: r.interactsh,
|
||||
ProjectFile: r.projectFile,
|
||||
Browser: r.browser,
|
||||
}
|
||||
template, err := templates.Parse(file, executerOpts)
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if template == nil {
|
||||
return nil, nil
|
||||
defer f.Close()
|
||||
|
||||
data, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
template := &templates.Template{}
|
||||
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return template, nil
|
||||
}
|
||||
@ -52,7 +52,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) {
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
|
||||
} else {
|
||||
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
|
||||
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
|
||||
}
|
||||
}
|
||||
|
||||
@ -89,38 +89,12 @@ func (r *Runner) listAvailableTemplates() {
|
||||
}
|
||||
}
|
||||
|
||||
func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
|
||||
for _, s := range allowedSeverities {
|
||||
finalSeverities := []string{}
|
||||
if strings.Contains(s, ",") {
|
||||
finalSeverities = strings.Split(s, ",")
|
||||
} else {
|
||||
finalSeverities = append(finalSeverities, s)
|
||||
}
|
||||
|
||||
for _, sev := range finalSeverities {
|
||||
sev = strings.ToLower(sev)
|
||||
if sev != "" && strings.HasPrefix(templateSeverity, sev) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
|
||||
err := godirwalk.Walk(fsPath, &godirwalk.Options{
|
||||
return godirwalk.Walk(fsPath, &godirwalk.Options{
|
||||
Callback: callback,
|
||||
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
|
||||
return godirwalk.SkipNode
|
||||
},
|
||||
Unsorted: true,
|
||||
})
|
||||
|
||||
// directory couldn't be walked
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@ -14,6 +15,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@ -23,6 +25,7 @@ import (
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -30,6 +33,13 @@ const (
|
||||
repoName = "nuclei-templates"
|
||||
)
|
||||
|
||||
const nucleiIgnoreFile = ".nuclei-ignore"
|
||||
|
||||
// nucleiConfigFilename is the filename of nuclei configuration file.
|
||||
const nucleiConfigFilename = ".templates-config.json"
|
||||
|
||||
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
|
||||
|
||||
// updateTemplates checks if the default list of nuclei-templates
|
||||
// exist in the users home directory, if not the latest revision
|
||||
// is downloaded from github.
|
||||
@ -46,7 +56,7 @@ func (r *Runner) updateTemplates() error {
|
||||
|
||||
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
|
||||
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
|
||||
config, readErr := readConfiguration()
|
||||
config, readErr := config.ReadConfiguration()
|
||||
if err != nil {
|
||||
return readErr
|
||||
}
|
||||
@ -55,12 +65,12 @@ func (r *Runner) updateTemplates() error {
|
||||
|
||||
ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
|
||||
if r.templatesConfig == nil {
|
||||
currentConfig := &nucleiConfig{
|
||||
currentConfig := &config.Config{
|
||||
TemplatesDirectory: path.Join(home, "nuclei-templates"),
|
||||
IgnoreURL: ignoreURL,
|
||||
NucleiVersion: Version,
|
||||
NucleiVersion: config.Version,
|
||||
}
|
||||
if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
|
||||
if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
|
||||
return errors.Wrap(writeErr, "could not write template configuration")
|
||||
}
|
||||
r.templatesConfig = currentConfig
|
||||
@ -68,12 +78,19 @@ func (r *Runner) updateTemplates() error {
|
||||
|
||||
// Check if last checked for nuclei-ignore is more than 1 hours.
|
||||
// and if true, run the check.
|
||||
//
|
||||
// Also at the same time fetch latest version from github to do outdated nuclei
|
||||
// and templates check.
|
||||
checkedIgnore := false
|
||||
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
|
||||
r.fetchLatestVersionsFromGithub()
|
||||
|
||||
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
|
||||
ignoreURL = r.templatesConfig.IgnoreURL
|
||||
}
|
||||
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
|
||||
|
||||
checkedIgnore = true
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
|
||||
if reqErr == nil {
|
||||
@ -106,7 +123,7 @@ func (r *Runner) updateTemplates() error {
|
||||
}
|
||||
|
||||
// Use custom location if user has given a template directory
|
||||
r.templatesConfig = &nucleiConfig{
|
||||
r.templatesConfig = &config.Config{
|
||||
TemplatesDirectory: path.Join(home, "nuclei-templates"),
|
||||
}
|
||||
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
|
||||
@ -120,13 +137,14 @@ func (r *Runner) updateTemplates() error {
|
||||
}
|
||||
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
|
||||
|
||||
r.fetchLatestVersionsFromGithub() // also fetch latest versions
|
||||
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.templatesConfig.CurrentVersion = version.String()
|
||||
|
||||
err = r.writeConfiguration(r.templatesConfig)
|
||||
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -162,13 +180,13 @@ func (r *Runner) updateTemplates() error {
|
||||
|
||||
if version.EQ(oldVersion) {
|
||||
gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
|
||||
return r.writeConfiguration(r.templatesConfig)
|
||||
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
|
||||
}
|
||||
|
||||
if version.GT(oldVersion) {
|
||||
if !r.options.UpdateTemplates {
|
||||
gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
|
||||
return r.writeConfiguration(r.templatesConfig)
|
||||
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
|
||||
}
|
||||
|
||||
if r.options.TemplatesDirectory != "" {
|
||||
@ -177,11 +195,12 @@ func (r *Runner) updateTemplates() error {
|
||||
r.templatesConfig.CurrentVersion = version.String()
|
||||
|
||||
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
|
||||
r.fetchLatestVersionsFromGithub()
|
||||
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = r.writeConfiguration(r.templatesConfig)
|
||||
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -461,3 +480,55 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
|
||||
}
|
||||
table.Render()
|
||||
}
|
||||
|
||||
// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
|
||||
func (r *Runner) fetchLatestVersionsFromGithub() {
|
||||
nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
|
||||
}
|
||||
templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
|
||||
}
|
||||
if r.templatesConfig != nil {
|
||||
r.templatesConfig.NucleiLatestVersion = nucleiLatest
|
||||
r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
|
||||
}
|
||||
}
|
||||
|
||||
type githubTagData struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
// githubFetchLatestTagRepo fetches latest tag from github
|
||||
func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var tags []githubTagData
|
||||
err = json.Unmarshal(body, &tags)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(tags) == 0 {
|
||||
return "", fmt.Errorf("no tags found for %s", repo)
|
||||
}
|
||||
return tags[0].Name, nil
|
||||
}
|
||||
|
||||
@ -43,7 +43,6 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
|
||||
defer os.RemoveAll(templatesDirectory)
|
||||
|
||||
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}}
|
||||
|
||||
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
|
||||
require.Nil(t, err, "could not download release and unzip")
|
||||
require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
|
||||
|
||||
@ -3,7 +3,6 @@ package config
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
@ -19,13 +18,14 @@ type Config struct {
|
||||
IgnoreURL string `json:"ignore-url,omitempty"`
|
||||
NucleiVersion string `json:"nuclei-version,omitempty"`
|
||||
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
|
||||
|
||||
NucleiLatestVersion string `json:"nuclei-latest-version"`
|
||||
NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
|
||||
}
|
||||
|
||||
// nucleiConfigFilename is the filename of nuclei configuration file.
|
||||
const nucleiConfigFilename = ".templates-config.json"
|
||||
|
||||
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
|
||||
|
||||
// Version is the current version of nuclei
|
||||
const Version = `2.3.8`
|
||||
|
||||
@ -59,12 +59,16 @@ func ReadConfiguration() (*Config, error) {
|
||||
}
|
||||
|
||||
// WriteConfiguration writes the updated nuclei configuration to disk
|
||||
func WriteConfiguration(config *Config) error {
|
||||
func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
|
||||
if config.IgnoreURL == "" {
|
||||
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
|
||||
}
|
||||
if checked {
|
||||
config.LastChecked = time.Now()
|
||||
}
|
||||
if checkedIgnore {
|
||||
config.LastCheckedIgnore = time.Now()
|
||||
}
|
||||
config.NucleiVersion = Version
|
||||
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
|
||||
if err != nil {
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"gopkg.in/yaml.v2"
|
||||
@ -28,6 +29,7 @@ type Config struct {
|
||||
IncludeTags []string
|
||||
|
||||
Catalog *catalog.Catalog
|
||||
ExecutorOptions protocols.ExecuterOptions
|
||||
TemplatesDirectory string
|
||||
}
|
||||
|
||||
@ -37,6 +39,9 @@ type Store struct {
|
||||
config *Config
|
||||
finalTemplates []string
|
||||
templateMatched bool
|
||||
|
||||
templates []*templates.Template
|
||||
workflows []*templates.Template
|
||||
}
|
||||
|
||||
// New creates a new template store based on provided configuration
|
||||
@ -47,7 +52,6 @@ func New(config *Config) (*Store, error) {
|
||||
tagFilter: config.createTagFilter(),
|
||||
}
|
||||
|
||||
// hasFindByMetadata := config.hasFindByMetadata()
|
||||
// Handle a case with no templates or workflows, where we use base directory
|
||||
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
|
||||
config.Templates = append(config.Templates, config.TemplatesDirectory)
|
||||
@ -55,13 +59,24 @@ func New(config *Config) (*Store, error) {
|
||||
store.templateMatched = true
|
||||
}
|
||||
store.finalTemplates = append(store.finalTemplates, config.Templates...)
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// Templates returns all the templates in the store
|
||||
func (s *Store) Templates() []*templates.Template {
|
||||
return s.templates
|
||||
}
|
||||
|
||||
// Workflows returns all the workflows in the store
|
||||
func (s *Store) Workflows() []*templates.Template {
|
||||
return s.workflows
|
||||
}
|
||||
|
||||
// Load loads all the templates from a store, performs filtering and returns
|
||||
// the complete compiled templates for a nuclei execution configuration.
|
||||
func (s *Store) Load() (templates []string, workflows []string) {
|
||||
includedTemplates := s.config.Catalog.GetTemplatesPath(s.config.Templates)
|
||||
func (s *Store) Load() {
|
||||
includedTemplates := s.config.Catalog.GetTemplatesPath(s.finalTemplates)
|
||||
includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
|
||||
excludedTemplates := s.config.Catalog.GetTemplatesPath(s.config.ExcludeTemplates)
|
||||
alwaysIncludeTemplates := s.config.Catalog.GetTemplatesPath(s.config.IncludeTemplates)
|
||||
@ -89,7 +104,12 @@ func (s *Store) Load() (templates []string, workflows []string) {
|
||||
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
|
||||
}
|
||||
if loaded {
|
||||
templates = append(templates, k)
|
||||
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
|
||||
} else if parsed != nil {
|
||||
s.templates = append(s.templates, parsed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -110,10 +130,14 @@ func (s *Store) Load() (templates []string, workflows []string) {
|
||||
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
|
||||
}
|
||||
if loaded {
|
||||
workflows = append(workflows, k)
|
||||
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
|
||||
} else if parsed != nil {
|
||||
s.workflows = append(s.workflows, parsed)
|
||||
}
|
||||
}
|
||||
}
|
||||
return templates, workflows
|
||||
}
|
||||
|
||||
// loadTemplateParseMetadata loads a template by parsing metadata and running
|
||||
|
||||
@ -189,49 +189,3 @@ func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, o
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// matchTemplateWithTags matches if the template matches a tag
|
||||
func matchTemplateWithTags(tags, severity string, tagsInput []string) error {
|
||||
actualTags := strings.Split(tags, ",")
|
||||
if severity != "" {
|
||||
actualTags = append(actualTags, severity) // also add severity to tag
|
||||
}
|
||||
|
||||
matched := false
|
||||
mainLoop:
|
||||
for _, t := range tagsInput {
|
||||
commaTags := strings.Split(t, ",")
|
||||
for _, tag := range commaTags {
|
||||
tag = strings.TrimSpace(tag)
|
||||
key, value := getKeyValue(tag)
|
||||
|
||||
for _, templTag := range actualTags {
|
||||
templTag = strings.TrimSpace(templTag)
|
||||
tKey, tValue := getKeyValue(templTag)
|
||||
|
||||
if strings.EqualFold(key, tKey) && strings.EqualFold(value, tValue) {
|
||||
matched = true
|
||||
break mainLoop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return errors.New("could not match template tags with input")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getKeyValue returns key value pair for a data string
|
||||
func getKeyValue(data string) (key, value string) {
|
||||
if strings.Contains(data, ":") {
|
||||
parts := strings.SplitN(data, ":", 2)
|
||||
if len(parts) == 2 {
|
||||
key, value = parts[0], parts[1]
|
||||
}
|
||||
}
|
||||
if value == "" {
|
||||
value = data
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
@ -1,41 +0,0 @@
|
||||
package templates
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMatchTemplateWithTags(t *testing.T) {
|
||||
err := matchTemplateWithTags("php,linux,symfony", "", []string{"php"})
|
||||
require.Nil(t, err, "could not get php tag from input slice")
|
||||
|
||||
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"cms:symfony"})
|
||||
require.Nil(t, err, "could not get php tag from input key value")
|
||||
|
||||
err = matchTemplateWithTags("lang:php,os:linux,symfony", "", []string{"cms:symfony"})
|
||||
require.NotNil(t, err, "could get key value tag from input key value")
|
||||
|
||||
err = matchTemplateWithTags("lang:php,os:linux,cms:jira", "", []string{"cms:symfony"})
|
||||
require.NotNil(t, err, "could get key value tag from input key value")
|
||||
|
||||
t.Run("space", func(t *testing.T) {
|
||||
err = matchTemplateWithTags("lang:php, os:linux, cms:symfony", "", []string{"cms:symfony"})
|
||||
require.Nil(t, err, "could get key value tag from input key value with space")
|
||||
})
|
||||
|
||||
t.Run("comma-tags", func(t *testing.T) {
|
||||
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"test,cms:symfony"})
|
||||
require.Nil(t, err, "could get key value tag from input key value with comma")
|
||||
})
|
||||
|
||||
t.Run("severity", func(t *testing.T) {
|
||||
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "low", []string{"low"})
|
||||
require.Nil(t, err, "could get key value tag for severity")
|
||||
})
|
||||
|
||||
t.Run("blank-tags", func(t *testing.T) {
|
||||
err = matchTemplateWithTags("", "low", []string{"jira"})
|
||||
require.NotNil(t, err, "could get value tag for blank severity")
|
||||
})
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user