External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 19:35:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added initial catalog interface implementation (#2318)

Browse Source 
* Added initial catalog interface implementation

* Added OpenFile to Catalog + disk catalog implementation

* Fixed merge issues

Co-authored-by: sandeep <sandeep@projectdiscovery.io>
This commit is contained in:
Ice3man 2022-08-10 23:35:58 +05:30 committed by GitHub
parent 4dc98a1d95
commit 67d5769cd9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 120 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
v2/cmd/cve-annotate/main.go
View File

@ -13,12 +13,11 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nvd"
"github.com/projectdiscovery/sliceutil"
"github.com/projectdiscovery/stringsutil"
"gopkg.in/yaml.v3"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
)
const (
@ -61,7 +60,7 @@ func process() error {
if err != nil {
return err
}
catalog := catalog.New(*templateDir)
catalog := disk.NewCatalog(*templateDir)
paths, err := catalog.GetTemplatePath(*input)
if err != nil {

4
v2/cmd/integration-test/code.go
View File

@ -13,8 +13,8 @@ import (
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
@ -89,7 +89,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
defer interactClient.Close()
home, _ := os.UserHomeDir()
catalog := catalog.New(path.Join(home, "nuclei-templates"))
catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
executerOpts := protocols.ExecuterOptions{
Output: outputWriter,
Options: defaultOpts,

2
v2/go.mod
View File

@ -74,7 +74,7 @@ require (
github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7
github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963
github.com/projectdiscovery/iputil v0.0.0-20220620153941-036d511e4097
github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d
github.com/projectdiscovery/nvd v1.0.9
github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9
github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
github.com/projectdiscovery/wappalyzergo v0.0.54

4
v2/go.sum
View File

@ -559,8 +559,8 @@ github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALateb
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs=
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d h1:WK/zpWYGuZhAudzVjbnCYk12L4SqH66jagBzHRm3eCo=
github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og=
github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS6jzjE=
github.com/projectdiscovery/nvd v1.0.9/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og=
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.9-0.20220726060557-a045ab711701 h1:yD0xcwbqfxluOpThWJKBaIFJmsHYsBqVU5n5+6EoRJE=
github.com/projectdiscovery/rawhttp v0.0.9-0.20220726060557-a045ab711701/go.mod h1:Q5PDAmKzjAjweEp0CQr9301nyxCOkzA9ImK6qLjgk+8=

5
v2/internal/runner/runner.go
View File

@ -24,6 +24,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid"
@ -58,7 +59,7 @@ type Runner struct {
templatesConfig *config.Config
options *types.Options
projectFile *projectfile.ProjectFile
catalog *catalog.Catalog
catalog catalog.Catalog
progress progress.Progress
colorizer aurora.Aurora
issuesClient *reporting.Client
@ -110,7 +111,7 @@ func New(options *types.Options) (*Runner, error) {
runner.browser = browser
}
runner.catalog = catalog.New(runner.options.TemplatesDirectory)
runner.catalog = disk.NewCatalog(runner.options.TemplatesDirectory)
var httpclient *retryablehttp.Client
if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {

2
v2/internal/runner/templates.go
View File

@ -13,7 +13,7 @@ import (
)
func (r *Runner) logAvailableTemplate(tplPath string) {
t, err := parsers.ParseTemplate(tplPath)
t, err := parsers.ParseTemplate(tplPath, r.catalog)
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {

26
v2/pkg/catalog/catalogue.go
View File

@ -1,12 +1,22 @@
package catalog
// Catalog is a template catalog helper implementation
type Catalog struct {
templatesDirectory string
}
import "io"
// New creates a new Catalog structure using provided input items
func New(directory string) *Catalog {
catalog := &Catalog{templatesDirectory: directory}
return catalog
// Catalog is a catalog storage implementations
type Catalog interface {
// OpenFile opens a file and returns an io.ReadCloser to the file.
// It is used to read template and payload files based on catalog responses.
OpenFile(filename string) (io.ReadCloser, error)
// GetTemplatePath parses the specified input template path and returns a compiled
// list of finished absolute paths to the templates evaluating any glob patterns
// or folders provided as in.
GetTemplatePath(target string) ([]string, error)
// GetTemplatesPath returns a list of absolute paths for the provided template list.
GetTemplatesPath(definitions []string) []string
// ResolvePath resolves the path to an absolute one in various ways.
//
// It checks if the filename is an absolute path, looks in the current directory
// or checking the nuclei templates directory. If a second path is given,
// it also tries to find paths relative to that second path.
ResolvePath(templateName, second string) (string, error)
}

25
v2/pkg/catalog/disk/catalog.go Normal file
View File

@ -0,0 +1,25 @@
package disk
import (
"io"
"os"
)
// DiskCatalog is a template catalog helper implementation based on disk
type DiskCatalog struct {
templatesDirectory string
}
// NewCatalog creates a new Catalog structure using provided input items
// using disk based items
func NewCatalog(directory string) *DiskCatalog {
catalog := &DiskCatalog{templatesDirectory: directory}
return catalog
}
// OpenFile opens a file and returns an io.ReadCloser to the file.
// It is used to read template and payload files based on catalog responses.
func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
file, err := os.Open(filename)
return file, err
}

15
v2/pkg/catalog/find.go → v2/pkg/catalog/disk/find.go
View File

@ -1,4 +1,4 @@
package catalog
package disk
import (
"io/fs"
@ -7,12 +7,11 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
)
// GetTemplatesPath returns a list of absolute paths for the provided template list.
func (c *Catalog) GetTemplatesPath(definitions []string) []string {
func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
// keeps track of processed dirs and files
processed := make(map[string]bool)
allTemplates := []string{}
@ -42,7 +41,7 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string {
// GetTemplatePath parses the specified input template path and returns a compiled
// list of finished absolute paths to the templates evaluating any glob patterns
// or folders provided as in.
func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
processed := make(map[string]struct{})
absPath, err := c.convertPathToAbsolute(target)
@ -88,7 +87,7 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
// convertPathToAbsolute resolves the paths provided to absolute paths
// before doing any operations on them regardless of them being BLOB, folders, files, etc.
func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) {
if strings.Contains(t, "*") {
file := filepath.Base(t)
absPath, err := c.ResolvePath(filepath.Dir(t), "")
@ -101,7 +100,7 @@ func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
}
// findGlobPathMatches returns the matched files from a glob path
func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
matches, err := filepath.Glob(absPath)
if err != nil {
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
@ -118,7 +117,7 @@ func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struc
// findFileMatches finds if a path is an absolute file. If the path
// is a file, it returns true otherwise false with no errors.
func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
info, err := os.Stat(absPath)
if err != nil {
return "", false, err
@ -134,7 +133,7 @@ func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{})
}
// findDirectoryMatches finds matches for templates from a directory
func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
var results []string
err := filepath.WalkDir(
absPath,

6
v2/pkg/catalog/path.go → v2/pkg/catalog/disk/path.go
View File

@ -1,4 +1,4 @@
package catalog
package disk
import (
"fmt"
@ -15,7 +15,7 @@ import (
// It checks if the filename is an absolute path, looks in the current directory
// or checking the nuclei templates directory. If a second path is given,
// it also tries to find paths relative to that second path.
func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) {
if filepath.IsAbs(templateName) {
return templateName, nil
}
@ -48,7 +48,7 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
var errNoValidCombination = errors.New("no valid combination found")
// tryResolve attempts to load locate the target by iterating across all the folders tree
func (c *Catalog) tryResolve(fullPath string) (string, error) {
func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
dir, filename := filepath.Split(fullPath)
pathInfo, err := folderutil.NewPathInfo(dir)
if err != nil {

2
v2/pkg/catalog/loader/filter/path_filter.go
View File

@ -15,7 +15,7 @@ type PathFilterConfig struct {
}
// NewPathFilter creates a new path filter from provided config
func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter {
func NewPathFilter(config *PathFilterConfig, catalogClient catalog.Catalog) *PathFilter {
filter := &PathFilter{
excludedTemplates: catalogClient.GetTemplatesPath(config.ExcludedTemplates),
alwaysIncludedTemplatesMap: make(map[string]struct{}),

14
v2/pkg/catalog/loader/loader.go
View File

@ -39,7 +39,7 @@ type Config struct {
IncludeIds []string
ExcludeIds []string
Catalog *catalog.Catalog
Catalog catalog.Catalog
ExecutorOptions protocols.ExecuterOptions
TemplatesDirectory string
}
@ -59,7 +59,7 @@ type Store struct {
}
// NewConfig returns a new loader config
func NewConfig(options *types.Options, templateConfig *config.Config, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
func NewConfig(options *types.Options, templateConfig *config.Config, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
loaderConfig := Config{
Templates: options.Templates,
Workflows: options.Workflows,
@ -180,13 +180,13 @@ func (store *Store) ValidateTemplates() error {
func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, true, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
return parsers.LoadWorkflow(templatePath)
return parsers.LoadWorkflow(templatePath, store.config.Catalog)
})
}
func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, false, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
return parsers.LoadTemplate(templatePath, store.tagFilter, nil)
return parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
})
}
@ -260,7 +260,7 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
if loaded || store.pathFilter.MatchIncluded(templatePath) {
parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
@ -283,7 +283,7 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
for workflowPath := range workflowPathMap {
loaded, err := parsers.LoadWorkflow(workflowPath)
loaded, err := parsers.LoadWorkflow(workflowPath, store.config.Catalog)
if err != nil {
gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
}
@ -307,7 +307,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags)
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
if loaded || store.pathFilter.MatchIncluded(templatePath) {
parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {

13
v2/pkg/catalog/loader/loader_test.go
View File

@ -4,12 +4,16 @@ import (
"reflect"
"testing"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/stretchr/testify/require"
)
func TestLoadTemplates(t *testing.T) {
catalog := disk.NewCatalog("")
store, err := New(&Config{
Templates: []string{"cves/CVE-2021-21315.yaml"},
Catalog: catalog,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
@ -18,6 +22,7 @@ func TestLoadTemplates(t *testing.T) {
t.Run("blank", func(t *testing.T) {
store, err := New(&Config{
TemplatesDirectory: templatesDirectory,
Catalog: catalog,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
@ -26,6 +31,7 @@ func TestLoadTemplates(t *testing.T) {
store, err := New(&Config{
Tags: []string{"cves"},
TemplatesDirectory: templatesDirectory,
Catalog: catalog,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
@ -34,6 +40,7 @@ func TestLoadTemplates(t *testing.T) {
store, err := New(&Config{
Tags: []string{"cves"},
TemplatesDirectory: templatesDirectory,
Catalog: catalog,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
@ -41,6 +48,8 @@ func TestLoadTemplates(t *testing.T) {
}
func TestRemoteTemplates(t *testing.T) {
catalog := disk.NewCatalog("")
var nilStringSlice []string
type args struct {
config *Config
@ -56,7 +65,8 @@ func TestRemoteTemplates(t *testing.T) {
args: args{
config: &Config{
TemplateURLs: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
RemoteTemplateDomainList: []string{"localhost","raw.githubusercontent.com"},
RemoteTemplateDomainList: []string{"localhost", "raw.githubusercontent.com"},
Catalog: catalog,
},
},
want: &Store{
@ -70,6 +80,7 @@ func TestRemoteTemplates(t *testing.T) {
config: &Config{
TemplateURLs: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
RemoteTemplateDomainList: []string{"localhost"},
Catalog: catalog,
},
},
want: &Store{

13
v2/pkg/parsers/parser.go
View File

@ -7,6 +7,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@ -22,8 +23,8 @@ const (
)
// LoadTemplate returns true if the template is valid and matches the filtering criteria.
func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string, catalog catalog.Catalog) (bool, error) {
template, templateParseError := ParseTemplate(templatePath, catalog)
if templateParseError != nil {
return false, templateParseError
}
@ -43,8 +44,8 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
}
// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
func LoadWorkflow(templatePath string) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
func LoadWorkflow(templatePath string, catalog catalog.Catalog) (bool, error) {
template, templateParseError := ParseTemplate(templatePath, catalog)
if templateParseError != nil {
return false, templateParseError
}
@ -122,11 +123,11 @@ func init() {
}
// ParseTemplate parses a template and returns a *templates.Template structure
func ParseTemplate(templatePath string) (*templates.Template, error) {
func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Template, error) {
if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
return value.(*templates.Template), err
}
data, err := utils.ReadFromPathOrURL(templatePath)
data, err := utils.ReadFromPathOrURL(templatePath, catalog)
if err != nil {
return nil, err
}

6
v2/pkg/parsers/parser_test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
@ -14,6 +15,7 @@ import (
)
func TestLoadTemplate(t *testing.T) {
catalog := disk.NewCatalog("")
origTemplatesCache := parsedTemplatesCache
defer func() { parsedTemplatesCache = origTemplatesCache }()
@ -56,7 +58,7 @@ func TestLoadTemplate(t *testing.T) {
parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr)
tagFilter := filter.New(&filter.Config{})
success, err := LoadTemplate(tc.name, tagFilter, nil)
success, err := LoadTemplate(tc.name, tagFilter, nil, catalog)
if tc.expectedErr == nil {
require.NoError(t, err)
require.True(t, success)
@ -97,7 +99,7 @@ func TestLoadTemplate(t *testing.T) {
parsedTemplatesCache.Store(name, template, nil)
tagFilter := filter.New(&filter.Config{})
success, err := LoadTemplate(name, tagFilter, nil)
success, err := LoadTemplate(name, tagFilter, nil, catalog)
if tc.success {
require.NoError(t, err)
require.True(t, success)

4
v2/pkg/parsers/workflow_loader.go
View File

@ -37,7 +37,7 @@ func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string
loadedTemplates := make([]string, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags, w.options.Catalog)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if loaded {
@ -53,7 +53,7 @@ func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate boo
loadedTemplates := make([]string, 0, len(templatesPathMap))
for templatePath := range templatesPathMap {
matched, err := LoadTemplate(templatePath, w.tagFilter, nil)
matched, err := LoadTemplate(templatePath, w.tagFilter, nil, w.options.Catalog)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if matched || noValidate {

7
v2/pkg/protocols/common/generators/generators.go
View File

@ -11,11 +11,12 @@ import (
// PayloadGenerator is the generator struct for generating payloads
type PayloadGenerator struct {
Type AttackType
catalog catalog.Catalog
payloads map[string][]string
}
// New creates a new generator structure for payload generation
func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*PayloadGenerator, error) {
func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog) (*PayloadGenerator, error) {
if attackType.String() == "" {
attackType = BatteringRamAttack
}
@ -36,12 +37,12 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
}
}
generator := &PayloadGenerator{}
generator := &PayloadGenerator{catalog: catalog}
if err := generator.validate(payloadsFinal, templatePath); err != nil {
return nil, err
}
compiled, err := loadPayloads(payloadsFinal)
compiled, err := generator.loadPayloads(payloadsFinal)
if err != nil {
return nil, err
}

8
v2/pkg/protocols/common/generators/generators_test.go
View File

@ -5,13 +5,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
)
func TestBatteringRamGenerator(t *testing.T) {
usernames := []string{"admin", "password"}
catalogInstance := catalog.New("")
catalogInstance := disk.NewCatalog("")
generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance)
require.Nil(t, err, "could not create generator")
@ -31,7 +31,7 @@ func TestPitchforkGenerator(t *testing.T) {
usernames := []string{"admin", "token"}
passwords := []string{"password1", "password2", "password3"}
catalogInstance := catalog.New("")
catalogInstance := disk.NewCatalog("")
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance)
require.Nil(t, err, "could not create generator")
@ -53,7 +53,7 @@ func TestClusterbombGenerator(t *testing.T) {
usernames := []string{"admin"}
passwords := []string{"admin", "password", "token"}
catalogInstance := catalog.New("")
catalogInstance := disk.NewCatalog("")
generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance)
require.Nil(t, err, "could not create generator")

9
v2/pkg/protocols/common/generators/load.go
View File

@ -3,7 +3,6 @@ package generators
import (
"bufio"
"io"
"os"
"strings"
"github.com/pkg/errors"
@ -11,7 +10,7 @@ import (
)
// loadPayloads loads the input payloads from a map to a data map
func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) {
func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}) (map[string][]string, error) {
loadedPayloads := make(map[string][]string)
for name, payload := range payloads {
@ -22,7 +21,7 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error)
if len(elements) >= 2 {
loadedPayloads[name] = elements
} else {
payloads, err := loadPayloadsFromFile(pt)
payloads, err := generator.loadPayloadsFromFile(pt)
if err != nil {
return nil, errors.Wrap(err, "could not load payloads")
}
@ -36,10 +35,10 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error)
}
// loadPayloadsFromFile loads a file to a string slice
func loadPayloadsFromFile(filepath string) ([]string, error) {
func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]string, error) {
var lines []string
file, err := os.Open(filepath)
file, err := generator.catalog.OpenFile(filepath)
if err != nil {
return nil, err
}

6
v2/pkg/protocols/http/request_generator_test.go
View File

@ -5,7 +5,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
)
@ -33,7 +33,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) {
AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`},
}
catalogInstance := catalog.New("")
catalogInstance := disk.NewCatalog("")
req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance)
require.Nil(t, err, "could not create generator")
@ -57,7 +57,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) {
AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},
}
catalogInstance := catalog.New("")
catalogInstance := disk.NewCatalog("")
req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance)
require.Nil(t, err, "could not create generator")

2
v2/pkg/protocols/protocols.go
View File

@ -54,7 +54,7 @@ type ExecuterOptions struct {
// RateLimiter is a rate-limiter for limiting sent number of requests.
RateLimiter ratelimit.Limiter
// Catalog is a template catalog implementation for nuclei
Catalog *catalog.Catalog
Catalog catalog.Catalog
// ProjectFile is the project file for nuclei
ProjectFile *projectfile.ProjectFile
// Browser is a browser engine for running headless templates

2
v2/pkg/templates/compile.go
View File

@ -36,7 +36,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
template := &Template{}
data, err := utils.ReadFromPathOrURL(filePath)
data, err := utils.ReadFromPathOrURL(filePath, options.Catalog)
if err != nil {
return nil, err
}

4
v2/pkg/testutils/testutils.go
View File

@ -6,7 +6,7 @@ import (
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -85,7 +85,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
ProjectFile: nil,
IssuesClient: nil,
Browser: nil,
Catalog: catalog.New(options.TemplatesDirectory),
Catalog: disk.NewCatalog(options.TemplatesDirectory),
RateLimiter: ratelimit.New(options.RateLimit),
}
return executerOpts

7
v2/pkg/utils/utils.go
View File

@ -5,8 +5,9 @@ import (
"io"
"net/http"
"net/url"
"os"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
)
func IsBlank(value string) bool {
@ -44,7 +45,7 @@ func IsURL(input string) bool {
}
// ReadFromPathOrURL reads and returns the contents of a file or url.
func ReadFromPathOrURL(templatePath string) (data []byte, err error) {
func ReadFromPathOrURL(templatePath string, catalog catalog.Catalog) (data []byte, err error) {
if IsURL(templatePath) {
resp, err := http.Get(templatePath)
if err != nil {
@ -56,7 +57,7 @@ func ReadFromPathOrURL(templatePath string) (data []byte, err error) {
return nil, err
}
} else {
f, err := os.Open(templatePath)
f, err := catalog.OpenFile(templatePath)
if err != nil {
return nil, err
}