From 67d5769cd9e4320b15ecca25e91c92b77d8cdb21 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 10 Aug 2022 23:35:58 +0530 Subject: [PATCH] Added initial catalog interface implementation (#2318) * Added initial catalog interface implementation * Added OpenFile to Catalog + disk catalog implementation * Fixed merge issues Co-authored-by: sandeep --- v2/cmd/cve-annotate/main.go | 5 ++-- v2/cmd/integration-test/code.go | 4 +-- v2/go.mod | 2 +- v2/go.sum | 4 +-- v2/internal/runner/runner.go | 5 ++-- v2/internal/runner/templates.go | 2 +- v2/pkg/catalog/catalogue.go | 26 +++++++++++++------ v2/pkg/catalog/disk/catalog.go | 25 ++++++++++++++++++ v2/pkg/catalog/{ => disk}/find.go | 15 +++++------ v2/pkg/catalog/{ => disk}/path.go | 6 ++--- v2/pkg/catalog/loader/filter/path_filter.go | 2 +- v2/pkg/catalog/loader/loader.go | 14 +++++----- v2/pkg/catalog/loader/loader_test.go | 13 +++++++++- v2/pkg/parsers/parser.go | 13 +++++----- v2/pkg/parsers/parser_test.go | 6 +++-- v2/pkg/parsers/workflow_loader.go | 4 +-- .../protocols/common/generators/generators.go | 7 ++--- .../common/generators/generators_test.go | 8 +++--- v2/pkg/protocols/common/generators/load.go | 9 +++---- .../protocols/http/request_generator_test.go | 6 ++--- v2/pkg/protocols/protocols.go | 2 +- v2/pkg/templates/compile.go | 2 +- v2/pkg/testutils/testutils.go | 4 +-- v2/pkg/utils/utils.go | 7 ++--- 24 files changed, 120 insertions(+), 71 deletions(-) create mode 100644 v2/pkg/catalog/disk/catalog.go rename v2/pkg/catalog/{ => disk}/find.go (87%) rename v2/pkg/catalog/{ => disk}/path.go (91%) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index 5e6bfeabe..f64bb9d87 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -13,12 +13,11 @@ import ( "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nvd" "github.com/projectdiscovery/sliceutil" "github.com/projectdiscovery/stringsutil" "gopkg.in/yaml.v3" - - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) const ( @@ -61,7 +60,7 @@ func process() error { if err != nil { return err } - catalog := catalog.New(*templateDir) + catalog := disk.NewCatalog(*templateDir) paths, err := catalog.GetTemplatePath(*input) if err != nil { diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index 1d6da50fc..58f518e99 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -13,8 +13,8 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/goflags" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs" @@ -89,7 +89,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) { defer interactClient.Close() home, _ := os.UserHomeDir() - catalog := catalog.New(path.Join(home, "nuclei-templates")) + catalog := disk.NewCatalog(path.Join(home, "nuclei-templates")) executerOpts := protocols.ExecuterOptions{ Output: outputWriter, Options: defaultOpts, diff --git a/v2/go.mod b/v2/go.mod index 577559986..b0274cb5b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -74,7 +74,7 @@ require ( github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 github.com/projectdiscovery/iputil v0.0.0-20220620153941-036d511e4097 - github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d + github.com/projectdiscovery/nvd v1.0.9 github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9 github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 github.com/projectdiscovery/wappalyzergo v0.0.54 diff --git a/v2/go.sum b/v2/go.sum index b6ba696b6..7ff2a6bcf 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -559,8 +559,8 @@ github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALateb github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs= github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= -github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d h1:WK/zpWYGuZhAudzVjbnCYk12L4SqH66jagBzHRm3eCo= -github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og= +github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS6jzjE= +github.com/projectdiscovery/nvd v1.0.9/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/rawhttp v0.0.9-0.20220726060557-a045ab711701 h1:yD0xcwbqfxluOpThWJKBaIFJmsHYsBqVU5n5+6EoRJE= github.com/projectdiscovery/rawhttp v0.0.9-0.20220726060557-a045ab711701/go.mod h1:Q5PDAmKzjAjweEp0CQr9301nyxCOkzA9ImK6qLjgk+8= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 15c701807..eb9430228 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -24,6 +24,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid" @@ -58,7 +59,7 @@ type Runner struct { templatesConfig *config.Config options *types.Options projectFile *projectfile.ProjectFile - catalog *catalog.Catalog + catalog catalog.Catalog progress progress.Progress colorizer aurora.Aurora issuesClient *reporting.Client @@ -110,7 +111,7 @@ func New(options *types.Options) (*Runner, error) { runner.browser = browser } - runner.catalog = catalog.New(runner.options.TemplatesDirectory) + runner.catalog = disk.NewCatalog(runner.options.TemplatesDirectory) var httpclient *retryablehttp.Client if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index fd6d6abc8..426afb8ee 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -13,7 +13,7 @@ import ( ) func (r *Runner) logAvailableTemplate(tplPath string) { - t, err := parsers.ParseTemplate(tplPath) + t, err := parsers.ParseTemplate(tplPath, r.catalog) if err != nil { gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err) } else { diff --git a/v2/pkg/catalog/catalogue.go b/v2/pkg/catalog/catalogue.go index 38a58d7cc..3d20752bd 100644 --- a/v2/pkg/catalog/catalogue.go +++ b/v2/pkg/catalog/catalogue.go @@ -1,12 +1,22 @@ package catalog -// Catalog is a template catalog helper implementation -type Catalog struct { - templatesDirectory string -} +import "io" -// New creates a new Catalog structure using provided input items -func New(directory string) *Catalog { - catalog := &Catalog{templatesDirectory: directory} - return catalog +// Catalog is a catalog storage implementations +type Catalog interface { + // OpenFile opens a file and returns an io.ReadCloser to the file. + // It is used to read template and payload files based on catalog responses. + OpenFile(filename string) (io.ReadCloser, error) + // GetTemplatePath parses the specified input template path and returns a compiled + // list of finished absolute paths to the templates evaluating any glob patterns + // or folders provided as in. + GetTemplatePath(target string) ([]string, error) + // GetTemplatesPath returns a list of absolute paths for the provided template list. + GetTemplatesPath(definitions []string) []string + // ResolvePath resolves the path to an absolute one in various ways. + // + // It checks if the filename is an absolute path, looks in the current directory + // or checking the nuclei templates directory. If a second path is given, + // it also tries to find paths relative to that second path. + ResolvePath(templateName, second string) (string, error) } diff --git a/v2/pkg/catalog/disk/catalog.go b/v2/pkg/catalog/disk/catalog.go new file mode 100644 index 000000000..4fd53c963 --- /dev/null +++ b/v2/pkg/catalog/disk/catalog.go @@ -0,0 +1,25 @@ +package disk + +import ( + "io" + "os" +) + +// DiskCatalog is a template catalog helper implementation based on disk +type DiskCatalog struct { + templatesDirectory string +} + +// NewCatalog creates a new Catalog structure using provided input items +// using disk based items +func NewCatalog(directory string) *DiskCatalog { + catalog := &DiskCatalog{templatesDirectory: directory} + return catalog +} + +// OpenFile opens a file and returns an io.ReadCloser to the file. +// It is used to read template and payload files based on catalog responses. +func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) { + file, err := os.Open(filename) + return file, err +} diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/disk/find.go similarity index 87% rename from v2/pkg/catalog/find.go rename to v2/pkg/catalog/disk/find.go index fc3fc2255..c1f9727b1 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/disk/find.go @@ -1,4 +1,4 @@ -package catalog +package disk import ( "io/fs" @@ -7,12 +7,11 @@ import ( "strings" "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *Catalog) GetTemplatesPath(definitions []string) []string { +func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} @@ -42,7 +41,7 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string { // GetTemplatePath parses the specified input template path and returns a compiled // list of finished absolute paths to the templates evaluating any glob patterns // or folders provided as in. -func (c *Catalog) GetTemplatePath(target string) ([]string, error) { +func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) { processed := make(map[string]struct{}) absPath, err := c.convertPathToAbsolute(target) @@ -88,7 +87,7 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) { // convertPathToAbsolute resolves the paths provided to absolute paths // before doing any operations on them regardless of them being BLOB, folders, files, etc. -func (c *Catalog) convertPathToAbsolute(t string) (string, error) { +func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) { if strings.Contains(t, "*") { file := filepath.Base(t) absPath, err := c.ResolvePath(filepath.Dir(t), "") @@ -101,7 +100,7 @@ func (c *Catalog) convertPathToAbsolute(t string) (string, error) { } // findGlobPathMatches returns the matched files from a glob path -func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) { +func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) { matches, err := filepath.Glob(absPath) if err != nil { return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) @@ -118,7 +117,7 @@ func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struc // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. -func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) { +func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) { info, err := os.Stat(absPath) if err != nil { return "", false, err @@ -134,7 +133,7 @@ func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) } // findDirectoryMatches finds matches for templates from a directory -func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) { +func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) { var results []string err := filepath.WalkDir( absPath, diff --git a/v2/pkg/catalog/path.go b/v2/pkg/catalog/disk/path.go similarity index 91% rename from v2/pkg/catalog/path.go rename to v2/pkg/catalog/disk/path.go index fc6f8189b..15a2eab31 100644 --- a/v2/pkg/catalog/path.go +++ b/v2/pkg/catalog/disk/path.go @@ -1,4 +1,4 @@ -package catalog +package disk import ( "fmt" @@ -15,7 +15,7 @@ import ( // It checks if the filename is an absolute path, looks in the current directory // or checking the nuclei templates directory. If a second path is given, // it also tries to find paths relative to that second path. -func (c *Catalog) ResolvePath(templateName, second string) (string, error) { +func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) { if filepath.IsAbs(templateName) { return templateName, nil } @@ -48,7 +48,7 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) { var errNoValidCombination = errors.New("no valid combination found") // tryResolve attempts to load locate the target by iterating across all the folders tree -func (c *Catalog) tryResolve(fullPath string) (string, error) { +func (c *DiskCatalog) tryResolve(fullPath string) (string, error) { dir, filename := filepath.Split(fullPath) pathInfo, err := folderutil.NewPathInfo(dir) if err != nil { diff --git a/v2/pkg/catalog/loader/filter/path_filter.go b/v2/pkg/catalog/loader/filter/path_filter.go index ba65bf717..1fb168c1d 100644 --- a/v2/pkg/catalog/loader/filter/path_filter.go +++ b/v2/pkg/catalog/loader/filter/path_filter.go @@ -15,7 +15,7 @@ type PathFilterConfig struct { } // NewPathFilter creates a new path filter from provided config -func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter { +func NewPathFilter(config *PathFilterConfig, catalogClient catalog.Catalog) *PathFilter { filter := &PathFilter{ excludedTemplates: catalogClient.GetTemplatesPath(config.ExcludedTemplates), alwaysIncludedTemplatesMap: make(map[string]struct{}), diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 18e7b3335..a1679cbd1 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -39,7 +39,7 @@ type Config struct { IncludeIds []string ExcludeIds []string - Catalog *catalog.Catalog + Catalog catalog.Catalog ExecutorOptions protocols.ExecuterOptions TemplatesDirectory string } @@ -59,7 +59,7 @@ type Store struct { } // NewConfig returns a new loader config -func NewConfig(options *types.Options, templateConfig *config.Config, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { +func NewConfig(options *types.Options, templateConfig *config.Config, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { loaderConfig := Config{ Templates: options.Templates, Workflows: options.Workflows, @@ -180,13 +180,13 @@ func (store *Store) ValidateTemplates() error { func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool { return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, true, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) { - return parsers.LoadWorkflow(templatePath) + return parsers.LoadWorkflow(templatePath, store.config.Catalog) }) } func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool { return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, false, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) { - return parsers.LoadTemplate(templatePath, store.tagFilter, nil) + return parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog) }) } @@ -260,7 +260,7 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template loadedTemplates := make([]*templates.Template, 0, len(templatePathMap)) for templatePath := range templatePathMap { - loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil) + loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog) if loaded || store.pathFilter.MatchIncluded(templatePath) { parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) if err != nil { @@ -283,7 +283,7 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap)) for workflowPath := range workflowPathMap { - loaded, err := parsers.LoadWorkflow(workflowPath) + loaded, err := parsers.LoadWorkflow(workflowPath, store.config.Catalog) if err != nil { gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err) } @@ -307,7 +307,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ loadedTemplates := make([]*templates.Template, 0, len(templatePathMap)) for templatePath := range templatePathMap { - loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags) + loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog) if loaded || store.pathFilter.MatchIncluded(templatePath) { parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) if err != nil { diff --git a/v2/pkg/catalog/loader/loader_test.go b/v2/pkg/catalog/loader/loader_test.go index ce8c77822..6f624e729 100644 --- a/v2/pkg/catalog/loader/loader_test.go +++ b/v2/pkg/catalog/loader/loader_test.go @@ -4,12 +4,16 @@ import ( "reflect" "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/stretchr/testify/require" ) func TestLoadTemplates(t *testing.T) { + catalog := disk.NewCatalog("") + store, err := New(&Config{ Templates: []string{"cves/CVE-2021-21315.yaml"}, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates") @@ -18,6 +22,7 @@ func TestLoadTemplates(t *testing.T) { t.Run("blank", func(t *testing.T) { store, err := New(&Config{ TemplatesDirectory: templatesDirectory, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") @@ -26,6 +31,7 @@ func TestLoadTemplates(t *testing.T) { store, err := New(&Config{ Tags: []string{"cves"}, TemplatesDirectory: templatesDirectory, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") @@ -34,6 +40,7 @@ func TestLoadTemplates(t *testing.T) { store, err := New(&Config{ Tags: []string{"cves"}, TemplatesDirectory: templatesDirectory, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") @@ -41,6 +48,8 @@ func TestLoadTemplates(t *testing.T) { } func TestRemoteTemplates(t *testing.T) { + catalog := disk.NewCatalog("") + var nilStringSlice []string type args struct { config *Config @@ -56,7 +65,8 @@ func TestRemoteTemplates(t *testing.T) { args: args{ config: &Config{ TemplateURLs: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"}, - RemoteTemplateDomainList: []string{"localhost","raw.githubusercontent.com"}, + RemoteTemplateDomainList: []string{"localhost", "raw.githubusercontent.com"}, + Catalog: catalog, }, }, want: &Store{ @@ -70,6 +80,7 @@ func TestRemoteTemplates(t *testing.T) { config: &Config{ TemplateURLs: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"}, RemoteTemplateDomainList: []string{"localhost"}, + Catalog: catalog, }, }, want: &Store{ diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 912064ab2..b8e847e8f 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -7,6 +7,7 @@ import ( "gopkg.in/yaml.v2" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -22,8 +23,8 @@ const ( ) // LoadTemplate returns true if the template is valid and matches the filtering criteria. -func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) { - template, templateParseError := ParseTemplate(templatePath) +func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string, catalog catalog.Catalog) (bool, error) { + template, templateParseError := ParseTemplate(templatePath, catalog) if templateParseError != nil { return false, templateParseError } @@ -43,8 +44,8 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. -func LoadWorkflow(templatePath string) (bool, error) { - template, templateParseError := ParseTemplate(templatePath) +func LoadWorkflow(templatePath string, catalog catalog.Catalog) (bool, error) { + template, templateParseError := ParseTemplate(templatePath, catalog) if templateParseError != nil { return false, templateParseError } @@ -122,11 +123,11 @@ func init() { } // ParseTemplate parses a template and returns a *templates.Template structure -func ParseTemplate(templatePath string) (*templates.Template, error) { +func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Template, error) { if value, err := parsedTemplatesCache.Has(templatePath); value != nil { return value.(*templates.Template), err } - data, err := utils.ReadFromPathOrURL(templatePath) + data, err := utils.ReadFromPathOrURL(templatePath, catalog) if err != nil { return nil, err } diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index 32a69ef5b..9eb66016e 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" @@ -14,6 +15,7 @@ import ( ) func TestLoadTemplate(t *testing.T) { + catalog := disk.NewCatalog("") origTemplatesCache := parsedTemplatesCache defer func() { parsedTemplatesCache = origTemplatesCache }() @@ -56,7 +58,7 @@ func TestLoadTemplate(t *testing.T) { parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr) tagFilter := filter.New(&filter.Config{}) - success, err := LoadTemplate(tc.name, tagFilter, nil) + success, err := LoadTemplate(tc.name, tagFilter, nil, catalog) if tc.expectedErr == nil { require.NoError(t, err) require.True(t, success) @@ -97,7 +99,7 @@ func TestLoadTemplate(t *testing.T) { parsedTemplatesCache.Store(name, template, nil) tagFilter := filter.New(&filter.Config{}) - success, err := LoadTemplate(name, tagFilter, nil) + success, err := LoadTemplate(name, tagFilter, nil, catalog) if tc.success { require.NoError(t, err) require.True(t, success) diff --git a/v2/pkg/parsers/workflow_loader.go b/v2/pkg/parsers/workflow_loader.go index 4b73f5559..70669e9ba 100644 --- a/v2/pkg/parsers/workflow_loader.go +++ b/v2/pkg/parsers/workflow_loader.go @@ -37,7 +37,7 @@ func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string loadedTemplates := make([]string, 0, len(templatePathMap)) for templatePath := range templatePathMap { - loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags) + loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags, w.options.Catalog) if err != nil { gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err) } else if loaded { @@ -53,7 +53,7 @@ func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate boo loadedTemplates := make([]string, 0, len(templatesPathMap)) for templatePath := range templatesPathMap { - matched, err := LoadTemplate(templatePath, w.tagFilter, nil) + matched, err := LoadTemplate(templatePath, w.tagFilter, nil, w.options.Catalog) if err != nil { gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err) } else if matched || noValidate { diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index fb1597d25..b99d4cc5c 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -11,11 +11,12 @@ import ( // PayloadGenerator is the generator struct for generating payloads type PayloadGenerator struct { Type AttackType + catalog catalog.Catalog payloads map[string][]string } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -36,12 +37,12 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st } } - generator := &PayloadGenerator{} + generator := &PayloadGenerator{catalog: catalog} if err := generator.validate(payloadsFinal, templatePath); err != nil { return nil, err } - compiled, err := loadPayloads(payloadsFinal) + compiled, err := generator.loadPayloads(payloadsFinal) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 24a9e2731..a3ba1bbec 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -5,13 +5,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" ) func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} - catalogInstance := catalog.New("") + catalogInstance := disk.NewCatalog("") generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") @@ -31,7 +31,7 @@ func TestPitchforkGenerator(t *testing.T) { usernames := []string{"admin", "token"} passwords := []string{"password1", "password2", "password3"} - catalogInstance := catalog.New("") + catalogInstance := disk.NewCatalog("") generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") @@ -53,7 +53,7 @@ func TestClusterbombGenerator(t *testing.T) { usernames := []string{"admin"} passwords := []string{"admin", "password", "token"} - catalogInstance := catalog.New("") + catalogInstance := disk.NewCatalog("") generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index d5b00e4d3..248f4c25b 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -3,7 +3,6 @@ package generators import ( "bufio" "io" - "os" "strings" "github.com/pkg/errors" @@ -11,7 +10,7 @@ import ( ) // loadPayloads loads the input payloads from a map to a data map -func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) { +func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}) (map[string][]string, error) { loadedPayloads := make(map[string][]string) for name, payload := range payloads { @@ -22,7 +21,7 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) if len(elements) >= 2 { loadedPayloads[name] = elements } else { - payloads, err := loadPayloadsFromFile(pt) + payloads, err := generator.loadPayloadsFromFile(pt) if err != nil { return nil, errors.Wrap(err, "could not load payloads") } @@ -36,10 +35,10 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) } // loadPayloadsFromFile loads a file to a string slice -func loadPayloadsFromFile(filepath string) ([]string, error) { +func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]string, error) { var lines []string - file, err := os.Open(filepath) + file, err := generator.catalog.OpenFile(filepath) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 8fb0fe70b..2a0666064 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) @@ -33,7 +33,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } - catalogInstance := catalog.New("") + catalogInstance := disk.NewCatalog("") req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance) require.Nil(t, err, "could not create generator") @@ -57,7 +57,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } - catalogInstance := catalog.New("") + catalogInstance := disk.NewCatalog("") req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance) require.Nil(t, err, "could not create generator") diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 57d1ecbb9..0e37e7a17 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -54,7 +54,7 @@ type ExecuterOptions struct { // RateLimiter is a rate-limiter for limiting sent number of requests. RateLimiter ratelimit.Limiter // Catalog is a template catalog implementation for nuclei - Catalog *catalog.Catalog + Catalog catalog.Catalog // ProjectFile is the project file for nuclei ProjectFile *projectfile.ProjectFile // Browser is a browser engine for running headless templates diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index a6dcae500..506127b93 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -36,7 +36,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template := &Template{} - data, err := utils.ReadFromPathOrURL(filePath) + data, err := utils.ReadFromPathOrURL(filePath, options.Catalog) if err != nil { return nil, err } diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index d28ebeb63..d6db6df7e 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -6,7 +6,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger/levels" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -85,7 +85,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco ProjectFile: nil, IssuesClient: nil, Browser: nil, - Catalog: catalog.New(options.TemplatesDirectory), + Catalog: disk.NewCatalog(options.TemplatesDirectory), RateLimiter: ratelimit.New(options.RateLimit), } return executerOpts diff --git a/v2/pkg/utils/utils.go b/v2/pkg/utils/utils.go index f01dd636c..03eff4396 100644 --- a/v2/pkg/utils/utils.go +++ b/v2/pkg/utils/utils.go @@ -5,8 +5,9 @@ import ( "io" "net/http" "net/url" - "os" "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) func IsBlank(value string) bool { @@ -44,7 +45,7 @@ func IsURL(input string) bool { } // ReadFromPathOrURL reads and returns the contents of a file or url. -func ReadFromPathOrURL(templatePath string) (data []byte, err error) { +func ReadFromPathOrURL(templatePath string, catalog catalog.Catalog) (data []byte, err error) { if IsURL(templatePath) { resp, err := http.Get(templatePath) if err != nil { @@ -56,7 +57,7 @@ func ReadFromPathOrURL(templatePath string) (data []byte, err error) { return nil, err } } else { - f, err := os.Open(templatePath) + f, err := catalog.OpenFile(templatePath) if err != nil { return nil, err }