Added initial catalog interface implementation (#2318)

* Added initial catalog interface implementation

* Added OpenFile to Catalog + disk catalog implementation

* Fixed merge issues

Co-authored-by: sandeep <sandeep@projectdiscovery.io>

v2/cmd/cve-annotate/main.go

@@ -13,12 +13,11 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nvd"
 	"github.com/projectdiscovery/sliceutil"
 	"github.com/projectdiscovery/stringsutil"
 	"gopkg.in/yaml.v3"
-
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 )
 
 const (
@@ -61,7 +60,7 @@ func process() error {
 	if err != nil {
 		return err
 	}
-	catalog := catalog.New(*templateDir)
+	catalog := disk.NewCatalog(*templateDir)
 
 	paths, err := catalog.GetTemplatePath(*input)
 	if err != nil {

v2/cmd/integration-test/code.go

@@ -13,8 +13,8 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/goflags"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v2/pkg/core"
 	"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
@@ -89,7 +89,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
 	defer interactClient.Close()
 
 	home, _ := os.UserHomeDir()
-	catalog := catalog.New(path.Join(home, "nuclei-templates"))
+	catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
 	executerOpts := protocols.ExecuterOptions{
 		Output:          outputWriter,
 		Options:         defaultOpts,

v2/go.mod

@@ -74,7 +74,7 @@ require (
 	github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7
 	github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963
 	github.com/projectdiscovery/iputil v0.0.0-20220620153941-036d511e4097
-	github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d
+	github.com/projectdiscovery/nvd v1.0.9
 	github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9
 	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
 	github.com/projectdiscovery/wappalyzergo v0.0.54

v2/go.sum

@@ -559,8 +559,8 @@ github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALateb
 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs=
 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
 github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
-github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d h1:WK/zpWYGuZhAudzVjbnCYk12L4SqH66jagBzHRm3eCo=
+github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS6jzjE=
-github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og=
+github.com/projectdiscovery/nvd v1.0.9/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og=
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/rawhttp v0.0.9-0.20220726060557-a045ab711701 h1:yD0xcwbqfxluOpThWJKBaIFJmsHYsBqVU5n5+6EoRJE=
 github.com/projectdiscovery/rawhttp v0.0.9-0.20220726060557-a045ab711701/go.mod h1:Q5PDAmKzjAjweEp0CQr9301nyxCOkzA9ImK6qLjgk+8=

v2/internal/runner/runner.go

@@ -24,6 +24,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v2/pkg/core"
 	"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid"
@@ -58,7 +59,7 @@ type Runner struct {
 	templatesConfig   *config.Config
 	options           *types.Options
 	projectFile       *projectfile.ProjectFile
-	catalog           *catalog.Catalog
+	catalog           catalog.Catalog
 	progress          progress.Progress
 	colorizer         aurora.Aurora
 	issuesClient      *reporting.Client
@@ -110,7 +111,7 @@ func New(options *types.Options) (*Runner, error) {
 		runner.browser = browser
 	}
 
-	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
+	runner.catalog = disk.NewCatalog(runner.options.TemplatesDirectory)
 
 	var httpclient *retryablehttp.Client
 	if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {

v2/internal/runner/templates.go

@@ -13,7 +13,7 @@ import (
 )
 
 func (r *Runner) logAvailableTemplate(tplPath string) {
-	t, err := parsers.ParseTemplate(tplPath)
+	t, err := parsers.ParseTemplate(tplPath, r.catalog)
 	if err != nil {
 		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
 	} else {

v2/pkg/catalog/catalogue.go

@@ -1,12 +1,22 @@
 package catalog
 
-// Catalog is a template catalog helper implementation
+import "io"
-type Catalog struct {
-	templatesDirectory string
-}
 
-// New creates a new Catalog structure using provided input items
+// Catalog is a catalog storage implementations
-func New(directory string) *Catalog {
+type Catalog interface {
-	catalog := &Catalog{templatesDirectory: directory}
+	// OpenFile opens a file and returns an io.ReadCloser to the file.
-	return catalog
+	// It is used to read template and payload files based on catalog responses.
+	OpenFile(filename string) (io.ReadCloser, error)
+	// GetTemplatePath parses the specified input template path and returns a compiled
+	// list of finished absolute paths to the templates evaluating any glob patterns
+	// or folders provided as in.
+	GetTemplatePath(target string) ([]string, error)
+	// GetTemplatesPath returns a list of absolute paths for the provided template list.
+	GetTemplatesPath(definitions []string) []string
+	// ResolvePath resolves the path to an absolute one in various ways.
+	//
+	// It checks if the filename is an absolute path, looks in the current directory
+	// or checking the nuclei templates directory. If a second path is given,
+	// it also tries to find paths relative to that second path.
+	ResolvePath(templateName, second string) (string, error)
 }

v2/pkg/catalog/disk/catalog.go

@@ -0,0 +1,25 @@
+package disk
+
+import (
+	"io"
+	"os"
+)
+
+// DiskCatalog is a template catalog helper implementation based on disk
+type DiskCatalog struct {
+	templatesDirectory string
+}
+
+// NewCatalog creates a new Catalog structure using provided input items
+// using disk based items
+func NewCatalog(directory string) *DiskCatalog {
+	catalog := &DiskCatalog{templatesDirectory: directory}
+	return catalog
+}
+
+// OpenFile opens a file and returns an io.ReadCloser to the file.
+// It is used to read template and payload files based on catalog responses.
+func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
+	file, err := os.Open(filename)
+	return file, err
+}

v2/pkg/catalog/disk/find.go

@@ -1,4 +1,4 @@
-package catalog
+package disk
 
 import (
 	"io/fs"
@@ -7,12 +7,11 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
-
 	"github.com/projectdiscovery/gologger"
 )
 
 // GetTemplatesPath returns a list of absolute paths for the provided template list.
-func (c *Catalog) GetTemplatesPath(definitions []string) []string {
+func (c *DiskCatalog) GetTemplatesPath(definitions []string) []string {
 	// keeps track of processed dirs and files
 	processed := make(map[string]bool)
 	allTemplates := []string{}
@@ -42,7 +41,7 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string {
 // GetTemplatePath parses the specified input template path and returns a compiled
 // list of finished absolute paths to the templates evaluating any glob patterns
 // or folders provided as in.
-func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
+func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
 	processed := make(map[string]struct{})
 
 	absPath, err := c.convertPathToAbsolute(target)
@@ -88,7 +87,7 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) {
 
 // convertPathToAbsolute resolves the paths provided to absolute paths
 // before doing any operations on them regardless of them being BLOB, folders, files, etc.
-func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
+func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) {
 	if strings.Contains(t, "*") {
 		file := filepath.Base(t)
 		absPath, err := c.ResolvePath(filepath.Dir(t), "")
@@ -101,7 +100,7 @@ func (c *Catalog) convertPathToAbsolute(t string) (string, error) {
 }
 
 // findGlobPathMatches returns the matched files from a glob path
-func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
 	matches, err := filepath.Glob(absPath)
 	if err != nil {
 		return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
@@ -118,7 +117,7 @@ func (c *Catalog) findGlobPathMatches(absPath string, processed map[string]struc
 
 // findFileMatches finds if a path is an absolute file. If the path
 // is a file, it returns true otherwise false with no errors.
-func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
+func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
 	info, err := os.Stat(absPath)
 	if err != nil {
 		return "", false, err
@@ -134,7 +133,7 @@ func (c *Catalog) findFileMatches(absPath string, processed map[string]struct{})
 }
 
 // findDirectoryMatches finds matches for templates from a directory
-func (c *Catalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
+func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
 	var results []string
 	err := filepath.WalkDir(
 		absPath,

v2/pkg/catalog/disk/path.go

@@ -1,4 +1,4 @@
-package catalog
+package disk
 
 import (
 	"fmt"
@@ -15,7 +15,7 @@ import (
 // It checks if the filename is an absolute path, looks in the current directory
 // or checking the nuclei templates directory. If a second path is given,
 // it also tries to find paths relative to that second path.
-func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
+func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) {
 	if filepath.IsAbs(templateName) {
 		return templateName, nil
 	}
@@ -48,7 +48,7 @@ func (c *Catalog) ResolvePath(templateName, second string) (string, error) {
 var errNoValidCombination = errors.New("no valid combination found")
 
 // tryResolve attempts to load locate the target by iterating across all the folders tree
-func (c *Catalog) tryResolve(fullPath string) (string, error) {
+func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
 	dir, filename := filepath.Split(fullPath)
 	pathInfo, err := folderutil.NewPathInfo(dir)
 	if err != nil {

v2/pkg/catalog/loader/filter/path_filter.go

@@ -15,7 +15,7 @@ type PathFilterConfig struct {
 }
 
 // NewPathFilter creates a new path filter from provided config
-func NewPathFilter(config *PathFilterConfig, catalogClient *catalog.Catalog) *PathFilter {
+func NewPathFilter(config *PathFilterConfig, catalogClient catalog.Catalog) *PathFilter {
 	filter := &PathFilter{
 		excludedTemplates:          catalogClient.GetTemplatesPath(config.ExcludedTemplates),
 		alwaysIncludedTemplatesMap: make(map[string]struct{}),

v2/pkg/catalog/loader/loader.go

@@ -39,7 +39,7 @@ type Config struct {
 	IncludeIds        []string
 	ExcludeIds        []string
 
-	Catalog            *catalog.Catalog
+	Catalog            catalog.Catalog
 	ExecutorOptions    protocols.ExecuterOptions
 	TemplatesDirectory string
 }
@@ -59,7 +59,7 @@ type Store struct {
 }
 
 // NewConfig returns a new loader config
-func NewConfig(options *types.Options, templateConfig *config.Config, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
+func NewConfig(options *types.Options, templateConfig *config.Config, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
 	loaderConfig := Config{
 		Templates:                options.Templates,
 		Workflows:                options.Workflows,
@@ -180,13 +180,13 @@ func (store *Store) ValidateTemplates() error {
 
 func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
 	return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, true, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
-		return parsers.LoadWorkflow(templatePath)
+		return parsers.LoadWorkflow(templatePath, store.config.Catalog)
 	})
 }
 
 func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
 	return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, false, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
-		return parsers.LoadTemplate(templatePath, store.tagFilter, nil)
+		return parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
 	})
 }
 
@@ -260,7 +260,7 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template
 
 	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
 	for templatePath := range templatePathMap {
-		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
+		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
 		if loaded || store.pathFilter.MatchIncluded(templatePath) {
 			parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
 			if err != nil {
@@ -283,7 +283,7 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
 
 	loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
 	for workflowPath := range workflowPathMap {
-		loaded, err := parsers.LoadWorkflow(workflowPath)
+		loaded, err := parsers.LoadWorkflow(workflowPath, store.config.Catalog)
 		if err != nil {
 			gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
 		}
@@ -307,7 +307,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 
 	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
 	for templatePath := range templatePathMap {
-		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags)
+		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
 		if loaded || store.pathFilter.MatchIncluded(templatePath) {
 			parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
 			if err != nil {

v2/pkg/catalog/loader/loader_test.go

@@ -4,12 +4,16 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/stretchr/testify/require"
 )
 
 func TestLoadTemplates(t *testing.T) {
+	catalog := disk.NewCatalog("")
+
 	store, err := New(&Config{
 		Templates: []string{"cves/CVE-2021-21315.yaml"},
+		Catalog:   catalog,
 	})
 	require.Nil(t, err, "could not load templates")
 	require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
@@ -18,6 +22,7 @@ func TestLoadTemplates(t *testing.T) {
 	t.Run("blank", func(t *testing.T) {
 		store, err := New(&Config{
 			TemplatesDirectory: templatesDirectory,
+			Catalog:            catalog,
 		})
 		require.Nil(t, err, "could not load templates")
 		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
@@ -26,6 +31,7 @@ func TestLoadTemplates(t *testing.T) {
 		store, err := New(&Config{
 			Tags:               []string{"cves"},
 			TemplatesDirectory: templatesDirectory,
+			Catalog:            catalog,
 		})
 		require.Nil(t, err, "could not load templates")
 		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
@@ -34,6 +40,7 @@ func TestLoadTemplates(t *testing.T) {
 		store, err := New(&Config{
 			Tags:               []string{"cves"},
 			TemplatesDirectory: templatesDirectory,
+			Catalog:            catalog,
 		})
 		require.Nil(t, err, "could not load templates")
 		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
@@ -41,6 +48,8 @@ func TestLoadTemplates(t *testing.T) {
 }
 
 func TestRemoteTemplates(t *testing.T) {
+	catalog := disk.NewCatalog("")
+
 	var nilStringSlice []string
 	type args struct {
 		config *Config
@@ -56,7 +65,8 @@ func TestRemoteTemplates(t *testing.T) {
 			args: args{
 				config: &Config{
 					TemplateURLs:             []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
-					RemoteTemplateDomainList: []string{"localhost","raw.githubusercontent.com"},
+					RemoteTemplateDomainList: []string{"localhost", "raw.githubusercontent.com"},
+					Catalog:                  catalog,
 				},
 			},
 			want: &Store{
@@ -70,6 +80,7 @@ func TestRemoteTemplates(t *testing.T) {
 				config: &Config{
 					TemplateURLs:             []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
 					RemoteTemplateDomainList: []string{"localhost"},
+					Catalog:                  catalog,
 				},
 			},
 			want: &Store{

v2/pkg/parsers/parser.go

@@ -7,6 +7,7 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -22,8 +23,8 @@ const (
 )
 
 // LoadTemplate returns true if the template is valid and matches the filtering criteria.
-func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
+func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string, catalog catalog.Catalog) (bool, error) {
-	template, templateParseError := ParseTemplate(templatePath)
+	template, templateParseError := ParseTemplate(templatePath, catalog)
 	if templateParseError != nil {
 		return false, templateParseError
 	}
@@ -43,8 +44,8 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
 }
 
 // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
-func LoadWorkflow(templatePath string) (bool, error) {
+func LoadWorkflow(templatePath string, catalog catalog.Catalog) (bool, error) {
-	template, templateParseError := ParseTemplate(templatePath)
+	template, templateParseError := ParseTemplate(templatePath, catalog)
 	if templateParseError != nil {
 		return false, templateParseError
 	}
@@ -122,11 +123,11 @@ func init() {
 }
 
 // ParseTemplate parses a template and returns a *templates.Template structure
-func ParseTemplate(templatePath string) (*templates.Template, error) {
+func ParseTemplate(templatePath string, catalog catalog.Catalog) (*templates.Template, error) {
 	if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
 		return value.(*templates.Template), err
 	}
-	data, err := utils.ReadFromPathOrURL(templatePath)
+	data, err := utils.ReadFromPathOrURL(templatePath, catalog)
 	if err != nil {
 		return nil, err
 	}

v2/pkg/parsers/parser_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
@@ -14,6 +15,7 @@ import (
 )
 
 func TestLoadTemplate(t *testing.T) {
+	catalog := disk.NewCatalog("")
 	origTemplatesCache := parsedTemplatesCache
 	defer func() { parsedTemplatesCache = origTemplatesCache }()
 
@@ -56,7 +58,7 @@ func TestLoadTemplate(t *testing.T) {
 			parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr)
 
 			tagFilter := filter.New(&filter.Config{})
-			success, err := LoadTemplate(tc.name, tagFilter, nil)
+			success, err := LoadTemplate(tc.name, tagFilter, nil, catalog)
 			if tc.expectedErr == nil {
 				require.NoError(t, err)
 				require.True(t, success)
@@ -97,7 +99,7 @@ func TestLoadTemplate(t *testing.T) {
 				parsedTemplatesCache.Store(name, template, nil)
 
 				tagFilter := filter.New(&filter.Config{})
-				success, err := LoadTemplate(name, tagFilter, nil)
+				success, err := LoadTemplate(name, tagFilter, nil, catalog)
 				if tc.success {
 					require.NoError(t, err)
 					require.True(t, success)

v2/pkg/parsers/workflow_loader.go

@@ -37,7 +37,7 @@ func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string
 
 	loadedTemplates := make([]string, 0, len(templatePathMap))
 	for templatePath := range templatePathMap {
-		loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
+		loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags, w.options.Catalog)
 		if err != nil {
 			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
 		} else if loaded {
@@ -53,7 +53,7 @@ func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate boo
 
 	loadedTemplates := make([]string, 0, len(templatesPathMap))
 	for templatePath := range templatesPathMap {
-		matched, err := LoadTemplate(templatePath, w.tagFilter, nil)
+		matched, err := LoadTemplate(templatePath, w.tagFilter, nil, w.options.Catalog)
 		if err != nil {
 			gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
 		} else if matched || noValidate {

v2/pkg/protocols/common/generators/generators.go

@@ -11,11 +11,12 @@ import (
 // PayloadGenerator is the generator struct for generating payloads
 type PayloadGenerator struct {
 	Type     AttackType
+	catalog  catalog.Catalog
 	payloads map[string][]string
 }
 
 // New creates a new generator structure for payload generation
-func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*PayloadGenerator, error) {
+func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog) (*PayloadGenerator, error) {
 	if attackType.String() == "" {
 		attackType = BatteringRamAttack
 	}
@@ -36,12 +37,12 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
 		}
 	}
 
-	generator := &PayloadGenerator{}
+	generator := &PayloadGenerator{catalog: catalog}
 	if err := generator.validate(payloadsFinal, templatePath); err != nil {
 		return nil, err
 	}
 
-	compiled, err := loadPayloads(payloadsFinal)
+	compiled, err := generator.loadPayloads(payloadsFinal)
 	if err != nil {
 		return nil, err
 	}

v2/pkg/protocols/common/generators/generators_test.go

@@ -5,13 +5,13 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 )
 
 func TestBatteringRamGenerator(t *testing.T) {
 	usernames := []string{"admin", "password"}
 
-	catalogInstance := catalog.New("")
+	catalogInstance := disk.NewCatalog("")
 	generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance)
 	require.Nil(t, err, "could not create generator")
 
@@ -31,7 +31,7 @@ func TestPitchforkGenerator(t *testing.T) {
 	usernames := []string{"admin", "token"}
 	passwords := []string{"password1", "password2", "password3"}
 
-	catalogInstance := catalog.New("")
+	catalogInstance := disk.NewCatalog("")
 	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance)
 	require.Nil(t, err, "could not create generator")
 
@@ -53,7 +53,7 @@ func TestClusterbombGenerator(t *testing.T) {
 	usernames := []string{"admin"}
 	passwords := []string{"admin", "password", "token"}
 
-	catalogInstance := catalog.New("")
+	catalogInstance := disk.NewCatalog("")
 	generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance)
 	require.Nil(t, err, "could not create generator")
 

v2/pkg/protocols/common/generators/load.go

@@ -3,7 +3,6 @@ package generators
 import (
 	"bufio"
 	"io"
-	"os"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -11,7 +10,7 @@ import (
 )
 
 // loadPayloads loads the input payloads from a map to a data map
-func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) {
+func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}) (map[string][]string, error) {
 	loadedPayloads := make(map[string][]string)
 
 	for name, payload := range payloads {
@@ -22,7 +21,7 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error)
 			if len(elements) >= 2 {
 				loadedPayloads[name] = elements
 			} else {
-				payloads, err := loadPayloadsFromFile(pt)
+				payloads, err := generator.loadPayloadsFromFile(pt)
 				if err != nil {
 					return nil, errors.Wrap(err, "could not load payloads")
 				}
@@ -36,10 +35,10 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error)
 }
 
 // loadPayloadsFromFile loads a file to a string slice
-func loadPayloadsFromFile(filepath string) ([]string, error) {
+func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]string, error) {
 	var lines []string
 
-	file, err := os.Open(filepath)
+	file, err := generator.catalog.OpenFile(filepath)
 	if err != nil {
 		return nil, err
 	}

v2/pkg/protocols/http/request_generator_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 )
 
@@ -33,7 +33,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) {
 		AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
 		Raw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`},
 	}
-	catalogInstance := catalog.New("")
+	catalogInstance := disk.NewCatalog("")
 	req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance)
 	require.Nil(t, err, "could not create generator")
 
@@ -57,7 +57,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) {
 		AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
 		Raw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},
 	}
-	catalogInstance := catalog.New("")
+	catalogInstance := disk.NewCatalog("")
 	req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance)
 	require.Nil(t, err, "could not create generator")
 

v2/pkg/protocols/protocols.go

@@ -54,7 +54,7 @@ type ExecuterOptions struct {
 	// RateLimiter is a rate-limiter for limiting sent number of requests.
 	RateLimiter ratelimit.Limiter
 	// Catalog is a template catalog implementation for nuclei
-	Catalog *catalog.Catalog
+	Catalog catalog.Catalog
 	// ProjectFile is the project file for nuclei
 	ProjectFile *projectfile.ProjectFile
 	// Browser is a browser engine for running headless templates

v2/pkg/templates/compile.go

@@ -36,7 +36,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 
 	template := &Template{}
 
-	data, err := utils.ReadFromPathOrURL(filePath)
+	data, err := utils.ReadFromPathOrURL(filePath, options.Catalog)
 	if err != nil {
 		return nil, err
 	}

v2/pkg/testutils/testutils.go

@@ -6,7 +6,7 @@ import (
 	"github.com/logrusorgru/aurora"
 
 	"github.com/projectdiscovery/gologger/levels"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -85,7 +85,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
 		ProjectFile:  nil,
 		IssuesClient: nil,
 		Browser:      nil,
-		Catalog:      catalog.New(options.TemplatesDirectory),
+		Catalog:      disk.NewCatalog(options.TemplatesDirectory),
 		RateLimiter:  ratelimit.New(options.RateLimit),
 	}
 	return executerOpts

v2/pkg/utils/utils.go

@@ -5,8 +5,9 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 )
 
 func IsBlank(value string) bool {
@@ -44,7 +45,7 @@ func IsURL(input string) bool {
 }
 
 // ReadFromPathOrURL reads and returns the contents of a file or url.
-func ReadFromPathOrURL(templatePath string) (data []byte, err error) {
+func ReadFromPathOrURL(templatePath string, catalog catalog.Catalog) (data []byte, err error) {
 	if IsURL(templatePath) {
 		resp, err := http.Get(templatePath)
 		if err != nil {
@@ -56,7 +57,7 @@ func ReadFromPathOrURL(templatePath string) (data []byte, err error) {
 			return nil, err
 		}
 	} else {
-		f, err := os.Open(templatePath)
+		f, err := catalog.OpenFile(templatePath)
 		if err != nil {
 			return nil, err
 		}