Purge cache on global callback set (#4840)

* purge cache on global callback set

* lint

* purging cache

* purge cache in runner after loading templates

* include internal cache from parsers + add global cache register/purge via config

* remove disable cache purge option

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

internal/runner/runner.go

@@ -470,6 +470,9 @@ func (r *Runner) RunEnumeration() error {
 	disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent)
 	templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
 
+	// purge global caches primarily used for loading templates
+	config.DefaultConfig.PurgeGlobalCache()
+
 	// add the hosts from the metadata queries of loaded templates into input provider
 	if r.options.Uncover && len(r.options.UncoverQuery) == 0 {
 		uncoverOpts := &uncoverlib.Options{

lib/multi.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core/inputs"
@@ -88,6 +89,7 @@ func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error {
 // GlobalResultCallback sets a callback function which will be called for each result
 func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *output.ResultEvent)) {
 	e.eng.resultCallbacks = []func(*output.ResultEvent){callback}
+	config.DefaultConfig.PurgeGlobalCache()
 }
 
 // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)

pkg/catalog/config/nucleiconfig.go

@@ -45,6 +45,9 @@ type Config struct {
 	LatestNucleiTemplatesVersion string `json:"nuclei-templates-latest-version"`
 	LatestNucleiIgnoreHash       string `json:"nuclei-latest-ignore-hash,omitempty"`
 
+	// Other AppLevel/Global Settings
+	registerdCaches []GlobalCache `json:"-"` // registered global caches
+
 	// internal / unexported fields
 	disableUpdates bool   `json:"-"` // disable updates both version check and template updates
 	homeDir        string `json:"-"` //  User Home Directory
@@ -298,6 +301,19 @@ func (c *Config) WriteTemplatesIndex(index map[string]string) error {
 	return os.WriteFile(indexFile, buff.Bytes(), 0600)
 }
 
+// RegisterGlobalCache registers a global cache at app level
+// and is available to be purged on demand
+func (c *Config) RegisterGlobalCache(cache GlobalCache) {
+	c.registerdCaches = append(c.registerdCaches, cache)
+}
+
+// PurgeGlobalCache purges all registered global caches
+func (c *Config) PurgeGlobalCache() {
+	for _, cache := range c.registerdCaches {
+		cache.Purge()
+	}
+}
+
 // getTemplatesConfigFilePath returns configDir/.templates-config.json file path
 func (c *Config) getTemplatesConfigFilePath() string {
 	return filepath.Join(c.configDir, TemplateConfigFileName)

pkg/catalog/config/template.go

@@ -13,6 +13,13 @@ import (
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
+// GlobalCache are global cache that have global
+// scope and are not purged but can be purged
+// via config.DefaultConfig
+type GlobalCache interface {
+	Purge()
+}
+
 var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"}
 
 // TemplateFormat

pkg/parsers/parser.go

@@ -152,6 +152,7 @@ const (
 
 func init() {
 	parsedTemplatesCache = cache.New()
+	config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache)
 
 	stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)")
 	stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)")

pkg/templates/cache/cache.go

@@ -1,17 +1,17 @@
 package cache
 
 import (
-	"sync"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 )
 
 // Templates is a cache for caching and storing templates for reuse.
 type Templates struct {
-	items *sync.Map
+	items *mapsutil.SyncLockMap[string, parsedTemplateErrHolder]
 }
 
 // New returns a new templates cache
 func New() *Templates {
-	return &Templates{items: &sync.Map{}}
+	return &Templates{items: mapsutil.NewSyncLockMap[string, parsedTemplateErrHolder]()}
 }
 
 type parsedTemplateErrHolder struct {
@@ -22,18 +22,19 @@ type parsedTemplateErrHolder struct {
 // Has returns true if the cache has a template. The template
 // is returned along with any errors if found.
 func (t *Templates) Has(template string) (interface{}, error) {
-	value, ok := t.items.Load(template)
-	if !ok || value == nil {
-		return nil, nil
-	}
-	templateError, ok := value.(parsedTemplateErrHolder)
+	value, ok := t.items.Get(template)
 	if !ok {
 		return nil, nil
 	}
-	return templateError.template, templateError.err
+	return value.template, value.err
 }
 
 // Store stores a template with data and error
 func (t *Templates) Store(template string, data interface{}, err error) {
-	t.items.Store(template, parsedTemplateErrHolder{template: data, err: err})
+	_ = t.items.Set(template, parsedTemplateErrHolder{template: data, err: err})
+}
+
+// Purge the cache
+func (t *Templates) Purge() {
+	t.items.Clear()
 }

pkg/templates/compile.go

@@ -45,6 +45,7 @@ func init() {
 		SignatureStats[verifier.Identifier()] = &atomic.Uint64{}
 	}
 	SignatureStats[Unsigned] = &atomic.Uint64{}
+	config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache)
 }
 
 // Parse parses a yaml request template file