From 4916cf34f0083e0dc874ff969771ce15ced4acef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 10 Sep 2025 16:44:12 +0300 Subject: [PATCH 1/3] dont load templates with the same ID --- pkg/catalog/loader/loader.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index cfaf4e832..966486b22 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -7,6 +7,7 @@ import ( "os" "sort" "strings" + "sync" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -315,6 +316,8 @@ func (store *Store) LoadTemplatesOnlyMetadata() error { } templatesCache := parserItem.Cache() + loadedTemplateIDs := make(map[string]bool) + for templatePath := range validPaths { template, _, _ := templatesCache.Has(templatePath) @@ -339,6 +342,12 @@ func (store *Store) LoadTemplatesOnlyMetadata() error { } if template != nil { + if loadedTemplateIDs[template.ID] { + store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", template.ID, templatePath) + continue + } + + loadedTemplateIDs[template.ID] = true template.Path = templatePath store.templates = append(store.templates, template) } @@ -492,8 +501,20 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ templatePathMap := store.pathFilter.Match(includedTemplates) loadedTemplates := sliceutil.NewSyncSlice[*templates.Template]() + loadedTemplateIDs := make(map[string]bool) + var loadedTemplateIDsMutex sync.Mutex loadTemplate := func(tmpl *templates.Template) { + loadedTemplateIDsMutex.Lock() + if loadedTemplateIDs[tmpl.ID] { + loadedTemplateIDsMutex.Unlock() + store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", tmpl.ID, tmpl.Path) + return + } + + loadedTemplateIDs[tmpl.ID] = true + loadedTemplateIDsMutex.Unlock() + loadedTemplates.Append(tmpl) // increment signed/unsigned counters if tmpl.Verified { From b05359bc82a96d22989e7a69d4439ed93c145e51 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 10 Sep 2025 19:48:36 +0200 Subject: [PATCH 2/3] using synclockmap --- pkg/catalog/loader/loader.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 966486b22..79105bfbf 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -7,7 +7,6 @@ import ( "os" "sort" "strings" - "sync" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -26,6 +25,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/workflows" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/utils/errkit" + mapsutil "github.com/projectdiscovery/utils/maps" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" syncutil "github.com/projectdiscovery/utils/sync" @@ -316,7 +316,7 @@ func (store *Store) LoadTemplatesOnlyMetadata() error { } templatesCache := parserItem.Cache() - loadedTemplateIDs := make(map[string]bool) + loadedTemplateIDs := mapsutil.NewSyncLockMap[string, struct{}]() for templatePath := range validPaths { template, _, _ := templatesCache.Has(templatePath) @@ -342,12 +342,12 @@ func (store *Store) LoadTemplatesOnlyMetadata() error { } if template != nil { - if loadedTemplateIDs[template.ID] { + if loadedTemplateIDs.Has(template.ID) { store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", template.ID, templatePath) continue } - loadedTemplateIDs[template.ID] = true + loadedTemplateIDs.Set(template.ID, struct{}{}) template.Path = templatePath store.templates = append(store.templates, template) } @@ -501,19 +501,15 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ templatePathMap := store.pathFilter.Match(includedTemplates) loadedTemplates := sliceutil.NewSyncSlice[*templates.Template]() - loadedTemplateIDs := make(map[string]bool) - var loadedTemplateIDsMutex sync.Mutex + loadedTemplateIDs := mapsutil.NewSyncLockMap[string, struct{}]() loadTemplate := func(tmpl *templates.Template) { - loadedTemplateIDsMutex.Lock() - if loadedTemplateIDs[tmpl.ID] { - loadedTemplateIDsMutex.Unlock() + if loadedTemplateIDs.Has(tmpl.ID) { store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", tmpl.ID, tmpl.Path) return } - loadedTemplateIDs[tmpl.ID] = true - loadedTemplateIDsMutex.Unlock() + loadedTemplateIDs.Set(tmpl.ID, struct{}{}) loadedTemplates.Append(tmpl) // increment signed/unsigned counters From 608159bbbe92e2c74977180c34893c511721760c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 10 Sep 2025 19:53:23 +0200 Subject: [PATCH 3/3] lint --- pkg/catalog/loader/loader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 79105bfbf..40fbaa482 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -347,7 +347,7 @@ func (store *Store) LoadTemplatesOnlyMetadata() error { continue } - loadedTemplateIDs.Set(template.ID, struct{}{}) + _ = loadedTemplateIDs.Set(template.ID, struct{}{}) template.Path = templatePath store.templates = append(store.templates, template) } @@ -509,7 +509,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ return } - loadedTemplateIDs.Set(tmpl.ID, struct{}{}) + _ = loadedTemplateIDs.Set(tmpl.ID, struct{}{}) loadedTemplates.Append(tmpl) // increment signed/unsigned counters