From 2334195bacbf33f6d106824d50ad02bea840f64f Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 2 Mar 2022 08:31:23 +0100 Subject: [PATCH] adding deduping in matches + extract --- v2/go.mod | 1 + v2/go.sum | 2 ++ v2/pkg/operators/operators.go | 5 +++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index f51d9d347..8f5e9be13 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -73,6 +73,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 + github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d github.com/stretchr/testify v1.7.0 github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b ) diff --git a/v2/go.sum b/v2/go.sum index c56547a37..6ba455592 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -476,6 +476,8 @@ github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660/g github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= +github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d h1:wIQPYRZEwTeJuoZLv3NT9r+il2fAv1ObRzTdHkNgOxk= +github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 9f9439daa..4cc4e6f99 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/sliceutil" ) // Operators contains the operators that can be applied on protocols @@ -145,10 +146,10 @@ func (r *Result) Merge(result *Result) { } for k, v := range result.Matches { - r.Matches[k] = append(r.Matches[k], v...) + r.Matches[k] = sliceutil.Dedupe(append(r.Matches[k], v...)) } for k, v := range result.Extracts { - r.Extracts[k] = append(r.Extracts[k], v...) + r.Extracts[k] = sliceutil.Dedupe(append(r.Extracts[k], v...)) } r.outputUnique = make(map[string]struct{})