mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-22 16:25:26 +00:00
8f313629b8
* Replaced strings.Replaced with fasttemplate reducing allocations Custom template parsing logic was replaced with fasttemplate package for reducing allocations in the replacer.Replace hotpath leading to allocation reduction which accounted for 30% of total nuclei allocations. $ go test -bench=. -benchmem goos: darwin goarch: arm64 pkg: github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer BenchmarkReplacer-8 837232 1422 ns/op 2112 B/op 31 allocs/op BenchmarkReplacerNew-8 3672765 320.3 ns/op 48 B/op 4 allocs/op * Fixed tests failing * Use pre-compiled map of DSL expressions * Reworked expression parsing logic to reduce memory allocations $ go test -bench=. -benchmem goos: darwin goarch: arm64 pkg: github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions BenchmarkEvaluate-8 31560 37769 ns/op 31731 B/op 265 allocs/op BenchmarkEvaluateNew-8 109144 9621 ns/op 6253 B/op 116 allocs/op
64 lines
1.6 KiB
Go
64 lines
1.6 KiB
Go
package extractors
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/Knetic/govaluate"
|
|
"github.com/itchyny/gojq"
|
|
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
|
|
)
|
|
|
|
// CompileExtractors performs the initial setup operation on an extractor
|
|
func (e *Extractor) CompileExtractors() error {
|
|
// Set up the extractor type
|
|
computedType, err := toExtractorTypes(e.GetType().String())
|
|
if err != nil {
|
|
return fmt.Errorf("unknown extractor type specified: %s", e.Type)
|
|
}
|
|
e.extractorType = computedType
|
|
// Compile the regexes
|
|
for _, regex := range e.Regex {
|
|
compiled, err := regexp.Compile(regex)
|
|
if err != nil {
|
|
return fmt.Errorf("could not compile regex: %s", regex)
|
|
}
|
|
e.regexCompiled = append(e.regexCompiled, compiled)
|
|
}
|
|
for i, kval := range e.KVal {
|
|
e.KVal[i] = strings.ToLower(kval)
|
|
}
|
|
|
|
for _, query := range e.JSON {
|
|
query, err := gojq.Parse(query)
|
|
if err != nil {
|
|
return fmt.Errorf("could not parse json: %s", query)
|
|
}
|
|
compiled, err := gojq.Compile(query)
|
|
if err != nil {
|
|
return fmt.Errorf("could not compile json: %s", query)
|
|
}
|
|
e.jsonCompiled = append(e.jsonCompiled, compiled)
|
|
}
|
|
|
|
for _, dslExp := range e.DSL {
|
|
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(dslExp, dsl.HelperFunctions)
|
|
if err != nil {
|
|
return fmt.Errorf("could not compile dsl: %s", dslExp)
|
|
}
|
|
e.dslCompiled = append(e.dslCompiled, compiled)
|
|
}
|
|
|
|
if e.CaseInsensitive {
|
|
if e.GetType() != KValExtractor {
|
|
return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type)
|
|
}
|
|
for i := range e.KVal {
|
|
e.KVal[i] = strings.ToLower(e.KVal[i])
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|