External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 22:05:27 +00:00
Code Issues Packages Projects Releases Wiki Activity
nuclei/pkg/operators/matchers/match_test.go

275 lines
10 KiB
Go
Raw Normal View History

Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
package matchers
import (
"testing"
add unit and integration tests for dsl variable #555
2021-11-30 20:20:43 +05:30
"github.com/Knetic/govaluate"
nuclei v3 : misc updates (#4247) * use parsed options while signing * update project layout to v3 * fix .gitignore * remove example template * misc updates * bump tlsx version * hide template sig warning with env * js: retain value while using log * fix nil pointer derefernce * misc doc update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
2023-10-17 17:44:13 +05:30
"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
"github.com/stretchr/testify/require"
)
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
func TestWordANDCondition(t *testing.T) {
Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}}
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
isMatched, matched := m.MatchWords("a b", nil)
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
require.True(t, isMatched, "Could not match words with valid AND condition")
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
require.Equal(t, m.Words, matched)
Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
isMatched, matched = m.MatchWords("b", nil)
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
require.False(t, isMatched, "Could match words with invalid AND condition")
require.Equal(t, []string{}, matched)
}
func TestRegexANDCondition(t *testing.T) {
struct to enums changes for Matchers (#1246) * struct to enums changes for Matchers
2021-11-19 04:54:09 -06:00
m := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}}
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
err := m.CompileMatchers()
require.Nil(t, err)
isMatched, matched := m.MatchRegex("abc abcd 123")
require.True(t, isMatched, "Could not match regex with valid AND condition")
require.Equal(t, []string{"abc", "abc", "12"}, matched)
isMatched, matched = m.MatchRegex("bc 1")
require.False(t, isMatched, "Could match regex with invalid AND condition")
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
require.Equal(t, []string{}, matched)
Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
}
func TestORCondition(t *testing.T) {
m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}}
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
isMatched, matched := m.MatchWords("a b", nil)
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
require.True(t, isMatched, "Could not match valid word OR condition")
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
require.Equal(t, []string{"a"}, matched)
Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
isMatched, matched = m.MatchWords("b", nil)
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
require.True(t, isMatched, "Could not match valid word OR condition")
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
require.Equal(t, []string{"b"}, matched)
Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
isMatched, matched = m.MatchWords("c", nil)
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
require.False(t, isMatched, "Could match invalid word OR condition")
require.Equal(t, []string{}, matched)
}
func TestRegexOrCondition(t *testing.T) {
struct to enums changes for Matchers (#1246) * struct to enums changes for Matchers
2021-11-19 04:54:09 -06:00
m := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}}
[feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test)
2021-09-30 20:36:39 +03:00
err := m.CompileMatchers()
require.Nil(t, err)
isMatched, matched := m.MatchRegex("ab 123")
require.True(t, isMatched, "Could not match valid regex OR condition")
require.Equal(t, []string{"12"}, matched)
isMatched, matched = m.MatchRegex("bc 1")
require.False(t, isMatched, "Could match invalid regex OR condition")
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
require.Equal(t, []string{}, matched)
Tests + fixed a condition logic bug
2020-04-04 00:32:03 +05:30
}
Added hex encoding support in matchers
2021-02-24 11:23:22 +05:30
func TestHexEncoding(t *testing.T) {
struct to enums changes for Matchers (#1246) * struct to enums changes for Matchers
2021-11-19 04:54:09 -06:00
m := &Matcher{Encoding: "hex", Type: MatcherTypeHolder{MatcherType: WordsMatcher}, Part: "body", Words: []string{"50494e47"}}
Added hex encoding support in matchers
2021-02-24 11:23:22 +05:30
err := m.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
[feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files
2021-09-29 19:43:46 +03:00
isMatched, matched := m.MatchWords("PING", nil)
require.True(t, isMatched, "Could not match valid Hex condition")
require.Equal(t, m.Words, matched)
Added hex encoding support in matchers
2021-02-24 11:23:22 +05:30
}
add unit and integration tests for dsl variable #555
2021-11-30 20:20:43 +05:30
func TestMatcher_MatchDSL(t *testing.T) {
Memory usage optimizations (#2350) * Replaced strings.Replaced with fasttemplate reducing allocations Custom template parsing logic was replaced with fasttemplate package for reducing allocations in the replacer.Replace hotpath leading to allocation reduction which accounted for 30% of total nuclei allocations. $ go test -bench=. -benchmem goos: darwin goarch: arm64 pkg: github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer BenchmarkReplacer-8 837232 1422 ns/op 2112 B/op 31 allocs/op BenchmarkReplacerNew-8 3672765 320.3 ns/op 48 B/op 4 allocs/op * Fixed tests failing * Use pre-compiled map of DSL expressions * Reworked expression parsing logic to reduce memory allocations $ go test -bench=. -benchmem goos: darwin goarch: arm64 pkg: github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions BenchmarkEvaluate-8 31560 37769 ns/op 31731 B/op 265 allocs/op BenchmarkEvaluateNew-8 109144 9621 ns/op 6253 B/op 116 allocs/op
2022-08-23 13:16:41 +05:30
compiled, err := govaluate.NewEvaluableExpressionWithFunctions("contains(body, \"{{VARIABLE}}\")", dsl.HelperFunctions)
add unit and integration tests for dsl variable #555
2021-11-30 20:20:43 +05:30
require.Nil(t, err, "couldn't compile expression")
m := &Matcher{Type: MatcherTypeHolder{MatcherType: DSLMatcher}, dslCompiled: []*govaluate.EvaluableExpression{compiled}}
err = m.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
values := []string{"PING", "pong"}
cache, goroutine and unbounded workers management (#6420) * Enhance matcher compilation with caching for regex and DSL expressions to improve performance. Update template parsing to conditionally retain raw templates based on size constraints. * Implement caching for regex and DSL expressions in extractors and matchers to enhance performance. Introduce a buffer pool in raw requests to reduce memory allocations. Update template cache management for improved efficiency. * feat: improve concurrency to be bound * refactor: replace fmt.Sprintf with fmt.Fprintf for improved performance in header handling * feat: add regex matching tests and benchmarks for performance evaluation * feat: add prefix check in regex extraction to optimize matching process * feat: implement regex caching mechanism to enhance performance in extractors and matchers, along with tests and benchmarks for validation * feat: add unit tests for template execution in the core engine, enhancing test coverage and reliability * feat: enhance error handling in template execution and improve regex caching logic for better performance * Implement caching for regex and DSL expressions in the cache package, replacing previous sync.Map usage. Add unit tests for cache functionality, including eviction by capacity and retrieval of cached items. Update extractors and matchers to utilize the new cache system for improved performance and memory efficiency. * Add tests for SetCapacities in cache package to ensure cache behavior on capacity changes - Implemented TestSetCapacities_NoRebuildOnZero to verify that setting capacities to zero does not clear existing caches. - Added TestSetCapacities_BeforeFirstUse to confirm that initial cache settings are respected and not overridden by subsequent capacity changes. * Refactor matchers and update load test generator to use io package - Removed maxRegexScanBytes constant from match.go. - Replaced ioutil with io package in load_test.go for NopCloser usage. - Restored TestValidate_AllowsInlineMultiline in load_test.go to ensure inline validation functionality. * Add cancellation support in template execution and enhance test coverage - Updated executeTemplateWithTargets to respect context cancellation. - Introduced fakeTargetProvider and slowExecuter for testing. - Added Test_executeTemplateWithTargets_RespectsCancellation to validate cancellation behavior during template execution.
2025-09-15 23:48:02 +05:30
for _, value := range values {
add unit and integration tests for dsl variable #555
2021-11-30 20:20:43 +05:30
isMatched := m.MatchDSL(map[string]interface{}{"body": value, "VARIABLE": value})
require.True(t, isMatched)
}
}
XPath matcher support (#4087) * Added xpath response matching support * Add validation for user-supplied XPath * xpath matcher comment fix * Added XPath matched documentation * minor changes: remove warnings --------- Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
2023-08-26 05:11:51 +12:00
func TestMatcher_MatchXPath_HTML(t *testing.T) {
body := `<!doctype html>
<html>
<head>
<title>Example Domain</title>
<meta charset="utf-8" />
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
</head>
<body>
<div>
<h1>Example Domain</h1>
<p>This domain is for use in illustrative examples in documents. You may use this
domain in literature without prior coordination or asking for permission.</p>
<p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>
`
body2 := `<!doctype html>
<html>
<head>
<title>Example Domain</title>
</head>
<body>
<h1> It's test time! </h1>
</body>
</html>
`
// single match
m := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"/html/body/div/p[2]/a"}}
err := m.CompileMatchers()
require.Nil(t, err)
isMatched := m.MatchXPath(body)
require.True(t, isMatched, "Could not match valid XPath")
isMatched = m.MatchXPath("<h1>aaaaaaaaa")
require.False(t, isMatched, "Could match invalid XPath")
// OR match
m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "or", XPath: []string{"/html/head/title[contains(text(), 'PATRICAAA')]", "/html/body/div/p[2]/a"}}
err = m.CompileMatchers()
require.Nil(t, err)
isMatched = m.MatchXPath(body)
require.True(t, isMatched, "Could not match valid multi-XPath with OR condition")
isMatched = m.MatchXPath(body2)
require.False(t, isMatched, "Could match invalid multi-XPath with OR condition")
// AND match
m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "and", XPath: []string{"/html/head/title[contains(text(), 'Example Domain')]", "/html/body/div/p[2]/a"}}
err = m.CompileMatchers()
require.Nil(t, err)
isMatched = m.MatchXPath(body)
require.True(t, isMatched, "Could not match valid multi-XPath with AND condition")
isMatched = m.MatchXPath(body2)
require.False(t, isMatched, "Could match invalid multi-XPath with AND condition")
// invalid xpath
m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//a[@a==1]"}}
_ = m.CompileMatchers()
isMatched = m.MatchXPath(body)
require.False(t, isMatched, "Invalid xpath did not return false")
}
func TestMatcher_MatchXPath_XML(t *testing.T) {
body := `<?xml version="1.0" encoding="utf-8"?><foo>bar</foo><wibble id="1" /><parent><child>baz</child></parent>`
body2 := `<?xml version="1.0" encoding="utf-8"?><test>bar</test><wibble2 id="1" /><roditelj><dijete>alo</dijete></roditelj>`
// single match
m := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//foo[contains(text(), 'bar')]"}}
err := m.CompileMatchers()
require.Nil(t, err)
isMatched := m.MatchXPath(body)
require.True(t, isMatched, "Could not match valid XPath")
isMatched = m.MatchXPath("<h1>aaaaaaaaa</h1>")
require.False(t, isMatched, "Could match invalid XPath")
// OR match
m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "or", XPath: []string{"/foo[contains(text(), 'PATRICAAA')]", "/parent/child"}}
err = m.CompileMatchers()
require.Nil(t, err)
isMatched = m.MatchXPath(body)
require.True(t, isMatched, "Could not match valid multi-XPath with OR condition")
isMatched = m.MatchXPath(body2)
require.False(t, isMatched, "Could match invalid multi-XPath with OR condition")
// AND match
m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "and", XPath: []string{"/foo[contains(text(), 'bar')]", "/parent/child"}}
err = m.CompileMatchers()
require.Nil(t, err)
isMatched = m.MatchXPath(body)
require.True(t, isMatched, "Could not match valid multi-XPath with AND condition")
isMatched = m.MatchXPath(body2)
require.False(t, isMatched, "Could match invalid multi-XPath with AND condition")
// invalid xpath
m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//a[@a==1]"}}
_ = m.CompileMatchers()
isMatched = m.MatchXPath(body)
require.False(t, isMatched, "Invalid xpath did not return false")
// invalid xml
isMatched = m.MatchXPath("<h1> not right <q id=2/>notvalid")
require.False(t, isMatched, "Invalid xpath did not return false")
}
cache, goroutine and unbounded workers management (#6420) * Enhance matcher compilation with caching for regex and DSL expressions to improve performance. Update template parsing to conditionally retain raw templates based on size constraints. * Implement caching for regex and DSL expressions in extractors and matchers to enhance performance. Introduce a buffer pool in raw requests to reduce memory allocations. Update template cache management for improved efficiency. * feat: improve concurrency to be bound * refactor: replace fmt.Sprintf with fmt.Fprintf for improved performance in header handling * feat: add regex matching tests and benchmarks for performance evaluation * feat: add prefix check in regex extraction to optimize matching process * feat: implement regex caching mechanism to enhance performance in extractors and matchers, along with tests and benchmarks for validation * feat: add unit tests for template execution in the core engine, enhancing test coverage and reliability * feat: enhance error handling in template execution and improve regex caching logic for better performance * Implement caching for regex and DSL expressions in the cache package, replacing previous sync.Map usage. Add unit tests for cache functionality, including eviction by capacity and retrieval of cached items. Update extractors and matchers to utilize the new cache system for improved performance and memory efficiency. * Add tests for SetCapacities in cache package to ensure cache behavior on capacity changes - Implemented TestSetCapacities_NoRebuildOnZero to verify that setting capacities to zero does not clear existing caches. - Added TestSetCapacities_BeforeFirstUse to confirm that initial cache settings are respected and not overridden by subsequent capacity changes. * Refactor matchers and update load test generator to use io package - Removed maxRegexScanBytes constant from match.go. - Replaced ioutil with io package in load_test.go for NopCloser usage. - Restored TestValidate_AllowsInlineMultiline in load_test.go to ensure inline validation functionality. * Add cancellation support in template execution and enhance test coverage - Updated executeTemplateWithTargets to respect context cancellation. - Introduced fakeTargetProvider and slowExecuter for testing. - Added Test_executeTemplateWithTargets_RespectsCancellation to validate cancellation behavior during template execution.
2025-09-15 23:48:02 +05:30
func TestMatchRegex_CaseInsensitivePrefixSkip(t *testing.T) {
m := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "or", Regex: []string{"(?i)abc"}}
err := m.CompileMatchers()
require.NoError(t, err)
ok, got := m.MatchRegex("zzz AbC yyy")
require.True(t, ok)
require.NotEmpty(t, got)
}
func TestMatchStatusCodeAndSize(t *testing.T) {
mStatus := &Matcher{Status: []int{200, 302}}
require.True(t, mStatus.MatchStatusCode(200))
require.True(t, mStatus.MatchStatusCode(302))
require.False(t, mStatus.MatchStatusCode(404))
mSize := &Matcher{Size: []int{5, 10}}
require.True(t, mSize.MatchSize(5))
require.False(t, mSize.MatchSize(7))
}
func TestMatchBinary_AND_OR(t *testing.T) {
// AND should fail if any binary not present
mAnd := &Matcher{Type: MatcherTypeHolder{MatcherType: BinaryMatcher}, Condition: "and", Binary: []string{"50494e47", "414141"}} // "PING", "AAA"
require.NoError(t, mAnd.CompileMatchers())
ok, _ := mAnd.MatchBinary("PING")
require.False(t, ok)
// OR should succeed if any present
mOr := &Matcher{Type: MatcherTypeHolder{MatcherType: BinaryMatcher}, Condition: "or", Binary: []string{"414141", "50494e47"}} // "AAA", "PING"
require.NoError(t, mOr.CompileMatchers())
ok, got := mOr.MatchBinary("xxPINGyy")
require.True(t, ok)
require.NotEmpty(t, got)
}
func TestMatchRegex_LiteralPrefixShortCircuit(t *testing.T) {
// AND: first regex has literal prefix "abc"; corpus lacks it => early false
mAnd := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "and", Regex: []string{"abc[0-9]*", "[0-9]{2}"}}
require.NoError(t, mAnd.CompileMatchers())
ok, matches := mAnd.MatchRegex("zzz 12 yyy")
require.False(t, ok)
require.Empty(t, matches)
// OR: first regex skipped due to missing prefix, second matches => true
mOr := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "or", Regex: []string{"abc[0-9]*", "[0-9]{2}"}}
require.NoError(t, mOr.CompileMatchers())
ok, matches = mOr.MatchRegex("zzz 12 yyy")
require.True(t, ok)
require.Equal(t, []string{"12"}, matches)
}
func TestMatcher_MatchDSL_ErrorHandling(t *testing.T) {
// First expression errors (division by zero), second is true
bad, err := govaluate.NewEvaluableExpression("1 / 0")
require.NoError(t, err)
good, err := govaluate.NewEvaluableExpression("1 == 1")
require.NoError(t, err)
m := &Matcher{Type: MatcherTypeHolder{MatcherType: DSLMatcher}, Condition: "or", dslCompiled: []*govaluate.EvaluableExpression{bad, good}}
require.NoError(t, m.CompileMatchers())
ok := m.MatchDSL(map[string]interface{}{})
require.True(t, ok)
}
Reference in New Issue Copy Permalink