mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-17 19:45:28 +00:00
c4fa2c74c1
* Enhance matcher compilation with caching for regex and DSL expressions to improve performance. Update template parsing to conditionally retain raw templates based on size constraints. * Implement caching for regex and DSL expressions in extractors and matchers to enhance performance. Introduce a buffer pool in raw requests to reduce memory allocations. Update template cache management for improved efficiency. * feat: improve concurrency to be bound * refactor: replace fmt.Sprintf with fmt.Fprintf for improved performance in header handling * feat: add regex matching tests and benchmarks for performance evaluation * feat: add prefix check in regex extraction to optimize matching process * feat: implement regex caching mechanism to enhance performance in extractors and matchers, along with tests and benchmarks for validation * feat: add unit tests for template execution in the core engine, enhancing test coverage and reliability * feat: enhance error handling in template execution and improve regex caching logic for better performance * Implement caching for regex and DSL expressions in the cache package, replacing previous sync.Map usage. Add unit tests for cache functionality, including eviction by capacity and retrieval of cached items. Update extractors and matchers to utilize the new cache system for improved performance and memory efficiency. * Add tests for SetCapacities in cache package to ensure cache behavior on capacity changes - Implemented TestSetCapacities_NoRebuildOnZero to verify that setting capacities to zero does not clear existing caches. - Added TestSetCapacities_BeforeFirstUse to confirm that initial cache settings are respected and not overridden by subsequent capacity changes. * Refactor matchers and update load test generator to use io package - Removed maxRegexScanBytes constant from match.go. - Replaced ioutil with io package in load_test.go for NopCloser usage. - Restored TestValidate_AllowsInlineMultiline in load_test.go to ensure inline validation functionality. * Add cancellation support in template execution and enhance test coverage - Updated executeTemplateWithTargets to respect context cancellation. - Introduced fakeTargetProvider and slowExecuter for testing. - Added Test_executeTemplateWithTargets_RespectsCancellation to validate cancellation behavior during template execution.
132 lines
6.3 KiB
Go
132 lines
6.3 KiB
Go
package raw
|
|
|
|
import (
|
|
"testing"
|
|
|
|
urlutil "github.com/projectdiscovery/utils/url"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestTryFillCustomHeaders_BufferDetached(t *testing.T) {
|
|
r := &Request{
|
|
UnsafeRawBytes: []byte("GET / HTTP/1.1\r\nHost: example.com\r\n\r\nBody"),
|
|
}
|
|
// first fill
|
|
err := r.TryFillCustomHeaders([]string{"X-Test: 1"})
|
|
require.NoError(t, err, "unexpected error on first call")
|
|
prev := r.UnsafeRawBytes
|
|
prevStr := string(prev) // content snapshot
|
|
err = r.TryFillCustomHeaders([]string{"X-Another: 2"})
|
|
require.NoError(t, err, "unexpected error on second call")
|
|
require.Equal(t, prevStr, string(prev), "first slice mutated after second call; buffer not detached")
|
|
require.NotEqual(t, prevStr, string(r.UnsafeRawBytes), "request bytes did not change after second call")
|
|
}
|
|
|
|
func TestParseRawRequestWithPort(t *testing.T) {
|
|
request, err := Parse(`GET /gg/phpinfo.php HTTP/1.1
|
|
Host: {{Hostname}}:123
|
|
Origin: {{BaseURL}}
|
|
Connection: close
|
|
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
|
|
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
|
|
Accept-Language: en-US,en;q=0.9`, parseURL(t, "https://example.com:8080"), false, false)
|
|
require.Nil(t, err, "could not parse GET request")
|
|
require.Equal(t, "https://example.com:8080/gg/phpinfo.php", request.FullURL, "Could not parse request url correctly")
|
|
require.Equal(t, "/gg/phpinfo.php", request.Path, "Could not parse request path correctly")
|
|
|
|
t.Run("path-suffix", func(t *testing.T) {
|
|
request, err := Parse(`GET /hello HTTP/1.1
|
|
Host: {{Hostname}}`, parseURL(t, "https://example.com:8080/test"), false, false)
|
|
require.Nil(t, err, "could not parse GET request")
|
|
require.Equal(t, "https://example.com:8080/test/hello", request.FullURL, "Could not parse request url correctly")
|
|
})
|
|
|
|
t.Run("query-values", func(t *testing.T) {
|
|
request, err := Parse(`GET ?username=test&password=test HTTP/1.1
|
|
Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test"), false, false)
|
|
require.Nil(t, err, "could not parse GET request")
|
|
// url.values are sorted to avoid randomness of using maps
|
|
require.Equal(t, "https://example.com:8080/test?username=test&password=test", request.FullURL, "Could not parse request url correctly")
|
|
|
|
request, err = Parse(`GET ?username=test&password=test HTTP/1.1
|
|
Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false)
|
|
require.Nil(t, err, "could not parse GET request")
|
|
require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly")
|
|
|
|
request, err = Parse(`GET /?username=test&password=test HTTP/1.1
|
|
Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false)
|
|
require.Nil(t, err, "could not parse GET request")
|
|
require.Equal(t, "https://example.com:8080/test/?username=test&password=test", request.FullURL, "Could not parse request url correctly")
|
|
})
|
|
}
|
|
|
|
func TestParseRawRequest(t *testing.T) {
|
|
request, err := Parse(`GET /manager/html HTTP/1.1
|
|
Host: {{Hostname}}
|
|
Authorization: Basic {{base64('username:password')}}
|
|
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0
|
|
Accept-Language: en-US,en;q=0.9
|
|
Connection: close`, parseURL(t, "https://test.com"), false, false)
|
|
require.Nil(t, err, "could not parse GET request")
|
|
require.Equal(t, "GET", request.Method, "Could not parse GET method request correctly")
|
|
require.Equal(t, "/manager/html", request.Path, "Could not parse request path correctly")
|
|
|
|
request, err = Parse(`POST /login HTTP/1.1
|
|
Host: {{Hostname}}
|
|
Content-Type: application/x-www-form-urlencoded
|
|
Connection: close
|
|
|
|
username=admin&password=login`, parseURL(t, "https://test.com"), false, false)
|
|
require.Nil(t, err, "could not parse POST request")
|
|
require.Equal(t, "POST", request.Method, "Could not parse POST method request correctly")
|
|
require.Equal(t, "username=admin&password=login", request.Data, "Could not parse request data correctly")
|
|
}
|
|
|
|
func TestParseUnsafeRequestWithPath(t *testing.T) {
|
|
request, err := Parse(`GET /manager/html HTTP/1.1
|
|
Host: {{Hostname}}
|
|
Authorization: Basic {{base64('username:password')}}
|
|
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0
|
|
Accept-Language: en-US,en;q=0.9
|
|
Connection: close`, parseURL(t, "https://test.com/test/"), true, false)
|
|
require.Nil(t, err, "could not parse unsafe request")
|
|
require.Contains(t, string(request.UnsafeRawBytes), "GET /test/manager/html", "Could not parse unsafe method request path correctly")
|
|
|
|
request, err = Parse(`GET ?a=b HTTP/1.1
|
|
Host: {{Hostname}}
|
|
Origin: {{BaseURL}}`, parseURL(t, "https://test.com/test.js"), true, false)
|
|
require.Nil(t, err, "could not parse unsafe request")
|
|
require.Contains(t, string(request.UnsafeRawBytes), "GET /test.js?a=b", "Could not parse unsafe method request path correctly")
|
|
}
|
|
|
|
func TestTryFillCustomHeaders(t *testing.T) {
|
|
testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n"
|
|
expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n"
|
|
request, err := Parse(testValue, parseURL(t, "https://test.com/test/"), true, false)
|
|
require.Nil(t, err, "could not parse unsafe request")
|
|
err = request.TryFillCustomHeaders([]string{"test: test"})
|
|
require.Nil(t, err, "could not add custom headers")
|
|
require.Equal(t, expected, string(request.UnsafeRawBytes), "actual value and expected value are different")
|
|
}
|
|
|
|
func TestDisableMergePath(t *testing.T) {
|
|
request, err := Parse(` GET /api/v1/id=123 HTTP/1.1
|
|
Host: {{Hostname}}`, parseURL(t, "https://example.com/api/v1/user"), false, true)
|
|
require.Nil(t, err, "could not parse GET request with disable merge path")
|
|
require.Equal(t, "https://example.com/api/v1/id=123", request.FullURL, "Could not parse request url with disable merge path correctly")
|
|
|
|
request, err = Parse(` GET /api/v1/id=123 HTTP/1.1
|
|
Host: {{Hostname}}`, parseURL(t, "https://example.com/api/v1/user"), false, false)
|
|
require.Nil(t, err, "could not parse GET request with merge path")
|
|
require.Equal(t, "https://example.com/api/v1/user/api/v1/id=123", request.FullURL, "Could not parse request url with merge path correctly")
|
|
|
|
}
|
|
|
|
func parseURL(t *testing.T, inputurl string) *urlutil.URL {
|
|
urlx, err := urlutil.Parse(inputurl)
|
|
if err != nil {
|
|
t.Fatalf("failed to parse url %v", urlx)
|
|
}
|
|
return urlx
|
|
}
|