nuclei/pkg/protocols/http/request_test.go

529 lines
18 KiB
Go

package http
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/tarunKoyalwar/goleak"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
func TestHTTPExtractMultipleReuse(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Raw: []string{
`GET /robots.txt HTTP/1.1
Host: {{Hostname}}
User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
`,
`GET {{endpoint}} HTTP/1.1
Host: {{Hostname}}
User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
`,
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match /a", "match /b", "match /c"},
}},
Extractors: []*extractors.Extractor{{
Part: "body",
Name: "endpoint",
Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"(?m)/([a-zA-Z0-9-_/\\\\]+)"},
Internal: true,
}},
},
IterateAll: true,
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/robots.txt":
_, _ = fmt.Fprintf(w, `User-agent: Googlebot
Disallow: /a
Disallow: /b
Disallow: /c`)
default:
_, _ = fmt.Fprintf(w, `match %v`, r.URL.Path)
}
}))
defer ts.Close()
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
var finalEvent *output.InternalWrappedEvent
var matchCount int
t.Run("test", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matchCount++
}
finalEvent = event
})
require.Nil(t, err, "could not execute network request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 3, matchCount, "could not get correct match count")
}
func TestDisableTE(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "http-disable-transfer-encoding"
// in raw request format
request := &Request{
ID: templateID,
Raw: []string{
`POST / HTTP/1.1
Host: {{Hostname}}
User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
login=1&username=admin&password=admin
`,
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},
Status: []int{200},
}},
},
}
// in base request format
request2 := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Path: []string{"{{BaseURL}}"},
Body: "login=1&username=admin&password=admin",
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},
Status: []int{200},
}},
},
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if len(r.TransferEncoding) > 0 || r.ContentLength <= 0 {
t.Error("Transfer-Encoding header should not be set")
}
}))
defer ts.Close()
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http raw request")
err = request2.Compile(executerOpts)
require.Nil(t, err, "could not compile http base request")
var finalEvent *output.InternalWrappedEvent
var matchCount int
t.Run("test", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matchCount++
}
finalEvent = event
})
require.Nil(t, err, "could not execute network request")
})
t.Run("test2", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matchCount++
}
finalEvent = event
})
require.Nil(t, err, "could not execute network request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 2, matchCount, "could not get correct match count")
}
// consult @Ice3man543 before making any breaking changes to this test (context: vuln_hash)
func TestReqURLPattern(t *testing.T) {
options := testutils.DefaultOptions
// assume this was a preprocessor
// {{randstr}} => 2eNU2kbrOcUDzhnUL1RGvSo1it7
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Raw: []string{
`GET /{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data=2eNU2kbrOcUDzhnUL1RGvSo1it7 HTTP/1.1
Host: {{Hostname}}
User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
`,
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{"true"},
}},
},
IterateAll: true,
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// always return 200
w.WriteHeader(200)
_, _ = w.Write([]byte(`match`))
}))
defer ts.Close()
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
client, _ := interactsh.New(interactsh.DefaultOptions(executerOpts.Output, nil, executerOpts.Progress))
executerOpts.Interactsh = client
defer client.Close()
executerOpts.ExportReqURLPattern = true
// this is how generated constants are added to template
// generated constants are preprocessors that are executed while loading once
executerOpts.Constants = map[string]interface{}{
"{{randstr}}": "2eNU2kbrOcUDzhnUL1RGvSo1it7",
}
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
var finalEvent *output.InternalWrappedEvent
var matchCount int
t.Run("test", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matchCount++
}
finalEvent = event
})
require.Nil(t, err, "could not execute network request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 1, matchCount, "could not get correct match count")
require.NotEmpty(t, finalEvent.Results[0].ReqURLPattern, "could not get req url pattern")
require.Equal(t, `/{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data={{randstr}}`, finalEvent.Results[0].ReqURLPattern)
}
// fakeHostErrorsCache implements hosterrorscache.CacheInterface minimally for tests
type fakeHostErrorsCache struct{}
func (f *fakeHostErrorsCache) SetVerbose(bool) {}
func (f *fakeHostErrorsCache) Close() {}
func (f *fakeHostErrorsCache) Remove(*contextargs.Context) {}
func (f *fakeHostErrorsCache) MarkFailed(string, *contextargs.Context, error) {}
func (f *fakeHostErrorsCache) MarkFailedOrRemove(string, *contextargs.Context, error) {
}
// Check always returns true to simulate an already unresponsive host
func (f *fakeHostErrorsCache) Check(string, *contextargs.Context) bool { return true }
func TestExecuteParallelHTTP_StopAtFirstMatch(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
// server that always matches
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "match")
}))
defer ts.Close()
templateID := "parallel-stop-first"
req := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/p?x={{v}}"},
Threads: 2,
Payloads: map[string]interface{}{
"v": []string{"1", "2"},
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match"},
}},
},
StopAtFirstMatch: true,
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
err := req.Compile(executerOpts)
require.NoError(t, err)
var matches int32
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err = req.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
atomic.AddInt32(&matches, 1)
}
})
require.NoError(t, err)
require.Equal(t, int32(1), atomic.LoadInt32(&matches), "expected only first match to be processed")
}
func TestExecuteParallelHTTP_SkipOnUnresponsiveFromCache(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
// server that would match if reached
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "match")
}))
defer ts.Close()
templateID := "parallel-skip-unresponsive"
req := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/p?x={{v}}"},
Threads: 2,
Payloads: map[string]interface{}{
"v": []string{"1", "2"},
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
// inject fake host errors cache that forces skip
executerOpts.HostErrorsCache = &fakeHostErrorsCache{}
err := req.Compile(executerOpts)
require.NoError(t, err)
var matches int32
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err = req.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
atomic.AddInt32(&matches, 1)
}
})
require.NoError(t, err)
require.Equal(t, int32(0), atomic.LoadInt32(&matches), "expected no matches when host is marked unresponsive")
}
// TestExecuteParallelHTTP_GoroutineLeaks uses goleak to detect goroutine leaks in all HTTP parallel execution scenarios
func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
defer goleak.VerifyNone(t,
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/server"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/interactsh/pkg/client"),
goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"),
goleak.IgnoreAnyFunction("github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain"),
goleak.IgnoreAnyFunction("github.com/syndtr/goleveldb/leveldb.(*DB).compactionError"),
goleak.IgnoreAnyFunction("github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain"),
goleak.IgnoreAnyFunction("github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction"),
goleak.IgnoreAnyFunction("github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction"),
)
options := testutils.DefaultOptions
testutils.Init(options)
defer testutils.Cleanup(options)
// Test Case 1: Normal execution with StopAtFirstMatch
t.Run("StopAtFirstMatch", func(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(10 * time.Millisecond)
_, _ = fmt.Fprintf(w, "test response")
}))
defer ts.Close()
req := &Request{
ID: "parallel-stop-first-match",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/test?param={{payload}}"},
Threads: 4,
Payloads: map[string]interface{}{
"payload": []string{"1", "2", "3", "4", "5", "6", "7", "8"},
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"test response"},
}},
},
StopAtFirstMatch: true,
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "parallel-stop-first-match",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
err := req.Compile(executerOpts)
require.NoError(t, err)
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err = req.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {})
require.NoError(t, err)
})
// Test Case 2: Unresponsive host scenario
t.Run("UnresponsiveHost", func(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "response")
}))
defer ts.Close()
req := &Request{
ID: "parallel-unresponsive",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/test?param={{payload}}"},
Threads: 3,
Payloads: map[string]interface{}{
"payload": []string{"1", "2", "3", "4", "5"},
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"response"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "parallel-unresponsive",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.HostErrorsCache = &fakeHostErrorsCache{}
err := req.Compile(executerOpts)
require.NoError(t, err)
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
err = req.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {})
require.NoError(t, err)
})
// Test Case 3: Context cancellation scenario
t.Run("ContextCancellation", func(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(200 * time.Millisecond)
_, _ = fmt.Fprintf(w, "response")
}))
defer ts.Close()
req := &Request{
ID: "parallel-context-cancel",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/test?param={{payload}}"},
Threads: 3,
Payloads: map[string]interface{}{
"payload": []string{"1", "2", "3", "4", "5"},
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Part: "body",
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"response"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "parallel-context-cancel",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
err := req.Compile(executerOpts)
require.NoError(t, err)
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
ctx, cancel := context.WithCancel(context.Background())
ctxArgs := contextargs.NewWithInput(ctx, ts.URL)
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()
err = req.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {})
require.Error(t, err)
require.Equal(t, context.Canceled, err)
})
}