diff --git a/integration_tests/headless/file-upload-negative.yaml b/integration_tests/headless/file-upload-negative.yaml new file mode 100644 index 000000000..3d8c2bf4a --- /dev/null +++ b/integration_tests/headless/file-upload-negative.yaml @@ -0,0 +1,29 @@ +id: file-upload +# template for testing when file upload is disabled +info: + name: Basic File Upload + author: pdteam + severity: info + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}" + - action: waitload + - action: files + args: + by: xpath + xpath: /html/body/form/input[1] + value: headless/file-upload.yaml + - action: sleep + args: + duration: 2 + - action: click + args: + by: x + xpath: /html/body/form/input[2] + matchers: + - type: word + words: + - "Basic File Upload" \ No newline at end of file diff --git a/integration_tests/headless/headless-local.yaml b/integration_tests/headless/headless-local.yaml new file mode 100644 index 000000000..385859d0a --- /dev/null +++ b/integration_tests/headless/headless-local.yaml @@ -0,0 +1,15 @@ +id: nuclei-headless-local + +info: + name: Nuclei Headless Local + author: pdteam + severity: high + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}" + + - action: waitload + \ No newline at end of file diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go index 185cdc87f..b30c464cc 100644 --- a/v2/cmd/integration-test/headless.go +++ b/v2/cmd/integration-test/headless.go @@ -16,7 +16,9 @@ var headlessTestcases = []TestCaseInfo{ {Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}}, {Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}}, {Path: "headless/variables.yaml", TestCase: &headlessVariables{}}, + {Path: "headless/headless-local.yaml", TestCase: &headlessLocal{}}, {Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}}, + {Path: "headless/file-upload-negative.yaml", TestCase: &headlessFileUploadNegative{}}, {Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}}, } @@ -39,6 +41,27 @@ func (h *headlessBasic) Execute(filePath string) error { return expectResultsCount(results, 1) } +type headlessLocal struct{} + +// Execute executes a test case and returns an error if occurred +// in this testcases local network access is disabled +func (h *headlessLocal) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + args := []string{"-t", filePath, "-u", ts.URL, "-headless", "-lna"} + + results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...) + if err != nil { + return err + } + return expectResultsCount(results, 0) +} + type headlessHeaderActions struct{} // Execute executes a test case and returns an error if occurred @@ -171,3 +194,48 @@ func (h *headlessHeaderStatus) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type headlessFileUploadNegative struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessFileUploadNegative) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte(` + + +
+ + +
+ + + `)) + }) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + _, _ = w.Write(content) + }) + ts := httptest.NewServer(router) + defer ts.Close() + args := []string{"-t", filePath, "-u", ts.URL, "-headless"} + + results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...) + if err != nil { + return err + } + return expectResultsCount(results, 0) +} diff --git a/v2/pkg/protocols/common/protocolstate/headless.go b/v2/pkg/protocols/common/protocolstate/headless.go new file mode 100644 index 000000000..86cbb2511 --- /dev/null +++ b/v2/pkg/protocols/common/protocolstate/headless.go @@ -0,0 +1,90 @@ +package protocolstate + +import ( + "strings" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" + "github.com/projectdiscovery/networkpolicy" + errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" + "go.uber.org/multierr" +) + +// initalize state of headless protocol + +var ( + ErrURLDenied = errorutil.NewWithFmt("headless: url %v dropped by rule: %v") + networkPolicy *networkpolicy.NetworkPolicy + allowLocalFileAccess bool +) + +// ValidateNFailRequest validates and fails request +// if the request does not respect the rules, it will be canceled with reason +func ValidateNFailRequest(page *rod.Page, e *proto.FetchRequestPaused) error { + reqURL := e.Request.URL + normalized := strings.ToLower(reqURL) // normalize url to lowercase + normalized = strings.TrimSpace(normalized) // trim leading & trailing whitespaces + if !allowLocalFileAccess && stringsutil.HasPrefixI(normalized, "file:") { + return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "use of file:// protocol disabled use '-lfa' to enable")) + } + // validate potential invalid schemes + // javascript protocol is allowed for xss fuzzing + if HasPrefixAnyI(normalized, "ftp:", "externalfile:", "chrome:", "chrome-extension:") { + return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "protocol blocked by network policy")) + } + if !isValidHost(reqURL) { + return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "address blocked by network policy")) + } + return nil +} + +// FailWithReason fails request with AccessDenied reason +func FailWithReason(page *rod.Page, e *proto.FetchRequestPaused) error { + m := proto.FetchFailRequest{ + RequestID: e.RequestID, + ErrorReason: proto.NetworkErrorReasonAccessDenied, + } + return m.Call(page) +} + +// InitHeadless initializes headless protocol state +func InitHeadless(RestrictLocalNetworkAccess bool, localFileAccess bool) { + allowLocalFileAccess = localFileAccess + if !RestrictLocalNetworkAccess { + return + } + networkPolicy, _ = networkpolicy.New(networkpolicy.Options{ + DenyList: append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...), + }) +} + +// isValidHost checks if the host is valid (only limited to http/https protocols) +func isValidHost(targetUrl string) bool { + if !stringsutil.HasPrefixAny(targetUrl, "http:", "https:") { + return true + } + if networkPolicy == nil { + return true + } + urlx, err := urlutil.Parse(targetUrl) + if err != nil { + // not a valid url + return false + } + targetUrl = urlx.Hostname() + _, ok := networkPolicy.ValidateHost(targetUrl) + return ok +} + +// HasPrefixAnyI checks if the string has any of the prefixes +// TODO: replace with stringsutil.HasPrefixAnyI after implementation +func HasPrefixAnyI(s string, prefixes ...string) bool { + for _, prefix := range prefixes { + if stringsutil.HasPrefixI(s, prefix) { + return true + } + } + return false +} diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index 829c2ce34..d07d1ba1b 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -22,6 +22,7 @@ func Init(options *types.Options) error { return nil } opts := fastdialer.DefaultOptions + InitHeadless(options.RestrictLocalNetworkAccess, options.AllowLocalFileAccess) switch { case options.SourceIP != "" && options.Interface != "": diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index a566ab37a..7ce2cd810 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -13,6 +13,7 @@ import ( "github.com/go-rod/rod/lib/proto" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Page is a single page in an isolated browser instance @@ -40,6 +41,7 @@ type HistoryData struct { type Options struct { Timeout time.Duration CookieReuse bool + Options *types.Options } // Run runs a list of actions by creating a new page in the browser. diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 93cd5018c..1b3cee291 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -30,7 +30,8 @@ import ( ) var ( - errinvalidArguments = errors.New("invalid arguments provided") + errinvalidArguments = errorutil.New("invalid arguments provided") + ErrLFAccessDenied = errorutil.New("Use -allow-local-file-access flag to enable local file access") ) const ( @@ -70,7 +71,11 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var case ActionWaitEvent: err = p.WaitEvent(act, outData) case ActionFilesInput: - err = p.FilesInput(act, outData) + if p.options.Options.AllowLocalFileAccess { + err = p.FilesInput(act, outData) + } else { + err = ErrLFAccessDenied + } case ActionAddHeader: err = p.ActionAddHeader(act, outData) case ActionSetHeader: diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 7930598d2..6b6c50b8f 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -8,6 +8,7 @@ import ( "net/http/cookiejar" "net/http/httptest" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -20,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless" "github.com/projectdiscovery/nuclei/v2/pkg/types" + stringsutil "github.com/projectdiscovery/utils/strings" ) func TestActionNavigate(t *testing.T) { @@ -36,7 +38,7 @@ func TestActionNavigate(t *testing.T) { actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}} testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") + require.Nilf(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") }) } @@ -316,6 +318,29 @@ func TestActionFilesInput(t *testing.T) { }) } +// Negative testcase for files input where it should fail +func TestActionFilesInputNegative(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + } + t.Setenv("LOCAL_FILE_ACCESS", "false") + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.ErrorContains(t, err, ErrLFAccessDenied.Error(), "got file access when -lfa is false") + }) +} + func TestActionWaitLoad(t *testing.T) { response := ` @@ -569,7 +594,10 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle input.CookieJar, err = cookiejar.New(nil) require.Nil(t, err) - extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout}) + lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true) + rna := getBoolFromEnv("RESTRICED_LOCAL_NETWORK_ACCESS", false) + + extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout, Options: &types.Options{AllowLocalFileAccess: lfa, RestrictLocalNetworkAccess: rna}}) // allow file access in test assert(page, err, extractedData) if page != nil { @@ -591,3 +619,73 @@ func TestContainsAnyModificationActionType(t *testing.T) { t.Error("Expected true, got false") } } + +func TestBlockedHeadlessURLS(t *testing.T) { + + // run this test from binary since we are changing values + // of global variables + if os.Getenv("TEST_BLOCK_HEADLESS_URLS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestBlockedHeadlessURLS", "-test.v") + cmd.Env = append(cmd.Env, "TEST_BLOCK_HEADLESS_URLS=1") + out, err := cmd.CombinedOutput() + if !strings.Contains(string(out), "PASS\n") || err != nil { + t.Fatalf("%s\n(exit status %v)", string(out), err) + } + return + } + + opts := &types.Options{ + AllowLocalFileAccess: false, + RestrictLocalNetworkAccess: true, + } + err := protocolstate.Init(opts) + require.Nil(t, err, "could not init protocol state") + + browser, err := New(&types.Options{ShowBrowser: false, UseInstalledChrome: testheadless.HeadlessLocal}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(nil) + defer ts.Close() + + testcases := []string{ + "file:/etc/hosts", + " file:///etc/hosts\r\n", + " fILe:/../../../../etc/hosts", + ts.URL, // local test server + "fTP://example.com:21\r\n", + "ftp://example.com:21", + "chrome://settings", + " chROme://version", + "chrome-extension://version\r", + " chrOme-EXTension://settings", + "view-source:file:/etc/hosts", + } + + for _, testcase := range testcases { + actions := []*Action{ + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": testcase}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + } + + data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test + require.Error(t, err, "expected error for url %s got %v", testcase, data) + require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase) + require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data) + if page != nil { + page.Close() + } + } +} + +func getBoolFromEnv(key string, defaultValue bool) bool { + val := os.Getenv(key) + if val == "" { + return defaultValue + } + return strings.EqualFold(val, "true") +} diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 22c0057fe..a28176dc1 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -7,6 +7,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" ) // routingRuleHandler handles proxy rule for actions related to request/response modification @@ -103,6 +104,12 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { // routingRuleHandlerNative handles native proxy rule func (p *Page) routingRuleHandlerNative(e *proto.FetchRequestPaused) error { + // ValidateNFailRequest validates if Local file access is enabled + // and local network access is enables if not it will fail the request + // that don't match the rules + if err := protocolstate.ValidateNFailRequest(p.page, e); err != nil { + return err + } body, _ := FetchGetResponseBody(p.page, e) headers := make(map[string][]string) for _, h := range e.ResponseHeaders { diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 500deb18c..942eb603f 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -106,6 +106,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p options := &engine.Options{ Timeout: time.Duration(request.options.Options.PageTimeout) * time.Second, CookieReuse: request.CookieReuse, + Options: request.options.Options, } if options.CookieReuse && input.CookieJar == nil { diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index bd773a9c3..79cad7e6e 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -77,6 +77,33 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro return parts, nil } +// RunNucleiArgsAndGetResults returns result,and runtime errors +func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) { + cmd := exec.Command("./nuclei", args...) + if debug { + cmd.Args = append(cmd.Args, "-debug") + cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") + } + data, err := cmd.Output() + if debug { + fmt.Println(string(data)) + } + if len(data) < 1 && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + } + var parts []string + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} + // RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL) func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) { cmd := exec.Command("./nuclei")