Tarun Koyalwar 4f93520e47
javascript protocol for scripting (includes 15+ proto libs) (#4109)
* rebase js-layer PR from @ice3man543

* package restructuring

* working

* fix duplicated event & matcher status

* fix lint error

* fix response field

* add new functions

* multiple minor improvements

* fix incorrect stats in js protocol

* sort output metadata in cli

* remove temp files

* remove dead code

* add unit and integration test

* fix lint error

* add jsdoclint using llm

* fix error in test

* add js lint using llm

* generate docs of libs

* llm lint

* remove duplicated docs

* update generated docs

* update prompt in doclint

* update docs

* temp disable version check test

* fix unit test and add retry

* fix panic in it

* update and move jsdocs

* updated jsdocs

* update docs

* update container platform in test

* dir restructure and adding docs

* add api_reference and remove markdown docs

* fix imports

* add javascript design and contribution docs

* add js protocol documentation

* update integration test and docs

* update doc ext mdx->md

* minor update to docs

* new integration test and more

* move go libs and add docs

* gen new net docs and more

* final docs update

* add new devtool

* use fastdialer

* fix build fail

* use fastdialer + network sandbox support

* add reserved keyword 'Port'

* update Port to new syntax

* misc update

* always enable templatectx in js protocol

* move docs to 'js-proto-docs' repo

* remove scrapefuncs binary

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
2023-09-16 16:02:17 +05:30

90 lines
2.9 KiB
Go

package protocolstate
import (
"strings"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/networkpolicy"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
"go.uber.org/multierr"
)
// initalize state of headless protocol
var (
ErrURLDenied = errorutil.NewWithFmt("headless: url %v dropped by rule: %v")
ErrHostDenied = errorutil.NewWithFmt("host %v dropped by network policy")
networkPolicy *networkpolicy.NetworkPolicy
allowLocalFileAccess bool
)
// ValidateNFailRequest validates and fails request
// if the request does not respect the rules, it will be canceled with reason
func ValidateNFailRequest(page *rod.Page, e *proto.FetchRequestPaused) error {
reqURL := e.Request.URL
normalized := strings.ToLower(reqURL) // normalize url to lowercase
normalized = strings.TrimSpace(normalized) // trim leading & trailing whitespaces
if !allowLocalFileAccess && stringsutil.HasPrefixI(normalized, "file:") {
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "use of file:// protocol disabled use '-lfa' to enable"))
}
// validate potential invalid schemes
// javascript protocol is allowed for xss fuzzing
if stringsutil.HasPrefixAnyI(normalized, "ftp:", "externalfile:", "chrome:", "chrome-extension:") {
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "protocol blocked by network policy"))
}
if !isValidHost(reqURL) {
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "address blocked by network policy"))
}
return nil
}
// FailWithReason fails request with AccessDenied reason
func FailWithReason(page *rod.Page, e *proto.FetchRequestPaused) error {
m := proto.FetchFailRequest{
RequestID: e.RequestID,
ErrorReason: proto.NetworkErrorReasonAccessDenied,
}
return m.Call(page)
}
// InitHeadless initializes headless protocol state
func InitHeadless(RestrictLocalNetworkAccess bool, localFileAccess bool) {
allowLocalFileAccess = localFileAccess
if !RestrictLocalNetworkAccess {
return
}
networkPolicy, _ = networkpolicy.New(networkpolicy.Options{
DenyList: append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...),
})
}
// isValidHost checks if the host is valid (only limited to http/https protocols)
func isValidHost(targetUrl string) bool {
if !stringsutil.HasPrefixAny(targetUrl, "http:", "https:") {
return true
}
if networkPolicy == nil {
return true
}
urlx, err := urlutil.Parse(targetUrl)
if err != nil {
// not a valid url
return false
}
targetUrl = urlx.Hostname()
_, ok := networkPolicy.ValidateHost(targetUrl)
return ok
}
// IsHostAllowed checks if the host is allowed by network policy
func IsHostAllowed(targetUrl string) bool {
if networkPolicy == nil {
return true
}
_, ok := networkPolicy.ValidateHost(targetUrl)
return ok
}