diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index b4f957e69..5b218aa53 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -23,6 +23,7 @@ import ( logutil "github.com/projectdiscovery/utils/log" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" + unitutils "github.com/projectdiscovery/utils/unit" ) var httpTestcases = []TestCaseInfo{ @@ -509,7 +510,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - if err := r.ParseMultipartForm(1 * 1024); err != nil { + if err := r.ParseMultipartForm(unitutils.Mega); err != nil { routerErr = err return } diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index a1123b9ff..8be020779 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -38,6 +38,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/utils/monitor" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + unitutils "github.com/projectdiscovery/utils/unit" updateutils "github.com/projectdiscovery/utils/update" ) @@ -301,7 +302,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"), flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"), flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"), - flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"), + flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"), flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"), flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"), flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), diff --git a/go.mod b/go.mod index 1002e41ef..39156d2f6 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/fastdialer v0.0.71 - github.com/projectdiscovery/hmap v0.0.41 + github.com/projectdiscovery/hmap v0.0.42 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.47 github.com/projectdiscovery/retryabledns v1.0.59 @@ -94,7 +94,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.49 - github.com/projectdiscovery/utils v0.0.92 + github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c github.com/projectdiscovery/wappalyzergo v0.0.122 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index dbc2f68ae..4a95dbd04 100644 --- a/go.sum +++ b/go.sum @@ -853,8 +853,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE= -github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0= +github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= +github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ= github.com/projectdiscovery/httpx v1.6.0/go.mod h1:dzMzOWKfeofaXcXzac3O+YmuY24P0CRnviKGxvol6MM= github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc= @@ -889,8 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg= github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0= -github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= -github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= +github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c h1:vGsulLeJNcFjQTp+aSAO7auX5wriKrLrZ8+SILxtRcM= +github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw= github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c= github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index a76d4ea31..6e0541035 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/retryablehttp-go" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" errorutil "github.com/projectdiscovery/utils/errors" + unitutils "github.com/projectdiscovery/utils/unit" updateutils "github.com/projectdiscovery/utils/update" urlutil "github.com/projectdiscovery/utils/url" ) @@ -26,8 +27,8 @@ import ( const ( uploadEndpoint = "/v1/scans/import" appendEndpoint = "/v1/scans/%s/import" - flushTimer = time.Duration(1) * time.Minute - MaxChunkSize = 1024 * 1024 * 4 // 4 MB + flushTimer = time.Minute + MaxChunkSize = 4 * unitutils.Mega // 4 MB xidRe = `^[a-z0-9]{20}$` ) diff --git a/pkg/output/output.go b/pkg/output/output.go index 044d164f1..23b2f0ec9 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -30,6 +30,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/utils" fileutil "github.com/projectdiscovery/utils/file" osutils "github.com/projectdiscovery/utils/os" + unitutils "github.com/projectdiscovery/utils/unit" ) // Writer is an interface which writes output to somewhere for nuclei events. @@ -411,7 +412,7 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error return w.Write(data) } -var maxTemplateFileSizeForEncoding = 1024 * 1024 +var maxTemplateFileSizeForEncoding = unitutils.Mega func (w *StandardWriter) encodeTemplate(templatePath string) string { data, err := os.ReadFile(templatePath) diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go index 2a7988cb0..a5e51c177 100644 --- a/pkg/protocols/common/automaticscan/automaticscan.go +++ b/pkg/protocols/common/automaticscan/automaticscan.go @@ -32,13 +32,14 @@ import ( sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" syncutil "github.com/projectdiscovery/utils/sync" + unitutils "github.com/projectdiscovery/utils/unit" wappalyzer "github.com/projectdiscovery/wappalyzergo" "gopkg.in/yaml.v2" ) const ( mappingFilename = "wappalyzer-mapping.yml" - maxDefaultBody = 4 * 1024 * 1024 // 4MB + maxDefaultBody = 4 * unitutils.Mega ) // Options contains configuration options for automatic scan service diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index df914a785..fc3df039a 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -42,6 +42,7 @@ import ( "github.com/projectdiscovery/utils/reader" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" + unitutils "github.com/projectdiscovery/utils/unit" urlutil "github.com/projectdiscovery/utils/url" ) @@ -53,7 +54,7 @@ const ( ) var ( - MaxBodyRead = int64(10 * 1024 * 1024) // 10MB + MaxBodyRead = 10 * unitutils.Mega // ErrMissingVars is error occured when variables are missing ErrMissingVars = errors.New("stop execution due to unresolved variables") ) @@ -557,7 +558,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa return requestErr } -const drainReqSize = int64(8 * 1024) +const drainReqSize = int64(8 * unitutils.Kilo) // executeRequest executes the actual generated request and returns error if occurred func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) { @@ -809,15 +810,15 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // define max body read limit maxBodylimit := MaxBodyRead // 10MB if request.MaxSize > 0 { - maxBodylimit = int64(request.MaxSize) + maxBodylimit = request.MaxSize } if request.options.Options.ResponseReadSize != 0 { - maxBodylimit = int64(request.options.Options.ResponseReadSize) + maxBodylimit = request.options.Options.ResponseReadSize } // respChain is http response chain that reads response body // efficiently by reusing buffers and does all decoding and optimizations - respChain := httpUtils.NewResponseChain(resp, maxBodylimit) + respChain := httpUtils.NewResponseChain(resp, int64(maxBodylimit)) defer respChain.Close() // reuse buffers // we only intend to log/save the final redirected response diff --git a/pkg/protocols/offlinehttp/request.go b/pkg/protocols/offlinehttp/request.go index 4a440c167..e913e02d8 100644 --- a/pkg/protocols/offlinehttp/request.go +++ b/pkg/protocols/offlinehttp/request.go @@ -17,11 +17,12 @@ import ( templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/utils/conversion" syncutil "github.com/projectdiscovery/utils/sync" + unitutils "github.com/projectdiscovery/utils/unit" ) var _ protocols.Request = &Request{} -const maxSize = 5 * 1024 * 1024 +const maxSize = 5 * unitutils.Mega // Type returns the type of the protocol request func (request *Request) Type() templateTypes.ProtocolType { diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 6b4904c8d..7804ea3c2 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -33,10 +33,11 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + unitutils "github.com/projectdiscovery/utils/unit" ) var ( - MaxTemplateFileSizeForEncoding = 1024 * 1024 + MaxTemplateFileSizeForEncoding = unitutils.Mega ) // Executer is an interface implemented any protocol based request executer. diff --git a/pkg/reporting/format/format_utils.go b/pkg/reporting/format/format_utils.go index 62a3d75fe..d5ec0adfb 100644 --- a/pkg/reporting/format/format_utils.go +++ b/pkg/reporting/format/format_utils.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils" + unitutils "github.com/projectdiscovery/utils/unit" ) // Summary returns a formatted built one line summary of the event @@ -71,7 +72,7 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte if event.Response != "" { var responseString string // If the response is larger than 5 kb, truncate it before writing. - maxKbSize := 5 * 1024 + maxKbSize := 5 * unitutils.Kilo if len(event.Response) > maxKbSize { responseString = event.Response[:maxKbSize] responseString += ".... Truncated ...." diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 68410f7f1..a54fbbcf4 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -23,6 +23,7 @@ import ( protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils" + unitutils "github.com/projectdiscovery/utils/unit" ) // Init initializes the protocols and their configurations @@ -201,7 +202,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven return m.Write(data) } -var maxTemplateFileSizeForEncoding = 1024 * 1024 +var maxTemplateFileSizeForEncoding = unitutils.Mega func (w *MockOutputWriter) encodeTemplate(templatePath string) string { data, err := os.ReadFile(templatePath) diff --git a/pkg/types/types.go b/pkg/types/types.go index 0c31769ee..577461b21 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -15,6 +15,7 @@ import ( errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" + unitutils "github.com/projectdiscovery/utils/unit" ) var ( @@ -435,8 +436,8 @@ func DefaultOptions() *Options { Timeout: 5, Retries: 1, MaxHostError: 30, - ResponseReadSize: 10 * 1024 * 1024, - ResponseSaveSize: 1024 * 1024, + ResponseReadSize: 10 * unitutils.Mega, + ResponseSaveSize: unitutils.Mega, ResponseReadTimeout: 5 * time.Second, } } diff --git a/pkg/utils/monitor/monitor.go b/pkg/utils/monitor/monitor.go index 6441fdf20..cefb9506b 100644 --- a/pkg/utils/monitor/monitor.go +++ b/pkg/utils/monitor/monitor.go @@ -16,6 +16,7 @@ import ( "github.com/DataDog/gostackparse" "github.com/projectdiscovery/gologger" permissionutil "github.com/projectdiscovery/utils/permission" + unitutils "github.com/projectdiscovery/utils/unit" "github.com/rs/xid" ) @@ -118,7 +119,7 @@ func (s *Agent) monitorWorker(cancel context.CancelFunc) { // getStack returns full stack trace of the program var getStack = func(all bool) []byte { - for i := 1024 * 1024; ; i *= 2 { + for i := unitutils.Mega; ; i *= 2 { buf := make([]byte, i) if n := runtime.Stack(buf, all); n < i { return buf[:n-1]