uniforming sizes with utils

cmd/integration-test/http.go

@@ -23,6 +23,7 @@ import (
 	logutil "github.com/projectdiscovery/utils/log"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 var httpTestcases = []TestCaseInfo{
@@ -509,7 +510,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error {
 	var routerErr error
 
 	router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		if err := r.ParseMultipartForm(1 * 1024); err != nil {
+		if err := r.ParseMultipartForm(unitutils.Mega); err != nil {
 			routerErr = err
 			return
 		}

cmd/nuclei/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/monitor"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
+	unitutils "github.com/projectdiscovery/utils/unit"
 	updateutils "github.com/projectdiscovery/utils/update"
 )
 
@@ -301,7 +302,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
 		flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
 		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
-		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
+		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
 		flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
 	github.com/projectdiscovery/fastdialer v0.0.71
-	github.com/projectdiscovery/hmap v0.0.41
+	github.com/projectdiscovery/hmap v0.0.42
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.47
 	github.com/projectdiscovery/retryabledns v1.0.59
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.49
-	github.com/projectdiscovery/utils v0.0.92
+	github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c
 	github.com/projectdiscovery/wappalyzergo v0.0.122
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1

go.sum

@@ -853,8 +853,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
 github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
 github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE=
-github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0=
+github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM=
+github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c=
 github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ=
 github.com/projectdiscovery/httpx v1.6.0/go.mod h1:dzMzOWKfeofaXcXzac3O+YmuY24P0CRnviKGxvol6MM=
 github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
@@ -889,8 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg=
 github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
-github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
-github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
+github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c h1:vGsulLeJNcFjQTp+aSAO7auX5wriKrLrZ8+SILxtRcM=
+github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw=
 github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c=
 github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

internal/pdcp/writer.go

@@ -19,6 +19,7 @@ import (
 	"github.com/projectdiscovery/retryablehttp-go"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	unitutils "github.com/projectdiscovery/utils/unit"
 	updateutils "github.com/projectdiscovery/utils/update"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
@@ -26,8 +27,8 @@ import (
 const (
 	uploadEndpoint = "/v1/scans/import"
 	appendEndpoint = "/v1/scans/%s/import"
-	flushTimer     = time.Duration(1) * time.Minute
-	MaxChunkSize   = 1024 * 1024 * 4 // 4 MB
+	flushTimer     = time.Minute
+	MaxChunkSize   = 4 * unitutils.Mega // 4 MB
 	xidRe          = `^[a-z0-9]{20}$`
 )
 

pkg/output/output.go

@@ -30,6 +30,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	fileutil "github.com/projectdiscovery/utils/file"
 	osutils "github.com/projectdiscovery/utils/os"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 // Writer is an interface which writes output to somewhere for nuclei events.
@@ -411,7 +412,7 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error
 	return w.Write(data)
 }
 
-var maxTemplateFileSizeForEncoding = 1024 * 1024
+var maxTemplateFileSizeForEncoding = unitutils.Mega
 
 func (w *StandardWriter) encodeTemplate(templatePath string) string {
 	data, err := os.ReadFile(templatePath)

pkg/protocols/common/automaticscan/automaticscan.go

@@ -32,13 +32,14 @@ import (
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	syncutil "github.com/projectdiscovery/utils/sync"
+	unitutils "github.com/projectdiscovery/utils/unit"
 	wappalyzer "github.com/projectdiscovery/wappalyzergo"
 	"gopkg.in/yaml.v2"
 )
 
 const (
 	mappingFilename = "wappalyzer-mapping.yml"
-	maxDefaultBody  = 4 * 1024 * 1024 // 4MB
+	maxDefaultBody  = 4 * unitutils.Mega
 )
 
 // Options contains configuration options for automatic scan service

pkg/protocols/http/request.go

@@ -42,6 +42,7 @@ import (
 	"github.com/projectdiscovery/utils/reader"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	unitutils "github.com/projectdiscovery/utils/unit"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -53,7 +54,7 @@ const (
 )
 
 var (
-	MaxBodyRead = int64(10 * 1024 * 1024) // 10MB
+	MaxBodyRead = 10 * unitutils.Mega
 	// ErrMissingVars is error occured when variables are missing
 	ErrMissingVars = errors.New("stop execution due to unresolved variables")
 )
@@ -557,7 +558,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	return requestErr
 }
 
-const drainReqSize = int64(8 * 1024)
+const drainReqSize = int64(8 * unitutils.Kilo)
 
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
@@ -809,15 +810,15 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	// define max body read limit
 	maxBodylimit := MaxBodyRead // 10MB
 	if request.MaxSize > 0 {
-		maxBodylimit = int64(request.MaxSize)
+		maxBodylimit = request.MaxSize
 	}
 	if request.options.Options.ResponseReadSize != 0 {
-		maxBodylimit = int64(request.options.Options.ResponseReadSize)
+		maxBodylimit = request.options.Options.ResponseReadSize
 	}
 
 	// respChain is http response chain that reads response body
 	// efficiently by reusing buffers and does all decoding and optimizations
-	respChain := httpUtils.NewResponseChain(resp, maxBodylimit)
+	respChain := httpUtils.NewResponseChain(resp, int64(maxBodylimit))
 	defer respChain.Close() // reuse buffers
 
 	// we only intend to log/save the final redirected response

pkg/protocols/offlinehttp/request.go

@@ -17,11 +17,12 @@ import (
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/utils/conversion"
 	syncutil "github.com/projectdiscovery/utils/sync"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 var _ protocols.Request = &Request{}
 
-const maxSize = 5 * 1024 * 1024
+const maxSize = 5 * unitutils.Mega
 
 // Type returns the type of the protocol request
 func (request *Request) Type() templateTypes.ProtocolType {

pkg/protocols/protocols.go

@@ -33,10 +33,11 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 var (
-	MaxTemplateFileSizeForEncoding = 1024 * 1024
+	MaxTemplateFileSizeForEncoding = unitutils.Mega
 )
 
 // Executer is an interface implemented any protocol based request executer.

pkg/reporting/format/format_utils.go

@@ -12,6 +12,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 // Summary returns a formatted built one line summary of the event
@@ -71,7 +72,7 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte
 		if event.Response != "" {
 			var responseString string
 			// If the response is larger than 5 kb, truncate it before writing.
-			maxKbSize := 5 * 1024
+			maxKbSize := 5 * unitutils.Kilo
 			if len(event.Response) > maxKbSize {
 				responseString = event.Response[:maxKbSize]
 				responseString += ".... Truncated ...."

pkg/testutils/testutils.go

@@ -23,6 +23,7 @@ import (
 	protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 // Init initializes the protocols and their configurations
@@ -201,7 +202,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven
 	return m.Write(data)
 }
 
-var maxTemplateFileSizeForEncoding = 1024 * 1024
+var maxTemplateFileSizeForEncoding = unitutils.Mega
 
 func (w *MockOutputWriter) encodeTemplate(templatePath string) string {
 	data, err := os.ReadFile(templatePath)

pkg/types/types.go

@@ -15,6 +15,7 @@ import (
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 var (
@@ -435,8 +436,8 @@ func DefaultOptions() *Options {
 		Timeout:                 5,
 		Retries:                 1,
 		MaxHostError:            30,
-		ResponseReadSize:        10 * 1024 * 1024,
-		ResponseSaveSize:        1024 * 1024,
+		ResponseReadSize:        10 * unitutils.Mega,
+		ResponseSaveSize:        unitutils.Mega,
 		ResponseReadTimeout:     5 * time.Second,
 	}
 }

pkg/utils/monitor/monitor.go

@@ -16,6 +16,7 @@ import (
 	"github.com/DataDog/gostackparse"
 	"github.com/projectdiscovery/gologger"
 	permissionutil "github.com/projectdiscovery/utils/permission"
+	unitutils "github.com/projectdiscovery/utils/unit"
 	"github.com/rs/xid"
 )
 
@@ -118,7 +119,7 @@ func (s *Agent) monitorWorker(cancel context.CancelFunc) {
 
 // getStack returns full stack trace of the program
 var getStack = func(all bool) []byte {
-	for i := 1024 * 1024; ; i *= 2 {
+	for i := unitutils.Mega; ; i *= 2 {
 		buf := make([]byte, i)
 		if n := runtime.Stack(buf, all); n < i {
 			return buf[:n-1]