feat: fix utils and add goroutine leak unit tests (#5112)

* feat: fixed leak

* add go leak unit test in sdk

* added goleak unit tests

* bugfix: add random user agents to fuzzing requests

* misc

* misc

* fix lint + use utils pr + misc

* fix ratelimit memleak in sdk

* close protocolstate shared resources in nuclei sdk/lib

* add missing close references

* ignore read/write loop of intransit connections

* close unnecessary idle conns

* add ignore method

* using fixed utils

* dep update

---------

Co-authored-by: Ice3man <nizamulrana@gmail.com>
Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.68
+	github.com/projectdiscovery/fastdialer v0.0.69
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.47
 	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.57
+	github.com/projectdiscovery/retryablehttp-go v1.0.58
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -94,11 +94,12 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.47
-	github.com/projectdiscovery/utils v0.0.91
+	github.com/projectdiscovery/utils v0.0.92
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
+	github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
 	golang.org/x/term v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -142,8 +143,6 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/eapache/channels v1.1.0 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -179,6 +178,7 @@ require (
 	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -232,6 +232,7 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	github.com/zcalusic/sysinfo v1.0.2 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect

go.sum

@@ -296,11 +296,8 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
-github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
@@ -317,6 +314,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
@@ -671,6 +670,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
+github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
 github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 h1:wIONC+HMNRqmWBjuMxhatuSzHaljStc4gjDeKycxy0A=
 github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
@@ -836,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
 github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
-github.com/projectdiscovery/fastdialer v0.0.68 h1:JuIrr8aVGdGWkEwL4axsJWAWDY2uviSqBB0TCekeCOo=
+github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU=
-github.com/projectdiscovery/fastdialer v0.0.68/go.mod h1:asHSBFJgmwrXpiegcrcAgOyd/QewCVgeI4idH55+v7M=
+github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -876,8 +877,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
 github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
-github.com/projectdiscovery/retryablehttp-go v1.0.57 h1:OGfUXKXV4bE5msGxeRrNtMaDg2l8U1JcLXmwG7yXWrY=
+github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4=
-github.com/projectdiscovery/retryablehttp-go v1.0.57/go.mod h1:Lo2EU1wV1draQ/dHuiSkokW4gZ216F/qi/t12DIdMbA=
+github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -888,8 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0=
 github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M=
-github.com/projectdiscovery/utils v0.0.91 h1:aHAAnC0qX9pJZrWq4Qpl2PSTYLrSCL1dm1QWLjprE2w=
+github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
-github.com/projectdiscovery/utils v0.0.91/go.mod h1:O/6U3ZoU+tNw4lKurdjyVMZPVXL5IYq0YeaDc15PRls=
+github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -1015,6 +1016,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg=
+github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
@@ -1149,8 +1152,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

internal/runner/inputs.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"context"
 	"sync/atomic"
 	"time"
 
@@ -50,7 +51,9 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		}
 
 		if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency {
-			swg.Resize(r.options.ProbeConcurrency)
+			if err := swg.Resize(context.Background(), r.options.ProbeConcurrency); err != nil {
+				gologger.Error().Msgf("Could not resize workpool: %s\n", err)
+			}
 		}
 
 		swg.Add()

lib/multi.go

@@ -58,6 +58,22 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
 	return u, nil
 }
 
+// closeEphemeralObjects closes all resources used by ephemeral nuclei objects/instances/types
+func closeEphemeralObjects(u *unsafeOptions) {
+	if u.executerOpts.RateLimiter != nil {
+		u.executerOpts.RateLimiter.Stop()
+	}
+	// dereference all objects that were inherited from base nuclei engine
+	// since these are meant to be closed globally by base nuclei engine
+	u.executerOpts.Output = nil
+	u.executerOpts.IssuesClient = nil
+	u.executerOpts.Interactsh = nil
+	u.executerOpts.HostErrorsCache = nil
+	u.executerOpts.Progress = nil
+	u.executerOpts.Catalog = nil
+	u.executerOpts.Parser = nil
+}
+
 // ThreadSafeNucleiEngine is a tweaked version of nuclei.Engine whose methods are thread-safe
 // and can be used concurrently. Non-thread-safe methods start with Global prefix
 type ThreadSafeNucleiEngine struct {
@@ -107,11 +123,14 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ..
 			return err
 		}
 	}
+	defer tmpEngine.Close()
 	// create ephemeral nuclei objects/instances/types using base nuclei engine
 	unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
 	if err != nil {
 		return err
 	}
+	// cleanup and stop all resources
+	defer closeEphemeralObjects(unsafeOpts)
 
 	// load templates
 	workflowLoader, err := workflow.NewLoader(&unsafeOpts.executerOpts)

lib/sdk.go

@@ -18,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -184,11 +185,38 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by
 
 // Close all resources used by nuclei engine
 func (e *NucleiEngine) Close() {
-	e.interactshClient.Close()
+	if e.interactshClient != nil {
-	e.rc.Close()
+		e.interactshClient.Close()
-	e.customWriter.Close()
+	}
-	e.hostErrCache.Close()
+	if e.rc != nil {
-	e.executerOpts.RateLimiter.Stop()
+		e.rc.Close()
+	}
+	if e.customWriter != nil {
+		e.customWriter.Close()
+	}
+	if e.customProgress != nil {
+		e.customProgress.Stop()
+	}
+	if e.hostErrCache != nil {
+		e.hostErrCache.Close()
+	}
+	if e.executerOpts.RateLimiter != nil {
+		e.executerOpts.RateLimiter.Stop()
+	}
+	if e.rateLimiter != nil {
+		e.rateLimiter.Stop()
+	}
+	// close global shared resources
+	protocolstate.Close()
+	if e.inputProvider != nil {
+		e.inputProvider.Close()
+	}
+	if e.browserInstance != nil {
+		e.browserInstance.Close()
+	}
+	if e.httpxClient != nil {
+		_ = e.httpxClient.Close()
+	}
 }
 
 // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)

lib/sdk_private.go

@@ -35,6 +35,8 @@ import (
 	"github.com/projectdiscovery/ratelimit"
 )
 
+var sharedInit sync.Once = sync.Once{}
+
 // applyRequiredDefaults to options
 func (e *NucleiEngine) applyRequiredDefaults() {
 	mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate)
@@ -116,8 +118,11 @@ func (e *NucleiEngine) init() error {
 
 	e.parser = templates.NewParser()
 
-	_ = protocolstate.Init(e.opts)
+	sharedInit.Do(func() {
-	_ = protocolinit.Init(e.opts)
+		_ = protocolstate.Init(e.opts)
+		_ = protocolinit.Init(e.opts)
+	})
+
 	e.applyRequiredDefaults()
 	var err error
 

lib/sdk_test.go

@@ -1,60 +0,0 @@
-package nuclei_test
-
-import (
-	"testing"
-
-	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSimpleNuclei(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
-		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
-	)
-	require.Nil(t, err)
-	ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
-	// when callback is nil it nuclei will print JSON output to stdout
-	err = ne.ExecuteWithCallback(nil)
-	require.Nil(t, err)
-	defer ne.Close()
-}
-
-func TestSimpleNucleiRemote(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplatesOrWorkflows(
-			nuclei.TemplateSources{
-				RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"},
-			},
-		),
-	)
-	require.Nil(t, err)
-	ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
-	err = ne.LoadAllTemplates()
-	require.Nil(t, err, "could not load templates")
-	// when callback is nil it nuclei will print JSON output to stdout
-	err = ne.ExecuteWithCallback(nil)
-	require.Nil(t, err)
-	defer ne.Close()
-}
-
-func TestThreadSafeNuclei(t *testing.T) {
-	// create nuclei engine with options
-	ne, err := nuclei.NewThreadSafeNucleiEngine()
-	require.Nil(t, err)
-
-	// scan 1 = run dns templates on scanme.sh
-	t.Run("scanme.sh", func(t *testing.T) {
-		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
-		require.Nil(t, err)
-	})
-
-	// scan 2 = run dns templates on honey.scanme.sh
-	t.Run("honey.scanme.sh", func(t *testing.T) {
-		err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
-		require.Nil(t, err)
-	})
-
-	// wait for all scans to finish
-	defer ne.Close()
-}

lib/tests/sdk_test.go

@@ -0,0 +1,132 @@
+package sdk_test
+
+import (
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/utils/env"
+	"github.com/stretchr/testify/require"
+	"github.com/tarunKoyalwar/goleak"
+)
+
+var knownLeaks = []goleak.Option{
+	// prettyify the output and generate dependency graph and more details instead of just stack output
+	goleak.Pretty(),
+	// net/http transport maintains idle connections which are closed with cooldown
+	// hence they don't count as leaks
+	goleak.IgnoreAnyFunction("net/http.(*http2ClientConn).readLoop"),
+}
+
+func TestSimpleNuclei(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		ne, err := nuclei.NewNucleiEngine(
+			nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
+			nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
+		)
+		require.Nil(t, err)
+		ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
+		// when callback is nil it nuclei will print JSON output to stdout
+		err = ne.ExecuteWithCallback(nil)
+		require.Nil(t, err)
+		defer ne.Close()
+	}
+
+	// this is shared test so needs to be run as seperate process
+	if env.GetEnvOrDefault("TestSimpleNuclei", false) {
+		// run as new process
+		cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNuclei")
+		cmd.Env = append(os.Environ(), "TestSimpleNuclei=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}
+
+func TestSimpleNucleiRemote(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		ne, err := nuclei.NewNucleiEngine(
+			nuclei.WithTemplatesOrWorkflows(
+				nuclei.TemplateSources{
+					RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"},
+				},
+			),
+		)
+		require.Nil(t, err)
+		ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
+		err = ne.LoadAllTemplates()
+		require.Nil(t, err, "could not load templates")
+		// when callback is nil it nuclei will print JSON output to stdout
+		err = ne.ExecuteWithCallback(nil)
+		require.Nil(t, err)
+		defer ne.Close()
+	}
+	// this is shared test so needs to be run as seperate process
+	if env.GetEnvOrDefault("TestSimpleNucleiRemote", false) {
+		cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNucleiRemote")
+		cmd.Env = append(os.Environ(), "TestSimpleNucleiRemote=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}
+
+func TestThreadSafeNuclei(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		// create nuclei engine with options
+		ne, err := nuclei.NewThreadSafeNucleiEngine()
+		require.Nil(t, err)
+
+		// scan 1 = run dns templates on scanme.sh
+		t.Run("scanme.sh", func(t *testing.T) {
+			err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+			require.Nil(t, err)
+		})
+
+		// scan 2 = run dns templates on honey.scanme.sh
+		t.Run("honey.scanme.sh", func(t *testing.T) {
+			err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+			require.Nil(t, err)
+		})
+
+		// wait for all scans to finish
+		defer ne.Close()
+	}
+
+	if env.GetEnvOrDefault("TestThreadSafeNuclei", false) {
+		cmd := exec.Command(os.Args[0], "-test.run=TestThreadSafeNuclei")
+		cmd.Env = append(os.Environ(), "TestThreadSafeNuclei=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}

pkg/core/executors.go

@@ -162,7 +162,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
 	// wp is workpool that contains different waitgroups for
 	// headless and non-headless templates
 	// global waitgroup should not be used here
-	wp := e.GetWorkPool()
+	wp := e.WorkPool()
 
 	for _, tpl := range alltemplates {
 		select {
@@ -212,54 +212,3 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
 	}
 	wp.Wait()
 }
-
-type ChildExecuter struct {
-	e *Engine
-
-	results *atomic.Bool
-}
-
-// Close closes the executer returning bool results
-func (e *ChildExecuter) Close() *atomic.Bool {
-	e.e.workPool.Wait()
-	return e.results
-}
-
-// Execute executes a template and URLs
-func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
-	templateType := template.Type()
-
-	// resize check point - nop if there are no changes
-	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
-
-	var wg *syncutil.AdaptiveWaitGroup
-	if templateType == types.HeadlessProtocol {
-		wg = e.e.workPool.Headless
-	} else {
-		wg = e.e.workPool.Default
-	}
-
-	wg.Add()
-	go func(tpl *templates.Template) {
-		defer wg.Done()
-
-		// TODO: Workflows are a no-op for now. We need to
-		// implement them in the future with context cancellation
-		ctxArgs := contextargs.New(context.Background())
-		ctxArgs.MetaInput = value
-		ctx := scan.NewScanContext(context.Background(), ctxArgs)
-		match, err := template.Executer.Execute(ctx)
-		if err != nil {
-			gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
-		}
-		e.results.CompareAndSwap(false, match)
-	}(template)
-}
-
-// ExecuteWithOpts executes with the full options
-func (e *Engine) ChildExecuter() *ChildExecuter {
-	return &ChildExecuter{
-		e:       e,
-		results: &atomic.Bool{},
-	}
-}

pkg/core/workpool.go

@@ -1,6 +1,9 @@
 package core
 
 import (
+	"context"
+
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
@@ -71,14 +74,18 @@ func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
 	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
 		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
 	}
-	w.Refresh()
+	w.Refresh(context.Background())
 }
 
-func (w *WorkPool) Refresh() {
+func (w *WorkPool) Refresh(ctx context.Context) {
 	if w.Default.Size != w.config.TypeConcurrency {
-		w.Default.Resize(w.config.TypeConcurrency)
+		if err := w.Default.Resize(ctx, w.config.TypeConcurrency); err != nil {
+			gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+		}
 	}
 	if w.Headless.Size != w.config.HeadlessTypeConcurrency {
-		w.Headless.Resize(w.config.HeadlessTypeConcurrency)
+		if err := w.Headless.Resize(ctx, w.config.HeadlessTypeConcurrency); err != nil {
+			gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+		}
 	}
 }

pkg/input/types/http.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
 	"github.com/projectdiscovery/utils/conversion"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -73,6 +74,10 @@ func (rr *RequestResponse) BuildRequest() (*retryablehttp.Request, error) {
 			req.Header.Add(k, v)
 			return true
 		})
+		if req.Header.Get("User-Agent") == "" {
+			userAgent := useragent.PickRandom()
+			req.Header.Set("User-Agent", userAgent.Raw)
+		}
 		rr.req = req
 	})
 	return rr.req, rr.reqErr

pkg/input/types/probe.go

@@ -4,4 +4,6 @@ package types
 type InputLivenessProbe interface {
 	// ProbeURL probes the scheme for a URL. first HTTPS is tried
 	ProbeURL(input string) (string, error)
+	// Close closes the liveness probe
+	Close() error
 }

pkg/installer/versioncheck.go

@@ -83,6 +83,10 @@ func UpdateIgnoreFile() error {
 }
 
 func doVersionCheck(isSDK bool) error {
+	// we use global retryablehttp client so its not immeditely gc'd if any references are held
+	// and according our config we have idle connections which are shown as leaked by goleak in tests
+	// i.e we close all idle connections after our use and it doesn't affect any other part of the code
+	defer retryableHttpClient.HTTPClient.CloseIdleConnections()
 	resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK))
 	if err != nil {
 		return err

pkg/js/compiler/non-pool.go

@@ -8,7 +8,7 @@ import (
 )
 
 var (
-	ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
+	ephemeraljsc    *syncutil.AdaptiveWaitGroup
 	lazyFixedSgInit = sync.OnceFunc(func() {
 		ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	})

pkg/js/compiler/pool.go

@@ -2,6 +2,7 @@ package compiler
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
@@ -55,10 +56,12 @@ var (
 	lazySgInit = sync.OnceFunc(func() {
 		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
-	sgResizeCheck = func() {
+	sgResizeCheck = func(ctx context.Context) {
 		// resize check point
 		if pooljsc.Size != PoolingJsVmConcurrency {
-			pooljsc.Resize(PoolingJsVmConcurrency)
+			if err := pooljsc.Resize(ctx, PoolingJsVmConcurrency); err != nil {
+				gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+			}
 		}
 	}
 )
@@ -122,7 +125,7 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute
 	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
 	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
 	lazySgInit()
-	sgResizeCheck()
+	sgResizeCheck(opts.Context)
 
 	pooljsc.Add()
 	defer pooljsc.Done()

pkg/protocols/common/automaticscan/automaticscan.go

@@ -56,7 +56,6 @@ type Service struct {
 	engine             *core.Engine
 	target             provider.InputProvider
 	wappalyzer         *wappalyzer.Wappalyze
-	childExecuter      *core.ChildExecuter
 	httpclient         *retryablehttp.Client
 	templateDirs       []string // root Template Directories
 	technologyMappings map[string]string
@@ -95,7 +94,6 @@ func New(opts Options) (*Service, error) {
 		return nil, err
 	}
 
-	childExecuter := opts.Engine.ChildExecuter()
 	httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{
 		Connection: &httpclientpool.ConnectionConfiguration{
 			DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options),
@@ -111,7 +109,6 @@ func New(opts Options) (*Service, error) {
 		target:             opts.Target,
 		wappalyzer:         wappalyzer,
 		templateDirs:       templateDirs, // fix this
-		childExecuter:      childExecuter,
 		httpclient:         httpclient,
 		technologyMappings: mappingData,
 		techTemplates:      techDetectTemplates,

pkg/protocols/common/protocolstate/memguardian.go

@@ -1,6 +1,7 @@
 package protocolstate
 
 import (
+	"context"
 	"sync"
 	"time"
 
@@ -14,20 +15,27 @@ var (
 	MaxThreadsOnLowMemory          = env.GetEnvOrDefault("MEMGUARDIAN_THREADS", 0)
 	MaxBytesBufferAllocOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_ALLOC", 0)
 	memTimer                       *time.Ticker
+	cancelFunc                     context.CancelFunc
 )
 
-func StartActiveMemGuardian() {
+func StartActiveMemGuardian(ctx context.Context) {
 	if memguardian.DefaultMemGuardian == nil {
 		return
 	}
 
-	memTimer := time.NewTicker(memguardian.DefaultInterval)
+	memTimer = time.NewTicker(memguardian.DefaultInterval)
+	ctx, cancelFunc = context.WithCancel(ctx)
 	go func() {
-		for range memTimer.C {
+		for {
-			if IsLowOnMemory() {
+			select {
-				_ = GlobalGuardBytesBufferAlloc()
+			case <-ctx.Done():
-			} else {
+				return
-				GlobalRestoreBytesBufferAlloc()
+			case <-memTimer.C:
+				if IsLowOnMemory() {
+					_ = GlobalGuardBytesBufferAlloc()
+				} else {
+					GlobalRestoreBytesBufferAlloc()
+				}
 			}
 		}
 	}()
@@ -38,7 +46,10 @@ func StopActiveMemGuardian() {
 		return
 	}
 
-	memTimer.Stop()
+	if memTimer != nil {
+		memTimer.Stop()
+		cancelFunc()
+	}
 }
 
 func IsLowOnMemory() bool {

pkg/protocols/common/protocolstate/state.go

@@ -148,7 +148,7 @@ func Init(options *types.Options) error {
 		return Dialer.Dial(ctx, "tcp", addr)
 	})
 
-	StartActiveMemGuardian()
+	StartActiveMemGuardian(context.Background())
 
 	return nil
 }

pkg/protocols/dns/request.go

@@ -88,7 +88,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
+				if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+					return err
+				}
 			}
 
 			value = generators.MergeMaps(vars, value)

pkg/protocols/http/httputils/spm.go

@@ -181,10 +181,11 @@ func (h *StopAtFirstMatchHandler[T]) Release() {
 	}
 }
 
-func (h *StopAtFirstMatchHandler[T]) Resize(size int) {
+func (h *StopAtFirstMatchHandler[T]) Resize(ctx context.Context, size int) error {
 	if h.sgPool.Size != size {
-		h.sgPool.Resize(size)
+		return h.sgPool.Resize(ctx, size)
 	}
+	return nil
 }
 
 func (h *StopAtFirstMatchHandler[T]) Size() int {

pkg/protocols/http/request.go

@@ -243,7 +243,9 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 
 		// resize check point - nop if there are no changes
 		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
-			spmHandler.Resize(request.options.Options.PayloadConcurrency)
+			if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+				return err
+			}
 		}
 
 		// break if stop at first match is found or host is unresponsive

pkg/protocols/http/request_fuzz.go

@@ -24,6 +24,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -89,6 +90,9 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 	if err != nil {
 		return errors.Wrap(err, "fuzz: could not build request from url")
 	}
+	userAgent := useragent.PickRandom()
+	baseRequest.Header.Set("User-Agent", userAgent.Raw)
+
 	// execute with one value first to checks its applicability
 	err = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback)
 	if err != nil {

pkg/protocols/javascript/js.go

@@ -434,7 +434,9 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
-				sg.Resize(request.options.Options.PayloadConcurrency)
+				if err := sg.Resize(ctxParent, request.options.Options.PayloadConcurrency); err != nil {
+					gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+				}
 			}
 
 			sg.Add()

pkg/protocols/network/request.go

@@ -200,7 +200,11 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
+				if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+					m.Lock()
+					multiErr = multierr.Append(multiErr, err)
+					m.Unlock()
+				}
 			}
 
 			value = generators.MergeMaps(value, payloads)

pkg/utils/http_probe.go

@@ -43,6 +43,13 @@ func (i *inputLivenessChecker) ProbeURL(input string) (string, error) {
 	return ProbeURL(input, i.client), nil
 }
 
+func (i *inputLivenessChecker) Close() error {
+	if i.client.Dialer != nil {
+		i.client.Dialer.Close()
+	}
+	return nil
+}
+
 // GetInputLivenessChecker returns a new input liveness checker using provided httpx client
 func GetInputLivenessChecker(client *httpx.HTTPX) types.InputLivenessProbe {
 	x := &inputLivenessChecker{client: client}