testing fastdialer native

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.1.0
+	github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03
-	github.com/projectdiscovery/hmap v0.0.45
+	github.com/projectdiscovery/hmap v0.0.46
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.51
-	github.com/projectdiscovery/retryabledns v1.0.60
+	github.com/projectdiscovery/retryabledns v1.0.63
-	github.com/projectdiscovery/retryablehttp-go v1.0.62
+	github.com/projectdiscovery/retryablehttp-go v1.0.63
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.8
 	github.com/projectdiscovery/useragent v0.0.54
-	github.com/projectdiscovery/utils v0.1.0
+	github.com/projectdiscovery/utils v0.1.3
 	github.com/projectdiscovery/wappalyzergo v0.1.1
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1

go.sum

@@ -834,6 +834,10 @@ github.com/projectdiscovery/dsl v0.1.0 h1:10xoL0V/btU646xH6ioGXKwrGt6grT+BIa6id1
 github.com/projectdiscovery/dsl v0.1.0/go.mod h1:JgOjLk54Tl9HG98N2SM5OaKKJ/STpI7yDiKW5rhV+yg=
 github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
 github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59 h1:gHx3QQGu7My6SyN9PExd+fZ6XqnBfIIOazTlGz6wwts=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03 h1:+wi5E9edG2M6OznZjnMtuGgRwaHx7AckVPu5zNCATE0=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -850,6 +854,7 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c
 github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
 github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvHA3Gdr6U=
 github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk=
+github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
 github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
 github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
 github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
@@ -872,8 +877,10 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
 github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU=
+github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
 github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
 github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
+github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -886,6 +893,7 @@ github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQ
 github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
 github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
 github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
+github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ=
 github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU=
 github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -1,6 +1,7 @@
 package hosterrorscache
 
 import (
+	"log"
 	"net"
 	"net/url"
 	"regexp"
@@ -8,7 +9,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/bluele/gcache"
+	"github.com/Mzack9999/gcache"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
@@ -37,7 +38,7 @@ var (
 type Cache struct {
 	MaxHostError  int
 	verbose       bool
-	failedTargets gcache.Cache
+	failedTargets gcache.Cache[string, *cacheItem]
 	TrackError    []string
 }
 
@@ -52,7 +53,7 @@ const DefaultMaxHostsCount = 10000
 
 // New returns a new host max errors cache
 func New(maxHostError, maxHostsCount int, trackError []string) *Cache {
-	gc := gcache.New(maxHostsCount).
+	gc := gcache.New[string, *cacheItem](maxHostsCount).
 		ARC().
 		Build()
 	return &Cache{failedTargets: gc, MaxHostError: maxHostError, TrackError: trackError}
@@ -67,11 +68,7 @@ func (c *Cache) SetVerbose(verbose bool) {
 func (c *Cache) Close() {
 	if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) {
 		items := c.failedTargets.GetALL(false)
-		for k, v := range items {
+		for k, val := range items {
-			val, ok := v.(*cacheItem)
-			if !ok {
-				continue
-			}
 			gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load())
 		}
 	}
@@ -115,16 +112,15 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
 	if err != nil {
 		return false
 	}
-	existingCacheItemValue := existingCacheItem.(*cacheItem)
+	if existingCacheItem.isPermanentErr {
-	if existingCacheItemValue.isPermanentErr {
 		// skipping permanent errors is expected so verbose instead of info
-		gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause)
+		gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItem.cause)
 		return true
 	}
 
-	if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) {
+	if existingCacheItem.errors.Load() >= int32(c.MaxHostError) {
-		existingCacheItemValue.Do(func() {
+		existingCacheItem.Do(func() {
-			gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItemValue.errors.Load())
+			gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItem.errors.Load())
 		})
 		return true
 	}
@@ -133,6 +129,7 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
 
 // MarkFailed marks a host as failed previously
 func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
+	log.Printf("%#+v", err)
 	if !c.checkError(err) {
 		return
 	}
@@ -151,9 +148,8 @@ func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
 		_ = c.failedTargets.Set(finalValue, newItem)
 		return
 	}
-	existingCacheItemValue := existingCacheItem.(*cacheItem)
+	existingCacheItem.errors.Add(1)
-	existingCacheItemValue.errors.Add(1)
+	_ = c.failedTargets.Set(finalValue, existingCacheItem)
-	_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
 }
 
 // GetKeyFromContext returns the key for the cache from the context
@@ -186,10 +182,11 @@ func (c *Cache) checkError(err error) bool {
 		return false
 	}
 
+	// todo: fixed in fastdialer - ref: https://github.com/projectdiscovery/fastdialer/pull/301
 	// we do not consider timeouts as temporary
-	if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
+	// if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
-		return true
+	// 	return true
-	}
+	// }
 
 	kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
 	switch kind {

pkg/protocols/common/hosterrorscache/hosterrorscache_test.go

@@ -91,9 +91,7 @@ func TestCacheMarkFailed(t *testing.T) {
 		require.Nil(t, err)
 		require.NotNil(t, failedTarget)
 
-		value, ok := failedTarget.(*cacheItem)
+		require.EqualValues(t, test.expected, failedTarget.errors.Load())
-		require.True(t, ok)
-		require.EqualValues(t, test.expected, value.errors.Load())
 	}
 }
 
@@ -138,9 +136,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
 		require.Nil(t, err)
 		require.NotNil(t, failedTarget)
 
-		value, ok := failedTarget.(*cacheItem)
+		require.EqualValues(t, test.expected, failedTarget.errors.Load())
-		require.True(t, ok)
-		require.EqualValues(t, test.expected, value.errors.Load())
 	}
 }