diff --git a/go.mod b/go.mod index 07c36e7d9..c5e72d5dd 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.1.0 - github.com/projectdiscovery/hmap v0.0.45 + github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03 + github.com/projectdiscovery/hmap v0.0.46 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.51 - github.com/projectdiscovery/retryabledns v1.0.60 - github.com/projectdiscovery/retryablehttp-go v1.0.62 + github.com/projectdiscovery/retryabledns v1.0.63 + github.com/projectdiscovery/retryablehttp-go v1.0.63 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -94,7 +94,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.8 github.com/projectdiscovery/useragent v0.0.54 - github.com/projectdiscovery/utils v0.1.0 + github.com/projectdiscovery/utils v0.1.3 github.com/projectdiscovery/wappalyzergo v0.1.1 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index ccdec0fca..394f5fc48 100644 --- a/go.sum +++ b/go.sum @@ -834,6 +834,10 @@ github.com/projectdiscovery/dsl v0.1.0 h1:10xoL0V/btU646xH6ioGXKwrGt6grT+BIa6id1 github.com/projectdiscovery/dsl v0.1.0/go.mod h1:JgOjLk54Tl9HG98N2SM5OaKKJ/STpI7yDiKW5rhV+yg= github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= +github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59 h1:gHx3QQGu7My6SyN9PExd+fZ6XqnBfIIOazTlGz6wwts= +github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54= +github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03 h1:+wi5E9edG2M6OznZjnMtuGgRwaHx7AckVPu5zNCATE0= +github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -850,6 +854,7 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvHA3Gdr6U= github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk= +github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8= github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg= github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ= github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc= @@ -872,8 +877,10 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ= github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU= +github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0= github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY= github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE= +github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -886,6 +893,7 @@ github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQ github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM= github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= +github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ= github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU= github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index badff95dd..7eaf147f4 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -1,6 +1,7 @@ package hosterrorscache import ( + "log" "net" "net/url" "regexp" @@ -8,7 +9,7 @@ import ( "sync" "sync/atomic" - "github.com/bluele/gcache" + "github.com/Mzack9999/gcache" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" @@ -37,7 +38,7 @@ var ( type Cache struct { MaxHostError int verbose bool - failedTargets gcache.Cache + failedTargets gcache.Cache[string, *cacheItem] TrackError []string } @@ -52,7 +53,7 @@ const DefaultMaxHostsCount = 10000 // New returns a new host max errors cache func New(maxHostError, maxHostsCount int, trackError []string) *Cache { - gc := gcache.New(maxHostsCount). + gc := gcache.New[string, *cacheItem](maxHostsCount). ARC(). Build() return &Cache{failedTargets: gc, MaxHostError: maxHostError, TrackError: trackError} @@ -67,11 +68,7 @@ func (c *Cache) SetVerbose(verbose bool) { func (c *Cache) Close() { if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) { items := c.failedTargets.GetALL(false) - for k, v := range items { - val, ok := v.(*cacheItem) - if !ok { - continue - } + for k, val := range items { gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load()) } } @@ -115,16 +112,15 @@ func (c *Cache) Check(ctx *contextargs.Context) bool { if err != nil { return false } - existingCacheItemValue := existingCacheItem.(*cacheItem) - if existingCacheItemValue.isPermanentErr { + if existingCacheItem.isPermanentErr { // skipping permanent errors is expected so verbose instead of info - gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause) + gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItem.cause) return true } - if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) { - existingCacheItemValue.Do(func() { - gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItemValue.errors.Load()) + if existingCacheItem.errors.Load() >= int32(c.MaxHostError) { + existingCacheItem.Do(func() { + gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItem.errors.Load()) }) return true } @@ -133,6 +129,7 @@ func (c *Cache) Check(ctx *contextargs.Context) bool { // MarkFailed marks a host as failed previously func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) { + log.Printf("%#+v", err) if !c.checkError(err) { return } @@ -151,9 +148,8 @@ func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) { _ = c.failedTargets.Set(finalValue, newItem) return } - existingCacheItemValue := existingCacheItem.(*cacheItem) - existingCacheItemValue.errors.Add(1) - _ = c.failedTargets.Set(finalValue, existingCacheItemValue) + existingCacheItem.errors.Add(1) + _ = c.failedTargets.Set(finalValue, existingCacheItem) } // GetKeyFromContext returns the key for the cache from the context @@ -186,10 +182,11 @@ func (c *Cache) checkError(err error) bool { return false } + // todo: fixed in fastdialer - ref: https://github.com/projectdiscovery/fastdialer/pull/301 // we do not consider timeouts as temporary - if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") { - return true - } + // if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") { + // return true + // } kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic) switch kind { diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index 6fd04b475..112690d87 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -91,9 +91,7 @@ func TestCacheMarkFailed(t *testing.T) { require.Nil(t, err) require.NotNil(t, failedTarget) - value, ok := failedTarget.(*cacheItem) - require.True(t, ok) - require.EqualValues(t, test.expected, value.errors.Load()) + require.EqualValues(t, test.expected, failedTarget.errors.Load()) } } @@ -138,9 +136,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { require.Nil(t, err) require.NotNil(t, failedTarget) - value, ok := failedTarget.(*cacheItem) - require.True(t, ok) - require.EqualValues(t, test.expected, value.errors.Load()) + require.EqualValues(t, test.expected, failedTarget.errors.Load()) } }