External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 19:35:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

testing fastdialer native

Browse Source
This commit is contained in:
Mzack9999 2024-06-11 12:21:43 +02:00
parent fd2311d034
commit e13c7cc967
4 changed files with 32 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
go.mod
View File

@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.1.0
github.com/projectdiscovery/hmap v0.0.45
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03
github.com/projectdiscovery/hmap v0.0.46
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.51
github.com/projectdiscovery/retryabledns v1.0.60
github.com/projectdiscovery/retryablehttp-go v1.0.62
github.com/projectdiscovery/retryabledns v1.0.63
github.com/projectdiscovery/retryablehttp-go v1.0.63
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.54
github.com/projectdiscovery/utils v0.1.0
github.com/projectdiscovery/utils v0.1.3
github.com/projectdiscovery/wappalyzergo v0.1.1
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1

8
go.sum
View File

@ -834,6 +834,10 @@ github.com/projectdiscovery/dsl v0.1.0 h1:10xoL0V/btU646xH6ioGXKwrGt6grT+BIa6id1
github.com/projectdiscovery/dsl v0.1.0/go.mod h1:JgOjLk54Tl9HG98N2SM5OaKKJ/STpI7yDiKW5rhV+yg=
github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59 h1:gHx3QQGu7My6SyN9PExd+fZ6XqnBfIIOazTlGz6wwts=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03 h1:+wi5E9edG2M6OznZjnMtuGgRwaHx7AckVPu5zNCATE0=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@ -850,6 +854,7 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvHA3Gdr6U=
github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk=
github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
@ -872,8 +877,10 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU=
github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@ -886,6 +893,7 @@ github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQ
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ=
github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU=
github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

37
pkg/protocols/common/hosterrorscache/hosterrorscache.go
View File

@ -1,6 +1,7 @@
package hosterrorscache
import (
"log"
"net"
"net/url"
"regexp"
@ -8,7 +9,7 @@ import (
"sync"
"sync/atomic"
"github.com/bluele/gcache"
"github.com/Mzack9999/gcache"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
@ -37,7 +38,7 @@ var (
type Cache struct {
MaxHostError int
verbose bool
failedTargets gcache.Cache
failedTargets gcache.Cache[string, *cacheItem]
TrackError []string
}
@ -52,7 +53,7 @@ const DefaultMaxHostsCount = 10000
// New returns a new host max errors cache
func New(maxHostError, maxHostsCount int, trackError []string) *Cache {
gc := gcache.New(maxHostsCount).
gc := gcache.New[string, *cacheItem](maxHostsCount).
ARC().
Build()
return &Cache{failedTargets: gc, MaxHostError: maxHostError, TrackError: trackError}
@ -67,11 +68,7 @@ func (c *Cache) SetVerbose(verbose bool) {
func (c *Cache) Close() {
if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) {
items := c.failedTargets.GetALL(false)
for k, v := range items {
val, ok := v.(*cacheItem)
if !ok {
continue
}
for k, val := range items {
gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load())
}
}
@ -115,16 +112,15 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
if err != nil {
return false
}
existingCacheItemValue := existingCacheItem.(*cacheItem)
if existingCacheItemValue.isPermanentErr {
if existingCacheItem.isPermanentErr {
// skipping permanent errors is expected so verbose instead of info
gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause)
gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItem.cause)
return true
}
if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) {
existingCacheItemValue.Do(func() {
gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItemValue.errors.Load())
if existingCacheItem.errors.Load() >= int32(c.MaxHostError) {
existingCacheItem.Do(func() {
gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItem.errors.Load())
})
return true
}
@ -133,6 +129,7 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
// MarkFailed marks a host as failed previously
func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
log.Printf("%#+v", err)
if !c.checkError(err) {
return
}
@ -151,9 +148,8 @@ func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
_ = c.failedTargets.Set(finalValue, newItem)
return
}
existingCacheItemValue := existingCacheItem.(*cacheItem)
existingCacheItemValue.errors.Add(1)
_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
existingCacheItem.errors.Add(1)
_ = c.failedTargets.Set(finalValue, existingCacheItem)
}
// GetKeyFromContext returns the key for the cache from the context
@ -186,10 +182,11 @@ func (c *Cache) checkError(err error) bool {
return false
}
// todo: fixed in fastdialer - ref: https://github.com/projectdiscovery/fastdialer/pull/301
// we do not consider timeouts as temporary
if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
return true
}
// if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
// return true
// }
kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
switch kind {

8
pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
View File

@ -91,9 +91,7 @@ func TestCacheMarkFailed(t *testing.T) {
require.Nil(t, err)
require.NotNil(t, failedTarget)
value, ok := failedTarget.(*cacheItem)
require.True(t, ok)
require.EqualValues(t, test.expected, value.errors.Load())
require.EqualValues(t, test.expected, failedTarget.errors.Load())
}
}
@ -138,9 +136,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
require.Nil(t, err)
require.NotNil(t, failedTarget)
value, ok := failedTarget.(*cacheItem)
require.True(t, ok)
require.EqualValues(t, test.expected, value.errors.Load())
require.EqualValues(t, test.expected, failedTarget.errors.Load())
}
}