External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 03:45:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(runner): adjust max-host-error if gt concurrency (#5633)

Browse Source 
* refactor(common): use `ParseRequestURI` instead when `NormalizeCacheValue`

also it exports the method

Signed-off-by: Dwi Siswanto <git@dw1.io>

* refactor(runner): adjust `max-host-error` if gt `concurrency`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* fix lint

* chore(runner): expose adjusted `max-host-error` value

Signed-off-by: Dwi Siswanto <git@dw1.io>

---------

Signed-off-by: Dwi Siswanto <git@dw1.io>
Co-authored-by: Doğan Can Bakır <dogancanbakir@protonmail.com>
This commit is contained in:
Dwi Siswanto 2024-09-23 17:27:30 +07:00 committed by GitHub
parent a118daa375
commit 9983d7415c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 38 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/runner/runner.go
View File

@ -501,8 +501,17 @@ func (r *Runner) RunEnumeration() error {
}
if r.options.ShouldUseHostError() {
cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError)
maxHostError := r.options.MaxHostError
if r.options.TemplateThreads > maxHostError {
gologger.Print().Msgf("[%v] The concurrency value is higher than max-host-error", r.colorizer.BrightYellow("WRN"))
gologger.Info().Msgf("Adjusting max-host-error to the concurrency value: %d", r.options.TemplateThreads)
maxHostError = r.options.TemplateThreads
}
cache := hosterrorscache.New(maxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError)
cache.SetVerbose(r.options.Verbose)
r.hostErrors = cache
executorOpts.HostErrorsCache = cache
}

42
pkg/protocols/common/hosterrorscache/hosterrorscache.go
View File

@ -75,24 +75,34 @@ func (c *Cache) Close() {
c.failedTargets.Purge()
}
func (c *Cache) normalizeCacheValue(value string) string {
finalValue := value
if strings.HasPrefix(value, "http") {
if parsed, err := url.Parse(value); err == nil {
hostname := parsed.Host
finalPort := parsed.Port()
if finalPort == "" {
if parsed.Scheme == "https" {
finalPort = "443"
} else {
finalPort = "80"
}
hostname = net.JoinHostPort(parsed.Host, finalPort)
// NormalizeCacheValue processes the input value and returns a normalized cache
// value.
func (c *Cache) NormalizeCacheValue(value string) string {
var normalizedValue string = value
u, err := url.ParseRequestURI(value)
if err != nil || u.Host == "" {
u, err2 := url.ParseRequestURI("https://" + value)
if err2 != nil {
return normalizedValue
}
normalizedValue = u.Host
} else {
port := u.Port()
if port == "" {
switch u.Scheme {
case "https":
normalizedValue = net.JoinHostPort(u.Host, "443")
case "http":
normalizedValue = net.JoinHostPort(u.Host, "80")
}
finalValue = hostname
} else {
normalizedValue = u.Host
}
}
return finalValue
return normalizedValue
}
// ErrUnresponsiveHost is returned when a host is unresponsive
@ -166,7 +176,7 @@ func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string {
address = tmp.String()
}
}
finalValue := c.normalizeCacheValue(address)
finalValue := c.NormalizeCacheValue(address)
return finalValue
}

4
pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
View File

@ -109,7 +109,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
// the cache is not atomic during items creation, so we pre-create them with counter to zero
for _, test := range tests {
normalizedValue := cache.normalizeCacheValue(test.host)
normalizedValue := cache.NormalizeCacheValue(test.host)
newItem := &cacheItem{errors: atomic.Int32{}}
newItem.errors.Store(0)
_ = cache.failedTargets.Set(normalizedValue, newItem)
@ -131,7 +131,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
for _, test := range tests {
require.True(t, cache.Check(newCtxArgs(test.host)))
normalizedCacheValue := cache.normalizeCacheValue(test.host)
normalizedCacheValue := cache.NormalizeCacheValue(test.host)
failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
require.Nil(t, err)
require.NotNil(t, failedTarget)