nuclei/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go

package hosterrorscache

import (
	"net"
	"net/url"
	"regexp"
	"strings"

	"github.com/bluele/gcache"
	"github.com/projectdiscovery/gologger"
)

// Cache is a cache for host based errors. It allows skipping
// certain hosts based on an error threshold.
//
// It uses an LRU cache internally for skipping unresponsive hosts
// that remain so for a duration.
type Cache struct {
	hostMaxErrors int
	verbose       bool
	failedTargets gcache.Cache
}

const DefaultMaxHostsCount = 10000

// New returns a new host max errors cache
func New(hostMaxErrors, maxHostsCount int) *Cache {
	gc := gcache.New(maxHostsCount).
		ARC().
		Build()
	return &Cache{failedTargets: gc, hostMaxErrors: hostMaxErrors}
}

// SetVerbose sets the cache to log at verbose level
func (c *Cache) SetVerbose(verbose bool) *Cache {
	c.verbose = verbose
	return c
}

// Close closes the host errors cache
func (c *Cache) Close() {
	c.failedTargets.Purge()
}

func (c *Cache) normalizeCacheValue(value string) string {
	finalValue := value
	if strings.HasPrefix(value, "http") {
		if parsed, err := url.Parse(value); err == nil {

			hostname := parsed.Host
			finalPort := parsed.Port()
			if finalPort == "" {
				if parsed.Scheme == "https" {
					finalPort = "443"
				} else {
					finalPort = "80"
				}
				hostname = net.JoinHostPort(parsed.Host, finalPort)
			}
			finalValue = hostname
		}
	}
	return finalValue
}

// ErrUnresponsiveHost is returned when a host is unresponsive
//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive")

// Check returns true if a host should be skipped as it has been
// unresponsive for a certain number of times.
//
// The value can be many formats -
//   - URL: https?:// type
//   - Host:port type
//   - host type
func (c *Cache) Check(value string) bool {
	finalValue := c.normalizeCacheValue(value)
	if !c.failedTargets.Has(finalValue) {
		return false
	}

	numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
	if err != nil {
		return false
	}
	numberOfErrorsValue := numberOfErrors.(int)

	if numberOfErrors == -1 {
		return true
	}
	if numberOfErrorsValue >= c.hostMaxErrors {
		_ = c.failedTargets.Set(finalValue, -1)
		if c.verbose {
			gologger.Verbose().Msgf("Skipping %s as previously unresponsive %d times", finalValue, numberOfErrorsValue)
		}
		return true
	}
	return false
}

// MarkFailed marks a host as failed previously
func (c *Cache) MarkFailed(value string) {
	finalValue := c.normalizeCacheValue(value)
	if !c.failedTargets.Has(finalValue) {
		_ = c.failedTargets.Set(finalValue, 1)
		return
	}

	numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
	if err != nil || numberOfErrors == nil {
		_ = c.failedTargets.Set(finalValue, 1)
		return
	}
	numberOfErrorsValue := numberOfErrors.(int)

	_ = c.failedTargets.Set(finalValue, numberOfErrorsValue+1)
}

var checkErrorRegexp = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host)`)

// CheckError checks if an error represents a type that should be
// added to the host skipping table.
func (c *Cache) CheckError(err error) bool {
	errString := err.Error()
	return checkErrorRegexp.MatchString(errString)
}