Tarun Koyalwar 23bd0336fb
multiple bug fixes + performance improvements (#5148)
* prototype errkit

* complete errkit implementation

* add cause to all timeouts

* fix request timeout annotation @timeout

* increase responseHeaderTimeout to 8 for stability

* rawhttp error related improvements

* feat: add port status caching

* add port status caching to http

* migrate to new utils/errkit

* remote dialinterface + error cause

* debug dir support using .gitignore debug-*

* make nuclei easy to debug

* debug dir update .gitignore

* temp change (to revert)

* Revert "temp change (to revert)"

This reverts commit d3131f777713b9f80e2275142e80f36340a76d36.

* use available context instead of new one

* bump fastdialer

* fix hosterrorscache + misc improvements

* add 'address' field in error log

* fix js vague errors + pgwrap driver

* fix max host error + misc updates

* update tests as per changes

* fix request annotation context

* remove closed dialer reference

* fix sdk panic issue

* bump retryablehttp-go,utils,fastdialer

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
2024-05-25 00:29:04 +05:30

217 lines
5.5 KiB
Go

package protocolstate
import (
"context"
"fmt"
"net"
"net/url"
"github.com/go-sql-driver/mysql"
"github.com/pkg/errors"
"golang.org/x/net/proxy"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/mapcidr/asn"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
)
// Dialer is a shared fastdialer instance for host DNS resolution
var (
Dialer *fastdialer.Dialer
)
// Init creates the Dialer instance based on user configuration
func Init(options *types.Options) error {
if Dialer != nil {
return nil
}
lfaAllowed = options.AllowLocalFileAccess
opts := fastdialer.DefaultOptions
if options.DialerTimeout > 0 {
opts.DialerTimeout = options.DialerTimeout
}
if options.DialerKeepAlive > 0 {
opts.DialerKeepAlive = options.DialerKeepAlive
}
var expandedDenyList []string
for _, excludeTarget := range options.ExcludeTargets {
switch {
case asn.IsASN(excludeTarget):
expandedDenyList = append(expandedDenyList, expand.ASN(excludeTarget)...)
default:
expandedDenyList = append(expandedDenyList, excludeTarget)
}
}
if options.RestrictLocalNetworkAccess {
expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv4DenylistRanges...)
expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv6DenylistRanges...)
}
npOptions := &networkpolicy.Options{
DenyList: expandedDenyList,
}
opts.WithNetworkPolicyOptions = npOptions
NetworkPolicy, _ = networkpolicy.New(*npOptions)
InitHeadless(options.AllowLocalFileAccess, NetworkPolicy)
switch {
case options.SourceIP != "" && options.Interface != "":
isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, options.Interface)
if err != nil {
return err
}
if isAssociated {
opts.Dialer = &net.Dialer{
LocalAddr: &net.TCPAddr{
IP: net.ParseIP(options.SourceIP),
},
}
} else {
return fmt.Errorf("source ip (%s) is not associated with the interface (%s)", options.SourceIP, options.Interface)
}
case options.SourceIP != "":
isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, "any")
if err != nil {
return err
}
if isAssociated {
opts.Dialer = &net.Dialer{
LocalAddr: &net.TCPAddr{
IP: net.ParseIP(options.SourceIP),
},
}
} else {
return fmt.Errorf("source ip (%s) is not associated with any network interface", options.SourceIP)
}
case options.Interface != "":
ifadrr, err := interfaceAddress(options.Interface)
if err != nil {
return err
}
opts.Dialer = &net.Dialer{
LocalAddr: &net.TCPAddr{
IP: ifadrr,
},
}
}
if types.ProxySocksURL != "" {
proxyURL, err := url.Parse(types.ProxySocksURL)
if err != nil {
return err
}
var forward *net.Dialer
if opts.Dialer != nil {
forward = opts.Dialer
} else {
forward = &net.Dialer{
Timeout: opts.DialerTimeout,
KeepAlive: opts.DialerKeepAlive,
DualStack: true,
}
}
dialer, err := proxy.FromURL(proxyURL, forward)
if err != nil {
return err
}
opts.ProxyDialer = &dialer
}
if options.SystemResolvers {
opts.ResolversFile = true
opts.EnableFallback = true
}
if len(options.InternalResolversList) > 0 {
opts.BaseResolvers = options.InternalResolversList
}
opts.Deny = append(opts.Deny, expandedDenyList...)
opts.WithDialerHistory = true
opts.SNIName = options.SNI
// this instance is used in javascript protocol libraries and
// dial history is required to get dialed ip of a host
opts.WithDialerHistory = true
// fastdialer now by default fallbacks to ztls when there are tls related errors
dialer, err := fastdialer.NewDialer(opts)
if err != nil {
return errors.Wrap(err, "could not create dialer")
}
Dialer = dialer
// override dialer in mysql
mysql.RegisterDialContext("tcp", func(ctx context.Context, addr string) (net.Conn, error) {
return Dialer.Dial(ctx, "tcp", addr)
})
StartActiveMemGuardian(context.Background())
return nil
}
// isIpAssociatedWithInterface checks if the given IP is associated with the given interface.
func isIpAssociatedWithInterface(sourceIP, interfaceName string) (bool, error) {
addrs, err := interfaceAddresses(interfaceName)
if err != nil {
return false, err
}
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok {
if ipnet.IP.String() == sourceIP {
return true, nil
}
}
}
return false, nil
}
// interfaceAddress returns the first IPv4 address of the given interface.
func interfaceAddress(interfaceName string) (net.IP, error) {
addrs, err := interfaceAddresses(interfaceName)
if err != nil {
return nil, err
}
var address net.IP
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
address = ipnet.IP
break
}
}
}
if address == nil {
return nil, fmt.Errorf("no suitable address found for interface: `%s`", interfaceName)
}
return address, nil
}
// interfaceAddresses returns all interface addresses.
func interfaceAddresses(interfaceName string) ([]net.Addr, error) {
if interfaceName == "any" {
return net.InterfaceAddrs()
}
ief, err := net.InterfaceByName(interfaceName)
if err != nil {
return nil, errors.Wrapf(err, "failed to get interface: `%s`", interfaceName)
}
addrs, err := ief.Addrs()
if err != nil {
return nil, errors.Wrapf(err, "failed to get interface addresses for: `%s`", interfaceName)
}
return addrs, nil
}
// Close closes the global shared fastdialer
func Close() {
if Dialer != nil {
Dialer.Close()
Dialer = nil
}
StopActiveMemGuardian()
}