diff --git a/go.mod b/go.mod index ff3f7d43e..55b96ac5f 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/projectdiscovery/interactsh v1.2.4 github.com/projectdiscovery/rawhttp v0.1.90 github.com/projectdiscovery/retryabledns v1.0.99 - github.com/projectdiscovery/retryablehttp-go v1.0.110 + github.com/projectdiscovery/retryablehttp-go v1.0.111 github.com/projectdiscovery/yamldoc-go v1.0.6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.6.0 @@ -90,7 +90,7 @@ require ( github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/gologger v1.1.53 + github.com/projectdiscovery/gologger v1.1.54 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.3 github.com/projectdiscovery/httpx v1.7.0 @@ -103,7 +103,7 @@ require ( github.com/projectdiscovery/uncover v1.0.10 github.com/projectdiscovery/useragent v0.0.100 github.com/projectdiscovery/utils v0.4.18 - github.com/projectdiscovery/wappalyzergo v0.2.25 + github.com/projectdiscovery/wappalyzergo v0.2.27 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 @@ -203,7 +203,6 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mholt/archiver/v3 v3.5.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -222,7 +221,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/projectdiscovery/asnmap v1.1.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.15 // indirect + github.com/projectdiscovery/cdncheck v1.1.17 // indirect github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect @@ -362,10 +361,8 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/nwaples/rardecode v1.1.3 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index 18f5eb41f..324e90163 100644 --- a/go.sum +++ b/go.sum @@ -120,7 +120,6 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs= github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI= -github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= @@ -309,7 +308,6 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -474,7 +472,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -653,7 +650,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA= github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -661,7 +657,6 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= @@ -727,8 +722,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30= github.com/mholt/acmez v1.2.0/go.mod h1:VT9YwH1xgNX1kmYY89gY8xPJC84BFAisjo8Egigt4kE= -github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= -github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= @@ -782,9 +775,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= -github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode/v2 v2.0.1 h1:3MN6/R+Y4c7e+21U3yhWuUcf72sYmcmr6jtiuAVSH1A= github.com/nwaples/rardecode/v2 v2.0.1/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= @@ -829,7 +819,6 @@ github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= @@ -852,8 +841,8 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.15 h1:rRs3LW2MP7V8QeONVRYce6RhDcWp83O+AWmt+QQ4mBM= -github.com/projectdiscovery/cdncheck v1.1.15/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.17 h1:YSqKk05+UGSxmPIp7tlCvRegF63FUqO+mA2Wl/Je3gA= +github.com/projectdiscovery/cdncheck v1.1.17/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.4.2 h1:9PnD6EyDAZFvpQmJ0700gkQ96Fqlzl+lnTdcVHAagXI= @@ -868,8 +857,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.53 h1:Er5nty/kifUDSr9MLgi8pzr0bveC+jco76Ittlg/AlM= -github.com/projectdiscovery/gologger v1.1.53/go.mod h1:PLaWBQIjfIaSAfAVAJ3MZctIyStGcI3CpaN7NLIejo8= +github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= +github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.3 h1:tsYkrSvWw4WdIUJyisd4MB1vRiw1X57TuVVk3p8Z3G8= @@ -898,8 +887,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.99 h1:DJ6TewgkwqJozDOPXhoOy/cdtuzJzbIQ/BFfWNYzHpw= github.com/projectdiscovery/retryabledns v1.0.99/go.mod h1:pkPYuqtxhX6z1pYL+O6s4Lg7ubINt2jg12gsaW9O3kY= -github.com/projectdiscovery/retryablehttp-go v1.0.110 h1:qO3rkuoG4/N9KlxmfnpmdCAU2JDTq6EQzwUzvVaER/o= -github.com/projectdiscovery/retryablehttp-go v1.0.110/go.mod h1:kM6zKIqV0uRly7aTImm+3v7avRR3L2zrmgAEMNfxlbo= +github.com/projectdiscovery/retryablehttp-go v1.0.111 h1:HzkVN0IyC0RfVylBlgNoqaQgVvuUEvbDEVbpGyD/Y9M= +github.com/projectdiscovery/retryablehttp-go v1.0.111/go.mod h1:Tl6noELU9RpjwywMDw722HB038QXohl90g2ZtKSqbCI= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -912,8 +901,8 @@ github.com/projectdiscovery/useragent v0.0.100 h1:gDZSgPQCP8D0XUny41Ch4urP+FK5Oc github.com/projectdiscovery/useragent v0.0.100/go.mod h1:8je9oUPzT5R+gjKQNEFurDSvX7fCnqW2iDGYdKMH6hY= github.com/projectdiscovery/utils v0.4.18 h1:cSjMOLXI5gAajfA6KV+0iQG4dGx2IHWLQyND/Snvw7k= github.com/projectdiscovery/utils v0.4.18/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= -github.com/projectdiscovery/wappalyzergo v0.2.25 h1:K56XmuMrEBowlu2WqSFJDkUju8DBACRKDJ8JUQrqpDk= -github.com/projectdiscovery/wappalyzergo v0.2.25/go.mod h1:F8X79ljvmvrG+EIxdxWS9VbdkVTsQupHYz+kXlp8O0o= +github.com/projectdiscovery/wappalyzergo v0.2.27 h1:u5z/3YohzwtX3n4EGDGy4XOKAXZx4JorNzZlw9CzOK0= +github.com/projectdiscovery/wappalyzergo v0.2.27/go.mod h1:F8X79ljvmvrG+EIxdxWS9VbdkVTsQupHYz+kXlp8O0o= github.com/projectdiscovery/yamldoc-go v1.0.6 h1:GCEdIRlQjDux28xTXKszM7n3jlMf152d5nqVpVoetas= github.com/projectdiscovery/yamldoc-go v1.0.6/go.mod h1:R5lWrNzP+7Oyn77NDVPnBsxx2/FyQZBBkIAaSaCQFxw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1080,7 +1069,6 @@ github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1116,8 +1104,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index e4817848b..79d861363 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -702,6 +702,7 @@ func (r *Runner) RunEnumeration() error { }() } + now := time.Now() enumeration := false var results *atomic.Bool results, err = r.runStandardEnumeration(executorOpts, store, executorEngine) @@ -725,11 +726,17 @@ func (r *Runner) RunEnumeration() error { } r.fuzzFrequencyCache.Close() + r.progress.Stop() + timeTaken := time.Since(now) // todo: error propagation without canonical straight error check is required by cloud? // use safe dereferencing to avoid potential panics in case of previous unchecked errors if v := ptrutil.Safe(results); !v.Load() { - gologger.Info().Msgf("No results found. Better luck next time!") + gologger.Info().Msgf("Scan completed in %s. No results found.", shortDur(timeTaken)) + } else { + matchCount := r.output.ResultCount() + gologger.Info().Msgf("Scan completed in %s. %d matches found.", shortDur(timeTaken), matchCount) } + // check if a passive scan was requested but no target was provided if r.options.OfflineHTTP && len(r.options.Targets) == 0 && r.options.TargetsFilePath == "" { return errors.Wrap(err, "missing required input (http response) to run passive templates") @@ -738,6 +745,24 @@ func (r *Runner) RunEnumeration() error { return err } +func shortDur(d time.Duration) string { + if d < time.Minute { + return d.String() + } + + // Truncate to the nearest minute + d = d.Truncate(time.Minute) + s := d.String() + + if strings.HasSuffix(s, "m0s") { + s = s[:len(s)-2] + } + if strings.HasSuffix(s, "h0m") { + s = s[:len(s)-2] + } + return s +} + func (r *Runner) isInputNonHTTP() bool { var nonURLInput bool r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool { diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go index aa47bc44f..fae26b456 100644 --- a/pkg/core/execute_options.go +++ b/pkg/core/execute_options.go @@ -110,6 +110,8 @@ func (e *Engine) executeTemplateSpray(ctx context.Context, templatesList []*temp defer wp.Wait() for _, template := range templatesList { + template := template + select { case <-ctx.Done(): return results diff --git a/pkg/output/multi_writer.go b/pkg/output/multi_writer.go index 8ea729b4b..17b1c725a 100644 --- a/pkg/output/multi_writer.go +++ b/pkg/output/multi_writer.go @@ -65,3 +65,13 @@ func (mw *MultiWriter) RequestStatsLog(statusCode, response string) { writer.RequestStatsLog(statusCode, response) } } + +func (mw *MultiWriter) ResultCount() int { + count := 0 + for _, writer := range mw.writers { + if count := writer.ResultCount(); count > 0 { + return count + } + } + return count +} diff --git a/pkg/output/output.go b/pkg/output/output.go index 5c84bed30..e85774b83 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -54,6 +54,8 @@ type Writer interface { RequestStatsLog(statusCode, response string) // WriteStoreDebugData writes the request/response debug data to file WriteStoreDebugData(host, templateID, eventType string, data string) + // ResultCount returns the total number of results written + ResultCount() int } // StandardWriter is a writer writing output to file and screen for results. @@ -79,6 +81,8 @@ type StandardWriter struct { // JSONLogRequestHook is a hook that can be used to log request/response // when using custom server code with output JSONLogRequestHook func(*JSONLogRequest) + + resultCount atomic.Int32 } var _ Writer = &StandardWriter{} @@ -287,6 +291,10 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { return writer, nil } +func (w *StandardWriter) ResultCount() int { + return int(w.resultCount.Load()) +} + // Write writes the event to file and/or screen. func (w *StandardWriter) Write(event *ResultEvent) error { // Enrich the result event with extra metadata on the template-path and url. @@ -336,6 +344,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error { _, _ = w.outputFile.Write([]byte("\n")) } } + w.resultCount.Add(1) return nil } diff --git a/pkg/output/output_stats.go b/pkg/output/output_stats.go index 7b0d509cd..68a234d85 100644 --- a/pkg/output/output_stats.go +++ b/pkg/output/output_stats.go @@ -49,3 +49,6 @@ func (tw *StatsOutputWriter) RequestStatsLog(statusCode, response string) { tw.Tracker.TrackStatusCode(statusCode) tw.Tracker.TrackWAFDetected(response) } +func (tw *StatsOutputWriter) ResultCount() int { + return 0 +} diff --git a/pkg/progress/progress.go b/pkg/progress/progress.go index 1ffb22cee..853fb103d 100644 --- a/pkg/progress/progress.go +++ b/pkg/progress/progress.go @@ -120,9 +120,7 @@ func (p *StatsTicker) IncrementRequests() { // SetRequests sets the counter by incrementing it with a delta func (p *StatsTicker) SetRequests(count uint64) { - value, _ := p.stats.GetCounter("requests") - delta := count - value - p.stats.IncrementCounter("requests", int(delta)) + p.stats.IncrementCounter("requests", int(count)) } // IncrementMatched increments the matched counter by 1. diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 3943eef7e..3039dbdf0 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -89,6 +89,9 @@ func (c *Cache) NormalizeCacheValue(value string) string { u, err := url.ParseRequestURI(value) if err != nil || u.Host == "" { + if strings.Contains(value, ":") { + return normalizedValue + } u, err2 := url.ParseRequestURI("https://" + value) if err2 != nil { return normalizedValue @@ -236,14 +239,19 @@ func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string { // should be reflected in contextargs but it is not yet reflected in some cases // and needs refactor of ScanContext + ContextArgs to achieve that // i.e why we use real address from error if present - address := ctx.MetaInput.Address() - // get address override from error + var address string + + // 1. the address carried inside the error (if the transport sets it) if err != nil { - tmp := errkit.GetAttrValue(err, "address") - if tmp.Any() != nil { - address = tmp.String() + if v := errkit.GetAttrValue(err, "address"); v.Any() != nil { + address = v.String() } } + + if address == "" { + address = ctx.MetaInput.Address() + } + finalValue := c.NormalizeCacheValue(address) return finalValue } diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go index 3cde12d88..1cb9553c9 100644 --- a/pkg/protocols/http/build_request.go +++ b/pkg/protocols/http/build_request.go @@ -123,14 +123,6 @@ func (g *generatedRequest) URL() string { return "" } -// Total returns the total number of requests for the generator -func (r *requestGenerator) Total() int { - if r.payloadIterator != nil { - return len(r.request.Raw) * r.payloadIterator.Remaining() - } - return len(r.request.Path) -} - // Make creates a http request for the provided input. // It returns ErrNoMoreRequests as error when all the requests have been exhausted. func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (gr *generatedRequest, err error) { diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index 78710f79c..0b30a7408 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -501,7 +501,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads) } } - return nil } @@ -517,24 +516,8 @@ func (request *Request) RebuildGenerator() error { // Requests returns the total number of requests the YAML rule will perform func (request *Request) Requests() int { - if request.generator != nil { - payloadRequests := request.generator.NewIterator().Total() - if len(request.Raw) > 0 { - payloadRequests = payloadRequests * len(request.Raw) - } - if len(request.Path) > 0 { - payloadRequests = payloadRequests * len(request.Path) - } - return payloadRequests - } - if len(request.Raw) > 0 { - requests := len(request.Raw) - if requests == 1 && request.RaceNumberRequests != 0 { - requests *= request.RaceNumberRequests - } - return requests - } - return len(request.Path) + generator := request.newGenerator(false) + return generator.Total() } const ( diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index cfa5c1d57..110adfbc0 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -41,7 +41,6 @@ import ( "github.com/projectdiscovery/rawhttp" convUtil "github.com/projectdiscovery/utils/conversion" "github.com/projectdiscovery/utils/errkit" - errorutil "github.com/projectdiscovery/utils/errors" httpUtils "github.com/projectdiscovery/utils/http" "github.com/projectdiscovery/utils/reader" sliceutil "github.com/projectdiscovery/utils/slice" @@ -484,7 +483,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if err == types.ErrNoMoreRequests { return true, nil } - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return true, err } // ideally if http template used a custom port or hostname @@ -541,14 +539,19 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if errors.Is(execReqErr, ErrMissingVars) { return true, nil } + if execReqErr != nil { + request.markHostError(updatedInput, execReqErr) + // if applicable mark the host as unresponsive - requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL()) + reqKitErr := errkit.FromError(execReqErr) + reqKitErr.Msgf("got err while executing %v", generatedHttpRequest.URL()) + + requestErr = reqKitErr request.options.Progress.IncrementFailedRequestsBy(1) } else { request.options.Progress.IncrementRequests() } - request.markHostError(updatedInput, execReqErr) // If this was a match, and we want to stop at first match, skip all further requests. shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch @@ -585,6 +588,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa requestErr = gotErr } if skip || gotErr != nil { + request.options.Progress.SetRequests(uint64(generator.Remaining() + 1)) break } } @@ -1216,7 +1220,7 @@ func (request *Request) newContext(input *contextargs.Context) context.Context { // markHostError checks if the error is a unreponsive host error and marks it func (request *Request) markHostError(input *contextargs.Context, err error) { - if request.options.HostErrorsCache != nil { + if request.options.HostErrorsCache != nil && err != nil { request.options.HostErrorsCache.MarkFailedOrRemove(request.options.ProtocolType.String(), input, err) } } diff --git a/pkg/protocols/http/request_generator.go b/pkg/protocols/http/request_generator.go index b15df1be9..4c4c701a8 100644 --- a/pkg/protocols/http/request_generator.go +++ b/pkg/protocols/http/request_generator.go @@ -135,3 +135,67 @@ func (r *requestGenerator) hasMarker(request string, mark flowMark) bool { fo, hasOverrides := parseFlowAnnotations(request) return hasOverrides && fo == mark } + +// Remaining returns the number of requests that are still left to be +// generated (and therefore to be sent) by this generator. +func (r *requestGenerator) Remaining() int { + var sequence []string + switch { + case len(r.request.Path) > 0: + sequence = r.request.Path + case len(r.request.Raw) > 0: + sequence = r.request.Raw + default: + return 0 + } + + remainingInCurrentPass := 0 + for i := r.currentIndex; i < len(sequence); i++ { + if !r.hasMarker(sequence[i], Once) { + remainingInCurrentPass++ + } + } + + if r.payloadIterator == nil { + return remainingInCurrentPass + } + + numRemainingPayloadSets := r.payloadIterator.Remaining() + totalValidInSequence := 0 + for _, req := range sequence { + if !r.hasMarker(req, Once) { + totalValidInSequence++ + } + } + + // Total remaining = remaining in current pass + (remaining payload sets * requests per full pass) + return remainingInCurrentPass + numRemainingPayloadSets*totalValidInSequence +} + +func (r *requestGenerator) Total() int { + var sequence []string + switch { + case len(r.request.Path) > 0: + sequence = r.request.Path + case len(r.request.Raw) > 0: + sequence = r.request.Raw + default: + return 0 + } + + applicableRequests := 0 + additionalRequests := 0 + for _, request := range sequence { + if !r.hasMarker(request, Once) { + applicableRequests++ + } else { + additionalRequests++ + } + } + + if r.payloadIterator == nil { + return applicableRequests + additionalRequests + } + + return (applicableRequests * r.payloadIterator.Total()) + additionalRequests +} diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index d59af2f7b..5f791c2c1 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -133,6 +133,10 @@ func (m *MockOutputWriter) Colorizer() aurora.Aurora { return m.aurora } +func (m *MockOutputWriter) ResultCount() int { + return 0 +} + // Write writes the event to file and/or screen. func (m *MockOutputWriter) Write(result *output.ResultEvent) error { if m.WriteCallback != nil { diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 2548ad858..206914e08 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -51,6 +51,8 @@ type FlowExecutor struct { // these are keys whose values are meant to be flatten before executing // a request ex: if dynamic extractor returns ["value"] it will be converted to "value" flattenKeys []string + + executed *mapsutil.SyncLockMap[string, struct{}] } // NewFlowExecutor creates a new flow executor from a list of requests @@ -98,6 +100,7 @@ func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, option results: results, ctx: ctx, program: program, + executed: mapsutil.NewSyncLockMap[string, struct{}](), } return f, nil } @@ -246,6 +249,7 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { // pass flow and execute the js vm and handle errors _, err := runtime.RunProgram(f.program) + f.reconcileProgress() if err != nil { ctx.LogError(err) return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) @@ -259,6 +263,18 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { return nil } +func (f *FlowExecutor) reconcileProgress() { + for proto, list := range f.allProtocols { + for idx, req := range list { + key := requestKey(proto, req, strconv.Itoa(idx+1)) + if _, seen := f.executed.Get(key); !seen { + // never executed → pretend it finished so that stats match + f.options.Progress.SetRequests(uint64(req.Requests())) + } + } + } +} + // GetRuntimeErrors returns all runtime errors (i.e errors from all protocol combined) func (f *FlowExecutor) GetRuntimeErrors() error { errs := []error{} diff --git a/pkg/tmplexec/flow/flow_internal.go b/pkg/tmplexec/flow/flow_internal.go index 4e759ba44..1738eb023 100644 --- a/pkg/tmplexec/flow/flow_internal.go +++ b/pkg/tmplexec/flow/flow_internal.go @@ -75,6 +75,8 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma } } err := req.ExecuteWithResults(inputItem, output.InternalEvent(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()), output.InternalEvent{}, f.protocolResultCallback(req, matcherStatus, opts)) + // Mark the request as seen + _ = f.executed.Set(requestKey(opts.protoName, req, id), struct{}{}) if err != nil { index := id err = f.allErrs.Set(opts.protoName+":"+index, err) @@ -86,6 +88,13 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma return matcherStatus.Load() } +func requestKey(proto string, req protocols.Request, id string) string { + if id == "" { + id = req.GetID() + } + return proto + ":" + id +} + // protocolResultCallback returns a callback that is executed // after execution of each protocol request func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStatus *atomic.Bool, _ *ProtoOptions) func(result *output.InternalWrappedEvent) {