External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:35:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into issue-1594-regexp

Browse Source
This commit is contained in:
mzack 2022-02-28 11:42:46 +01:00
commit 6cb522c9b2
18 changed files with 179 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
.github/workflows/changelog-indexer.yaml vendored Normal file
View File

@ -0,0 +1,29 @@
name: 📑 Template-DB Indexer
on:
push:
tags:
- '*'
workflow_dispatch:
jobs:
index:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Intalling Indexer
run: |
git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github".insteadOf https://github
git clone https://github.com/projectdiscovery/nucleish-api.git
cd nucleish-api/cmd/generate-index/
go install
- name: Generate Index
env:
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
run: |
generate-index -mode changelog

16
SYNTAX-REFERENCE.md
View File

@ -489,7 +489,7 @@ Severity of the template.
<div class="dd">
<code>metadata</code> <i>map[string]string</i>
<code>metadata</code> <i>map[string]interface{}</i>
</div>
<div class="dt">
@ -1232,6 +1232,20 @@ all requests defined in raw section.
<div class="dd">
<code>read-all</code> <i>bool</i>
</div>
<div class="dt">
Enables force reading of the entire raw unsafe request body ignoring
any specified content length headers.
</div>
<hr />
<div class="dd">
<code>redirects</code> <i>bool</i>
</div>

7
nuclei-jsonschema.json
View File

@ -72,7 +72,7 @@
"metadata": {
"patternProperties": {
".*": {
"type": "string"
"additionalProperties": true
}
},
"type": "object",
@ -774,6 +774,11 @@
"title": "optional cookie reuse enable",
"description": "Optional setting that enables cookie reuse"
},
"read-all": {
"type": "boolean",
"title": "force read all body",
"description": "Enables force reading of entire unsafe http request body"
},
"redirects": {
"type": "boolean",
"title": "follow http redirects",

7
v2/go.mod
View File

@ -5,7 +5,7 @@ go 1.17
require (
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725
github.com/andygrunwald/go-jira v1.15.0
github.com/andygrunwald/go-jira v1.15.1
github.com/antchfx/htmlquery v1.2.4
github.com/apex/log v1.9.0
github.com/blang/semver v3.5.1+incompatible
@ -35,7 +35,7 @@ require (
github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
github.com/projectdiscovery/interactsh v1.0.1-0.20220131074403-ca8bb8f87cd0
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/rawhttp v0.0.8-0.20220225134552-b60c4c126e5b
github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660
github.com/projectdiscovery/retryablehttp-go v1.0.2
github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700
@ -63,7 +63,7 @@ require (
moul.io/http2curl v1.0.0
)
require github.com/aws/aws-sdk-go v1.43.6
require github.com/aws/aws-sdk-go v1.43.7
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
@ -72,6 +72,7 @@ require (
github.com/docker/go-units v0.4.0
github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7
github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3
github.com/rs/xid v1.3.0
github.com/stretchr/testify v1.7.0
github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b
)

11
v2/go.sum
View File

@ -67,8 +67,8 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgp
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE=
github.com/andygrunwald/go-jira v1.15.0 h1:krX3Ad4i44mOG0u+vIxQLhLkyyQKPbTsOISsDRgBNjU=
github.com/andygrunwald/go-jira v1.15.0/go.mod h1:GIYN1sHOIsENWUZ7B4pDeT/nxEtrZpE8l0987O67ZR8=
github.com/andygrunwald/go-jira v1.15.1 h1:6J9aYKb9sW8bxv3pBLYBrs0wdsFrmGI5IeTgWSKWKc8=
github.com/andygrunwald/go-jira v1.15.1/go.mod h1:GIYN1sHOIsENWUZ7B4pDeT/nxEtrZpE8l0987O67ZR8=
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
@ -83,8 +83,8 @@ github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3st
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.43.6 h1:FkwmndZR4LjnT2fiKaD18bnqfQ188E8A1IMNI5rcv00=
github.com/aws/aws-sdk-go v1.43.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go v1.43.7 h1:Gbs53KxXJWbO3txoVkevf56bhdDFqRisl7MQQ6581vc=
github.com/aws/aws-sdk-go v1.43.7/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@ -457,8 +457,9 @@ github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALateb
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs=
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M=
github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY=
github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.8-0.20220225134552-b60c4c126e5b h1:ODEtmulEsryrOR3z949wxAdwFiRlnqn2HUh+dHcBK6w=
github.com/projectdiscovery/rawhttp v0.0.8-0.20220225134552-b60c4c126e5b/go.mod h1:jAoQA4i8iu2v4u50ufmlky1t1WsBKpFP2XKd3BbY2y4=
github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=

1
v2/internal/runner/runner.go
View File

@ -171,6 +171,7 @@ func New(options *types.Options) (*Runner, error) {
opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress)
opts.Debug = runner.options.Debug
opts.NoColor = runner.options.NoColor
if options.InteractshURL != "" {
opts.ServerURL = options.InteractshURL
}

2
v2/pkg/model/model.go
View File

@ -57,7 +57,7 @@ type Info struct {
// examples:
// - value: >
// map[string]string{"customField1":"customValue1"}
Metadata map[string]string `json:"metadata,omitempty" yaml:"metadata,omitempty" jsonschema:"title=additional metadata for the template,description=Additional metadata fields for the template"`
Metadata map[string]interface{} `json:"metadata,omitempty" yaml:"metadata,omitempty" jsonschema:"title=additional metadata for the template,description=Additional metadata fields for the template"`
// description: |
// Classification contains classification information about the template.

33
v2/pkg/model/model_test.go
View File

@ -21,12 +21,19 @@ func TestInfoJsonMarshal(t *testing.T) {
SeverityHolder: severity.Holder{Severity: severity.High},
Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}},
Reference: stringslice.StringSlice{Value: "reference1"},
Metadata: map[string]interface{}{
"string_key": "string_value",
"array_key": []string{"array_value1", "array_value2"},
"map_key": map[string]string{
"key1": "val1",
},
},
}
result, err := json.Marshal(&info)
assert.Nil(t, err)
expected := `{"name":"Test Template Name","author":["forgedhallpass","ice3man"],"tags":["cve","misc"],"description":"Test description","reference":"reference1","severity":"high"}`
expected := `{"name":"Test Template Name","author":["forgedhallpass","ice3man"],"tags":["cve","misc"],"description":"Test description","reference":"reference1","severity":"high","metadata":{"array_key":["array_value1","array_value2"],"map_key":{"key1":"val1"},"string_key":"string_value"}}`
assert.Equal(t, expected, string(result))
}
@ -38,6 +45,13 @@ func TestInfoYamlMarshal(t *testing.T) {
SeverityHolder: severity.Holder{Severity: severity.High},
Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}},
Reference: stringslice.StringSlice{Value: "reference1"},
Metadata: map[string]interface{}{
"string_key": "string_value",
"array_key": []string{"array_value1", "array_value2"},
"map_key": map[string]string{
"key1": "val1",
},
},
}
result, err := yaml.Marshal(&info)
@ -53,6 +67,13 @@ tags:
description: Test description
reference: reference1
severity: high
metadata:
array_key:
- array_value1
- array_value2
map_key:
key1: val1
string_key: string_value
`
assert.Equal(t, expected, string(result))
}
@ -66,7 +87,7 @@ func TestUnmarshal(t *testing.T) {
dynamicKey1 := "customDynamicKey1"
dynamicKey2 := "customDynamicKey2"
dynamicKeysMap := map[string]string{
dynamicKeysMap := map[string]interface{}{
dynamicKey1: "customDynamicValue1",
dynamicKey2: "customDynamicValue2",
}
@ -92,8 +113,8 @@ func TestUnmarshal(t *testing.T) {
severity: critical
reference: ` + strings.Join(references, ", ") + `
metadata:
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1].(string) + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2].(string) + `
`
yamlPayload2 := `
name: ` + templateName + `
@ -108,8 +129,8 @@ func TestUnmarshal(t *testing.T) {
- ` + references[0] + ` # comments are not unmarshalled
- ` + references[1] + `
metadata:
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1].(string) + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2].(string) + `
`
info1 := assertUnmarshalledTemplateInfo(t, yamlPayload1)

15
v2/pkg/operators/common/dsl/dsl.go
View File

@ -11,6 +11,7 @@ import (
"errors"
"fmt"
"html"
"io"
"math"
"math/rand"
"net/url"
@ -101,12 +102,26 @@ func init() {
buffer := &bytes.Buffer{}
writer := gzip.NewWriter(buffer)
if _, err := writer.Write([]byte(args[0].(string))); err != nil {
_ = writer.Close()
return "", err
}
_ = writer.Close()
return buffer.String(), nil
}),
"gzip_decode": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
reader, err := gzip.NewReader(strings.NewReader(args[0].(string)))
if err != nil {
return "", err
}
data, err := io.ReadAll(reader)
if err != nil {
_ = reader.Close()
return "", err
}
_ = reader.Close()
return string(data), nil
}),
"base64_py": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
stdBase64 := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))

15
v2/pkg/operators/common/dsl/dsl_test.go
View File

@ -1,12 +1,9 @@
package dsl
import (
"compress/gzip"
"fmt"
"io"
"math"
"regexp"
"strings"
"testing"
"time"
@ -40,15 +37,18 @@ func TestDSLTimeComparison(t *testing.T) {
func TestDSLGzipSerialize(t *testing.T) {
compiled, err := govaluate.NewEvaluableExpressionWithFunctions("gzip(\"hello world\")", HelperFunctions())
require.Nil(t, err, "could not compare time")
require.Nil(t, err, "could not compile encoder")
result, err := compiled.Evaluate(make(map[string]interface{}))
require.Nil(t, err, "could not evaluate compare time")
reader, _ := gzip.NewReader(strings.NewReader(types.ToString(result)))
data, _ := io.ReadAll(reader)
compiled, err = govaluate.NewEvaluableExpressionWithFunctions("gzip_decode(data)", HelperFunctions())
require.Nil(t, err, "could not compile decoder")
require.Equal(t, "hello world", string(data), "could not get gzip encoded data")
data, err := compiled.Evaluate(map[string]interface{}{"data": result})
require.Nil(t, err, "could not evaluate decoded data")
require.Equal(t, "hello world", data.(string), "could not get gzip encoded data")
}
func TestDslFunctionSignatures(t *testing.T) {
@ -102,6 +102,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
contains(arg1, arg2 interface{}) interface{}
generate_java_gadget(arg1, arg2, arg3 interface{}) interface{}
gzip(arg1 interface{}) interface{}
gzip_decode(arg1 interface{}) interface{}
hex_decode(arg1 interface{}) interface{}
hex_encode(arg1 interface{}) interface{}
html_escape(arg1 interface{}) interface{}

30
v2/pkg/protocols/common/interactsh/interactsh.go
View File

@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
)
@ -82,6 +83,8 @@ type Options struct {
DisableHttpFallback bool
// NoInteractsh disables the engine
NoInteractsh bool
// NoColor dissbles printing colors for matches
NoColor bool
StopAtFirstMatch bool
}
@ -124,6 +127,7 @@ func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progre
IssuesClient: reporting,
Progress: progress,
DisableHttpFallback: true,
NoColor: false,
}
}
@ -148,9 +152,6 @@ func (c *Client) firstTimeInitializeClient() error {
c.hostname = interactDomain
interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse {
c.debugPrintInteraction(interaction)
}
item := c.requests.Get(interaction.UniqueID)
if item == nil {
@ -189,7 +190,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false)
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, c.options.Debug || c.options.DebugRequest || c.options.DebugResponse)
if !matched || result == nil {
return false // if we don't match, return
}
@ -205,6 +206,10 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
event.Interaction = interaction
}
if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse {
c.debugPrintInteraction(interaction, data.Event.OperatorsResult)
}
if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) {
c.matched = true
if _, ok := data.Event.InternalEvent["stop-at-first-match"]; ok || c.options.StopAtFirstMatch {
@ -350,35 +355,35 @@ func HasMarkers(data string) bool {
return strings.Contains(data, interactshURLMarker)
}
func (c *Client) debugPrintInteraction(interaction *server.Interaction) {
func (c *Client) debugPrintInteraction(interaction *server.Interaction, event *operators.Result) {
builder := &bytes.Buffer{}
switch interaction.Protocol {
case "dns":
builder.WriteString(formatInteractionHeader("DNS", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
if c.options.DebugRequest || c.options.Debug {
builder.WriteString(formatInteractionMessage("DNS Request", interaction.RawRequest))
builder.WriteString(formatInteractionMessage("DNS Request", interaction.RawRequest, event, c.options.NoColor))
}
if c.options.DebugResponse || c.options.Debug {
builder.WriteString(formatInteractionMessage("DNS Response", interaction.RawResponse))
builder.WriteString(formatInteractionMessage("DNS Response", interaction.RawResponse, event, c.options.NoColor))
}
case "http":
builder.WriteString(formatInteractionHeader("HTTP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
if c.options.DebugRequest || c.options.Debug {
builder.WriteString(formatInteractionMessage("HTTP Request", interaction.RawRequest))
builder.WriteString(formatInteractionMessage("HTTP Request", interaction.RawRequest, event, c.options.NoColor))
}
if c.options.DebugResponse || c.options.Debug {
builder.WriteString(formatInteractionMessage("HTTP Response", interaction.RawResponse))
builder.WriteString(formatInteractionMessage("HTTP Response", interaction.RawResponse, event, c.options.NoColor))
}
case "smtp":
builder.WriteString(formatInteractionHeader("SMTP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
if c.options.DebugRequest || c.options.Debug || c.options.DebugResponse {
builder.WriteString(formatInteractionMessage("SMTP Interaction", interaction.RawRequest))
builder.WriteString(formatInteractionMessage("SMTP Interaction", interaction.RawRequest, event, c.options.NoColor))
}
case "ldap":
builder.WriteString(formatInteractionHeader("LDAP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
if c.options.DebugRequest || c.options.Debug || c.options.DebugResponse {
builder.WriteString(formatInteractionMessage("LDAP Interaction", interaction.RawRequest))
builder.WriteString(formatInteractionMessage("LDAP Interaction", interaction.RawRequest, event, c.options.NoColor))
}
}
fmt.Fprint(os.Stderr, builder.String())
@ -388,7 +393,8 @@ func formatInteractionHeader(protocol, ID, address string, at time.Time) string
return fmt.Sprintf("[%s] Received %s interaction from %s at %s", ID, protocol, address, at.Format("2006-01-02 15:04:05"))
}
func formatInteractionMessage(key, value string) string {
func formatInteractionMessage(key, value string, event *operators.Result, noColor bool) string {
value = responsehighlighter.Highlight(event, value, noColor, false)
return fmt.Sprintf("\n------------\n%s\n------------\n\n%s\n\n", key, value)
}

4
v2/pkg/protocols/http/http.go
View File

@ -139,6 +139,10 @@ type Request struct {
// all requests defined in raw section.
CookieReuse bool `yaml:"cookie-reuse,omitempty" jsonschema:"title=optional cookie reuse enable,description=Optional setting that enables cookie reuse"`
// description: |
// Enables force reading of the entire raw unsafe request body ignoring
// any specified content length headers.
ForceReadAllBody bool `yaml:"read-all,omitempty" jsonschema:"title=force read all body,description=Enables force reading of entire unsafe http request body"`
// description: |
// Redirects specifies whether redirects should be followed by the HTTP Client.
//
// This can be used in conjunction with `max-redirects` to control the HTTP request redirects.

1
v2/pkg/protocols/http/request.go
View File

@ -383,6 +383,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
options := generatedRequest.original.rawhttpClient.Options
options.FollowRedirects = request.Redirects
options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes
options.ForceReadAllBody = request.ForceReadAllBody
resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), options)
} else {
hostname = generatedRequest.request.URL.Host

3
v2/pkg/protocols/network/request.go
View File

@ -69,9 +69,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
return err
}
variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address})
payloads := generators.BuildPayloadFromOptions(request.options.Options)
// add Hostname variable to the payload
payloads = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address})
if request.generator != nil {
iterator := request.generator.NewIterator()

9
v2/pkg/reporting/format/format.go
View File

@ -181,9 +181,12 @@ func ToMarkdownTableString(templateInfo *model.Info) string {
builder := &bytes.Buffer{}
toMarkDownTable := func(insertionOrderedStringMap *utils.InsertionOrderedStringMap) {
insertionOrderedStringMap.ForEach(func(key string, value string) {
if utils.IsNotBlank(value) {
builder.WriteString(fmt.Sprintf("| %s | %s |\n", key, value))
insertionOrderedStringMap.ForEach(func(key string, value interface{}) {
switch value := value.(type) {
case string:
if utils.IsNotBlank(value) {
builder.WriteString(fmt.Sprintf("| %s | %s |\n", key, value))
}
}
})
}

2
v2/pkg/reporting/format/format_test.go
View File

@ -19,7 +19,7 @@ func TestToMarkdownTableString(t *testing.T) {
SeverityHolder: severity.Holder{Severity: severity.High},
Tags: stringslice.StringSlice{Value: []string{"cve", "misc"}},
Reference: stringslice.StringSlice{Value: "reference1"},
Metadata: map[string]string{
Metadata: map[string]interface{}{
"customDynamicKey1": "customDynamicValue1",
"customDynamicKey2": "customDynamicValue2",
},

57
v2/pkg/templates/templates_doc.go
View File

@ -188,7 +188,7 @@ func init() {
MODELInfoDoc.Fields[5].Description = "Severity of the template."
MODELInfoDoc.Fields[5].Comments[encoder.LineComment] = "Severity of the template."
MODELInfoDoc.Fields[6].Name = "metadata"
MODELInfoDoc.Fields[6].Type = "map[string]string"
MODELInfoDoc.Fields[6].Type = "map[string]interface{}"
MODELInfoDoc.Fields[6].Note = ""
MODELInfoDoc.Fields[6].Description = "Metadata of the template."
MODELInfoDoc.Fields[6].Comments[encoder.LineComment] = "Metadata of the template."
@ -388,7 +388,7 @@ func init() {
Value: "HTTP response headers in name:value format",
},
}
HTTPRequestDoc.Fields = make([]encoder.Doc, 28)
HTTPRequestDoc.Fields = make([]encoder.Doc, 29)
HTTPRequestDoc.Fields[0].Name = "matchers"
HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher"
HTTPRequestDoc.Fields[0].Note = ""
@ -521,46 +521,51 @@ func init() {
HTTPRequestDoc.Fields[19].Note = ""
HTTPRequestDoc.Fields[19].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section."
HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for"
HTTPRequestDoc.Fields[20].Name = "redirects"
HTTPRequestDoc.Fields[20].Name = "read-all"
HTTPRequestDoc.Fields[20].Type = "bool"
HTTPRequestDoc.Fields[20].Note = ""
HTTPRequestDoc.Fields[20].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
HTTPRequestDoc.Fields[21].Name = "pipeline"
HTTPRequestDoc.Fields[20].Description = "Enables force reading of the entire raw unsafe request body ignoring\nany specified content length headers."
HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Enables force reading of the entire raw unsafe request body ignoring"
HTTPRequestDoc.Fields[21].Name = "redirects"
HTTPRequestDoc.Fields[21].Type = "bool"
HTTPRequestDoc.Fields[21].Note = ""
HTTPRequestDoc.Fields[21].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
HTTPRequestDoc.Fields[22].Name = "unsafe"
HTTPRequestDoc.Fields[21].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
HTTPRequestDoc.Fields[22].Name = "pipeline"
HTTPRequestDoc.Fields[22].Type = "bool"
HTTPRequestDoc.Fields[22].Note = ""
HTTPRequestDoc.Fields[22].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
HTTPRequestDoc.Fields[23].Name = "race"
HTTPRequestDoc.Fields[22].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
HTTPRequestDoc.Fields[23].Name = "unsafe"
HTTPRequestDoc.Fields[23].Type = "bool"
HTTPRequestDoc.Fields[23].Note = ""
HTTPRequestDoc.Fields[23].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
HTTPRequestDoc.Fields[24].Name = "req-condition"
HTTPRequestDoc.Fields[23].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
HTTPRequestDoc.Fields[24].Name = "race"
HTTPRequestDoc.Fields[24].Type = "bool"
HTTPRequestDoc.Fields[24].Note = ""
HTTPRequestDoc.Fields[24].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
HTTPRequestDoc.Fields[25].Name = "stop-at-first-match"
HTTPRequestDoc.Fields[24].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
HTTPRequestDoc.Fields[25].Name = "req-condition"
HTTPRequestDoc.Fields[25].Type = "bool"
HTTPRequestDoc.Fields[25].Note = ""
HTTPRequestDoc.Fields[25].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[26].Name = "skip-variables-check"
HTTPRequestDoc.Fields[25].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
HTTPRequestDoc.Fields[26].Name = "stop-at-first-match"
HTTPRequestDoc.Fields[26].Type = "bool"
HTTPRequestDoc.Fields[26].Note = ""
HTTPRequestDoc.Fields[26].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[27].Name = "iterate-all"
HTTPRequestDoc.Fields[26].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[27].Name = "skip-variables-check"
HTTPRequestDoc.Fields[27].Type = "bool"
HTTPRequestDoc.Fields[27].Note = ""
HTTPRequestDoc.Fields[27].Description = "IterateAll iterates all the values extracted from internal extractors"
HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
HTTPRequestDoc.Fields[27].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[28].Name = "iterate-all"
HTTPRequestDoc.Fields[28].Type = "bool"
HTTPRequestDoc.Fields[28].Note = ""
HTTPRequestDoc.Fields[28].Description = "IterateAll iterates all the values extracted from internal extractors"
HTTPRequestDoc.Fields[28].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
MATCHERSMatcherDoc.Type = "matchers.Matcher"
MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol."

10
v2/pkg/utils/insertion_ordered_map.go
View File

@ -2,17 +2,17 @@ package utils
type InsertionOrderedStringMap struct {
keys []string `yaml:"-"`
values map[string]string
values map[string]interface{}
}
func NewEmptyInsertionOrderedStringMap(size int) *InsertionOrderedStringMap {
return &InsertionOrderedStringMap{
keys: make([]string, 0, size),
values: make(map[string]string, size),
values: make(map[string]interface{}, size),
}
}
func NewInsertionOrderedStringMap(stringMap map[string]string) *InsertionOrderedStringMap {
func NewInsertionOrderedStringMap(stringMap map[string]interface{}) *InsertionOrderedStringMap {
result := NewEmptyInsertionOrderedStringMap(len(stringMap))
for k, v := range stringMap {
@ -22,13 +22,13 @@ func NewInsertionOrderedStringMap(stringMap map[string]string) *InsertionOrdered
return result
}
func (insertionOrderedStringMap *InsertionOrderedStringMap) ForEach(fn func(key string, data string)) {
func (insertionOrderedStringMap *InsertionOrderedStringMap) ForEach(fn func(key string, data interface{})) {
for _, key := range insertionOrderedStringMap.keys {
fn(key, insertionOrderedStringMap.values[key])
}
}
func (insertionOrderedStringMap *InsertionOrderedStringMap) Set(key string, value string) {
func (insertionOrderedStringMap *InsertionOrderedStringMap) Set(key string, value interface{}) {
_, present := insertionOrderedStringMap.values[key]
insertionOrderedStringMap.values[key] = value
if !present {