External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-23 17:25:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
mzack 2022-02-26 08:02:16 +01:00
parent 73d1247b71
commit a51d307967
3 changed files with 167 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
v2/pkg/protocols/file/operators.go
View File

@ -1,14 +1,10 @@
package file
import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -65,26 +61,12 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
return itemStr, true
}
type fileStatus struct {
results []*operators.Result
raw string
inputFilePath string
matchedFileName string
lines int
words int
bytes int
}
// responseToDSLMap converts a file chunk elaboration to a map for use in DSL matching
func (request *Request) responseToDSLMap(state *fileStatus) output.InternalEvent {
func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent {
return output.InternalEvent{
"results": state.results,
"path": state.inputFilePath,
"matched": state.matchedFileName,
"raw": state.raw,
"lines": state.lines,
"words": state.words,
"bytes": state.bytes,
"path": inputFilePath,
"matched": matchedFileName,
"raw": raw,
"type": request.Type().String(),
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
@ -93,27 +75,17 @@ func (request *Request) responseToDSLMap(state *fileStatus) output.InternalEvent
}
// MakeResultEvent creates a result event from internal wrapped event
// Deprecated: unused in stream mode, must be present for interface compatibility
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
panic("unused")
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
// MakeResultEventItem
// Deprecated: unused in stream mode, must be present for interface compatibility
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
MatcherStatus: true,
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Path: types.ToString(wrapped.InternalEvent["path"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Response: types.ToString(wrapped.InternalEvent["raw"]),
Timestamp: time.Now(),
}
return data
panic("unused")
}

39
v2/pkg/protocols/file/operators_test.go
View File

@ -1,6 +1,8 @@
package file
import (
"log"
"strings"
"testing"
"github.com/stretchr/testify/require"
@ -34,8 +36,8 @@ func TestResponseToDSLMap(t *testing.T) {
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n"
event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
require.Len(t, event, 11, "could not get correct number of items in dsl map")
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 7, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
}
@ -59,8 +61,8 @@ func TestFileOperatorMatch(t *testing.T) {
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
require.Len(t, event, 11, "could not get correct number of items in dsl map")
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 7, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
t.Run("valid", func(t *testing.T) {
@ -108,8 +110,8 @@ func TestFileOperatorMatch(t *testing.T) {
t.Run("caseInsensitive", func(t *testing.T) {
resp := "TEST-DATA\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
require.Len(t, event, 11, "could not get correct number of items in dsl map")
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 7, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
matcher := &matchers.Matcher{
@ -147,8 +149,8 @@ func TestFileOperatorExtract(t *testing.T) {
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
require.Len(t, event, 11, "could not get correct number of items in dsl map")
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 7, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
t.Run("extract", func(t *testing.T) {
@ -217,6 +219,7 @@ func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.
}
finalEvent := testFileMakeResult(t, matcher, matcherCondition, true)
log.Fatalf("%+v\n%+v\n", expectedValues, finalEvent.Results[0])
for matcherName, matchedValues := range expectedValues {
var matchesOne = false
for i := 0; i <= len(expectedValue); i++ {
@ -262,24 +265,16 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
matchedFileName := "test.txt"
fileContent := "test-data\r\n1.1.1.1\r\n"
matchedFileName := "test.txt"
input := "/tmp"
event := request.responseToDSLMap(&fileStatus{raw: fileContent, inputFilePath: "/tmp", matchedFileName: matchedFileName})
require.Len(t, event, 11, "could not get correct number of items in dsl map")
require.Equal(t, fileContent, event["raw"], "could not get correct resp")
fileMatches := request.collectMatches(strings.NewReader(fileContent), input, matchedFileName, "")
event := request.buildEvent(input, matchedFileName, fileMatches)
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
resultEvent := finalEvent.Results[0]
resultEvent := event.Results[0]
require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results")
require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value")
return finalEvent
return event
}

63
v2/pkg/protocols/file/request.go
View File

@ -65,17 +65,34 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
if stat.Size() >= request.maxSize {
gologger.Verbose().Msgf("Limiting %s processed data to %s bytes: exceeded max size\n", filePath, units.HumanSize(float64(request.maxSize)))
}
totalBytes := units.BytesSize(float64(stat.Size()))
fileReader := io.LimitReader(file, request.maxSize)
fileMatches := request.collectMatches(fileReader, input, filePath, units.BytesSize(float64(stat.Size())))
// build event structure to interface with internal logic
event := request.buildEvent(input, filePath, fileMatches)
dumpResponse(event, request.options, fileMatches, filePath)
callback(event)
request.options.Progress.IncrementRequests()
}(data)
})
wg.Wait()
if err != nil {
request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request")
}
return nil
}
func (request *Request) collectMatches(reader io.Reader, input, filePath, totalBytes string) []FileMatch {
var bytesCount, linesCount, wordsCount int
isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse
scanner := bufio.NewScanner(fileReader)
scanner := bufio.NewScanner(reader)
buffer := []byte{}
scanner.Buffer(buffer, int(chunkSize))
var fileMatches []FileMatch
exprLines := make(map[string][]int)
exprBytes := make(map[string][]int)
for scanner.Scan() {
lineContent := scanner.Text()
n := len(lineContent)
@ -85,15 +102,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
processedBytes := units.BytesSize(float64(currentBytes))
gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytes)
dslMap := request.responseToDSLMap(&fileStatus{
raw: lineContent,
inputFilePath: input,
matchedFileName: filePath,
lines: linesCount,
words: wordsCount,
bytes: bytesCount,
})
dslMap := request.responseToDSLMap(lineContent, input, filePath)
if parts, ok := request.CompiledOperators.Execute(dslMap, request.Match, request.Extract, isResponseDebug); parts != nil && ok {
if parts.Extracts != nil {
for expr, extracts := range parts.Extracts {
@ -129,14 +138,14 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
linesCount += currentLinesCount
wordsCount += strings.Count(lineContent, " ")
bytesCount = currentBytes
}
return fileMatches
}
// build event structure to interface with internal logic
internalEvent := request.responseToDSLMap(&fileStatus{
inputFilePath: input,
matchedFileName: filePath,
})
func (request *Request) buildEvent(input, filePath string, fileMatches []FileMatch) *output.InternalWrappedEvent {
exprLines := make(map[string][]int)
exprBytes := make(map[string][]int)
internalEvent := request.responseToDSLMap("", input, filePath)
operatorResult := &operators.Result{}
for _, fileMatch := range fileMatches {
operatorResult.Matched = operatorResult.Matched || fileMatch.Match
@ -180,7 +189,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
Info: internalEvent["template-info"].(model.Info),
Type: types.ToString(internalEvent["type"]),
Path: types.ToString(internalEvent["path"]),
Matched: types.ToString(internalEvent["path"]),
Matched: types.ToString(internalEvent["matched"]),
Host: types.ToString(internalEvent["host"]),
ExtractedResults: items,
// Response: types.ToString(wrapped.InternalEvent["raw"]),
@ -207,23 +216,11 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
})
}
event := &output.InternalWrappedEvent{
return &output.InternalWrappedEvent{
InternalEvent: internalEvent,
Results: results,
OperatorsResult: operatorResult,
}
dumpResponse(event, request.options, fileMatches, filePath)
callback(event)
request.options.Progress.IncrementRequests()
}(data)
})
wg.Wait()
if err != nil {
request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request")
}
return nil
}
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, filematches []FileMatch, filePath string) {