temporary line calculation with multiple file read

todo: replace with one pass scan via io.reader

v2/pkg/operators/operators.go

@@ -145,10 +145,10 @@ func (r *Result) Merge(result *Result) {
 	}
 
 	for k, v := range result.Matches {
-		r.Matches[k] = v
+		r.Matches[k] = append(r.Matches[k], v...)
 	}
 	for k, v := range result.Extracts {
-		r.Extracts[k] = v
+		r.Extracts[k] = append(r.Extracts[k], v...)
 	}
 
 	r.outputUnique = make(map[string]struct{})
@@ -201,7 +201,6 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 	// Start with the extractors first and evaluate them.
 	for _, extractor := range operators.Extractors {
 		var extractorResults []string
-
 		for match := range extract(data, extractor) {
 			extractorResults = append(extractorResults, match)
 

v2/pkg/output/format_screen.go

@@ -65,13 +65,13 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 		builder.WriteString("]")
 	}
 
-	if len(output.LineCount) > 0 {
+	if len(output.Lines) > 0 {
 		builder.WriteString(" [LN: ")
 
-		for i, line := range output.LineCount {
+		for i, line := range output.Lines {
 			builder.WriteString(strconv.Itoa(line))
 
-			if i != len(output.LineCount)-1 {
+			if i != len(output.Lines)-1 {
 				builder.WriteString(",")
 			}
 		}

v2/pkg/output/output.go

@@ -105,8 +105,8 @@ type ResultEvent struct {
 	CURLCommand string `json:"curl-command,omitempty"`
 	// MatcherStatus is the status of the match
 	MatcherStatus bool `json:"matcher-status"`
-	// LineCount is the line count for the specified match
-	LineCount []int `json:"matched-line"`
+	// Lines is the line count for the specified match
+	Lines []int `json:"matched-line"`
 
 	FileToIndexPosition map[string]int `json:"-"`
 }

v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go

@@ -1,6 +1,7 @@
 package eventcreator
 
 import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 )
@@ -29,3 +30,25 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out
 	}
 	return event
 }
+
+func CreateEventWithResults(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool, results ...*operators.Result) *output.InternalWrappedEvent {
+	event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
+	for _, result := range results {
+		event.OperatorsResult = result
+		event.Results = append(event.Results, request.MakeResultEvent(event)...)
+	}
+	return event
+}
+
+func GetEventResults(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) []*operators.Result {
+	var results []*operators.Result
+	for _, compiledOperator := range request.GetCompiledOperators() {
+		if compiledOperator != nil {
+			result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract, isResponseDebug)
+			if ok && results != nil {
+				results = append(results, result)
+			}
+		}
+	}
+	return nil
+}

v2/pkg/protocols/file/operators.go

@@ -2,6 +2,7 @@ package file
 
 import (
 	"bufio"
+	"os"
 	"strings"
 	"time"
 
@@ -76,8 +77,8 @@ type fileStatus struct {
 	bytes           int
 }
 
-// toDSLMap converts a file chunk elaboration to a map for use in DSL matching
-func (request *Request) toDSLMap(state *fileStatus) output.InternalEvent {
+// responseToDSLMap converts a file chunk elaboration to a map for use in DSL matching
+func (request *Request) responseToDSLMap(state *fileStatus) output.InternalEvent {
 	return output.InternalEvent{
 		"path":          state.inputFilePath,
 		"matched":       state.matchedFileName,
@@ -94,20 +95,8 @@ func (request *Request) toDSLMap(state *fileStatus) output.InternalEvent {
 
 // MakeResultEvent creates a result event from internal wrapped event
 func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
+	filePath := wrapped.InternalEvent["path"].(string)
 	results := protocols.MakeDefaultResultEvent(request, wrapped)
-
-	raw, ok := wrapped.InternalEvent["raw"]
-	if !ok {
-		return results
-	}
-
-	linesOffset := wrapped.InternalEvent["lines"].(int)
-
-	rawStr, ok := raw.(string)
-	if !ok {
-		return results
-	}
-
 	for _, result := range results {
 		lineWords := make(map[string]struct{})
 
@@ -123,13 +112,13 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 				lineWords[v] = struct{}{}
 			}
 		}
-		result.LineCount = calculateLineFunc(rawStr, linesOffset, lineWords)
+		result.Lines = calculateLineFunc(filePath, lineWords)
 	}
-
 	// Identify the position of match in file using a dirty hack.
 	for _, result := range results {
 		for _, extraction := range result.ExtractedResults {
-			scanner := bufio.NewScanner(strings.NewReader(rawStr))
+			file, _ := os.Open(filePath)
+			scanner := bufio.NewScanner(file)
 
 			line := 1
 			for scanner.Scan() {
@@ -137,11 +126,12 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 					if result.FileToIndexPosition == nil {
 						result.FileToIndexPosition = make(map[string]int)
 					}
-					result.FileToIndexPosition[result.Matched] = line + linesOffset
+					result.FileToIndexPosition[result.Matched] = line
 					continue
 				}
 				line++
 			}
+			file.Close()
 		}
 	}
 	return results

v2/pkg/protocols/file/operators_test.go

@@ -34,7 +34,7 @@ func TestResponseToDSLMap(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n"
-	event := request.toDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
 	require.Len(t, event, 10, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 }
@@ -59,7 +59,7 @@ func TestFileOperatorMatch(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n1.1.1.1\r\n"
-	event := request.toDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
 	require.Len(t, event, 10, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
@@ -108,7 +108,7 @@ func TestFileOperatorMatch(t *testing.T) {
 
 	t.Run("caseInsensitive", func(t *testing.T) {
 		resp := "TEST-DATA\r\n1.1.1.1\r\n"
-		event := request.toDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+		event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
 		require.Len(t, event, 10, "could not get correct number of items in dsl map")
 		require.Equal(t, resp, event["raw"], "could not get correct resp")
 
@@ -147,7 +147,7 @@ func TestFileOperatorExtract(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n1.1.1.1\r\n"
-	event := request.toDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
 	require.Len(t, event, 10, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
@@ -265,7 +265,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 	matchedFileName := "test.txt"
 	fileContent := "test-data\r\n1.1.1.1\r\n"
 
-	event := request.toDSLMap(&fileStatus{raw: fileContent, inputFilePath: "/tmp", matchedFileName: matchedFileName})
+	event := request.responseToDSLMap(&fileStatus{raw: fileContent, inputFilePath: "/tmp", matchedFileName: matchedFileName})
 	require.Len(t, event, 10, "could not get correct number of items in dsl map")
 	require.Equal(t, fileContent, event["raw"], "could not get correct resp")
 

v2/pkg/protocols/file/request.go

@@ -13,6 +13,7 @@ import (
 	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
@@ -56,9 +57,19 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 			totalBytes := units.BytesSize(float64(stat.Size()))
 			fileReader := io.LimitReader(file, request.maxSize)
 			var bytesCount, linesCount, wordsCount int
+			isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse
+			var result *operators.Result
 			scanner := bufio.NewScanner(fileReader)
 			buffer := []byte{}
 			scanner.Buffer(buffer, int(chunkSize))
+			outputEvent := request.responseToDSLMap(&fileStatus{
+				inputFilePath:   input,
+				matchedFileName: filePath,
+			})
+			for k, v := range previous {
+				outputEvent[k] = v
+			}
+
 			for scanner.Scan() {
 				fileContent := scanner.Text()
 				n := len(fileContent)
@@ -68,7 +79,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 				processedBytes := units.BytesSize(float64(currentBytes))
 
 				gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytes)
-				outputEvent := request.toDSLMap(&fileStatus{
+				chunkOutputEvent := request.responseToDSLMap(&fileStatus{
 					raw:             fileContent,
 					inputFilePath:   input,
 					matchedFileName: filePath,
@@ -77,20 +88,28 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 					bytes:           bytesCount,
 				})
 				for k, v := range previous {
-					outputEvent[k] = v
+					chunkOutputEvent[k] = v
 				}
 
-				event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
+				chunkEvent := eventcreator.CreateEvent(request, chunkOutputEvent, isResponseDebug)
+				if chunkEvent.OperatorsResult != nil {
 
-				dumpResponse(event, request.options, fileContent, filePath)
-				callback(event)
+					if result == nil {
+						result = chunkEvent.OperatorsResult
+					} else {
+						result.Merge(chunkEvent.OperatorsResult)
+					}
+					dumpResponse(chunkEvent, request.options, filePath, linesCount)
+				}
 
 				currentLinesCount := 1 + strings.Count(fileContent, "\n")
 				linesCount += currentLinesCount
 				wordsCount += strings.Count(fileContent, " ")
 				bytesCount = currentBytes
-				request.options.Progress.IncrementRequests()
+
 			}
+			callback(eventcreator.CreateEventWithResults(request, outputEvent, isResponseDebug, result))
+			request.options.Progress.IncrementRequests()
 		}(data)
 	})
 	wg.Wait()
@@ -102,43 +121,62 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 	return nil
 }
 
-func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, fileContent string, filePath string) {
+func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, filePath string, line int) {
 	cliOptions := requestOptions.Options
 	if cliOptions.Debug || cliOptions.DebugResponse {
+		fileContent := event.InternalEvent["raw"].(string)
 		hexDump := false
 		if responsehighlighter.HasBinaryContent(fileContent) {
 			hexDump = true
 			fileContent = hex.Dump([]byte(fileContent))
 		}
 		highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, fileContent, cliOptions.NoColor, hexDump)
-		gologger.Debug().Msgf("[%s] Dumped file request for %s\n\n%s", requestOptions.TemplateID, filePath, highlightedResponse)
+		gologger.Debug().Msgf("[%s] Dumped match/extract file snippet for %s at line %d\n\n%s", requestOptions.TemplateID, filePath, line, highlightedResponse)
 	}
 }
 
-func getAllStringSubmatchIndex(content string, word string) []int {
+func getAllStringSubmatchIndex(filePath string, word string) []int {
+	file, _ := os.Open(filePath)
+	defer file.Close()
+
 	indexes := []int{}
 
-	start := 0
-	for {
-		v := strings.Index(content[start:], word)
-		if v == -1 {
-			break
+	b := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		content := scanner.Text()
+		if v := strings.Index(content, word); v != -1 {
+			indexes = append(indexes, b+v)
 		}
-		indexes = append(indexes, v+start)
-		start += len(word) + v
+		b += len(content) + 1
 	}
+
 	return indexes
 }
 
-func calculateLineFunc(contents string, linesOffset int, words map[string]struct{}) []int {
+func calculateLineFunc(filePath string, words map[string]struct{}) []int {
 	var lines []int
 
 	for word := range words {
-		matches := getAllStringSubmatchIndex(contents, word)
+		matches := getAllStringSubmatchIndex(filePath, word)
 
 		for _, index := range matches {
-			lineCount := 1 + strings.Count(contents[:index], "\n")
-			lines = append(lines, linesOffset+lineCount)
+			f, _ := os.Open(filePath)
+			scanner := bufio.NewScanner(f)
+
+			lineCount := 0
+			b := 0
+			for scanner.Scan() {
+				lineCount++
+				b += len(scanner.Text()) + 1
+				if b > index {
+					break
+				}
+			}
+			if lineCount > 0 {
+				lines = append(lines, lineCount)
+			}
+			f.Close()
 		}
 	}
 	sort.Ints(lines)

v2/pkg/protocols/file/request_test.go

@@ -79,15 +79,23 @@ func TestFileExecuteWithResults(t *testing.T) {
 }
 
 func TestGenerateNewLineIndexes(t *testing.T) {
-	lines := calculateLineFunc(`aaa
-bbb
-ccc
-RequestDataTooBig
-dddd
-eeee
-RequestDataTooBig
-dd
-RequestDataTooBig3
-SuspiciousOperation`, 0, map[string]struct{}{"SuspiciousOperation": {}, "RequestDataTooBig": {}})
+	tempDir, err := ioutil.TempDir("", "test-*")
+	require.Nil(t, err, "could not create temporary directory")
+	defer os.RemoveAll(tempDir)
+
+	v := `aaa
+		bbb
+		ccc
+		RequestDataTooBig
+		dddd
+		eeee
+		RequestDataTooBig
+		dd
+		RequestDataTooBig3
+		SuspiciousOperation`
+	filename := filepath.Join(tempDir, "test")
+	err = os.WriteFile(filename, []byte(v), os.ModePerm)
+	require.Nil(t, err, "could not write temporary file")
+	lines := calculateLineFunc(filename, map[string]struct{}{"SuspiciousOperation": {}, "RequestDataTooBig": {}})
 	require.ElementsMatch(t, []int{4, 7, 9, 10}, lines, "could not calculate correct lines")
 }