Added line number for file results + stats fixes (#1495)

* Added line number for file results + stats fixes

* Misc

* Improved file result line calculation as per review

* Added new match-all attribute for file template matcher line count

v2/pkg/operators/matchers/match.go

@@ -74,17 +74,19 @@ func (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) (
 		}
 
 		// If the condition was an OR, return on the first match.
-		if matcher.condition == ORCondition {
+		if matcher.condition == ORCondition && !matcher.MatchAll {
 			return true, []string{word}
 		}
-
 		matchedWords = append(matchedWords, word)
 
 		// If we are at the end of the words, return with true
-		if len(matcher.Words)-1 == i {
+		if len(matcher.Words)-1 == i && !matcher.MatchAll {
 			return true, matchedWords
 		}
 	}
+	if len(matchedWords) > 0 && matcher.MatchAll {
+		return true, matchedWords
+	}
 	return false, []string{}
 }
 
@@ -107,17 +109,20 @@ func (matcher *Matcher) MatchRegex(corpus string) (bool, []string) {
 
 		currentMatches := regex.FindAllString(corpus, -1)
 		// If the condition was an OR, return on the first match.
-		if matcher.condition == ORCondition {
+		if matcher.condition == ORCondition && !matcher.MatchAll {
 			return true, currentMatches
 		}
 
 		matchedRegexes = append(matchedRegexes, currentMatches...)
 
 		// If we are at the end of the regex, return with true
-		if len(matcher.regexCompiled)-1 == i {
+		if len(matcher.regexCompiled)-1 == i && !matcher.MatchAll {
 			return true, matchedRegexes
 		}
 	}
+	if len(matchedRegexes) > 0 && matcher.MatchAll {
+		return true, matchedRegexes
+	}
 	return false, []string{}
 }
 

v2/pkg/operators/matchers/matchers.go

@@ -104,6 +104,12 @@ type Matcher struct {
 	//   - false
 	//   - true
 	CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive match,description=use case insensitive match"`
+	// description: |
+	//   MatchAll enables matching for all matcher values. Default is false.
+	// values:
+	//   - false
+	//   - true
+	MatchAll bool `yaml:"match-all,omitempty" jsonschema:"title=match all values,description=match all matcher values ignoring condition"`
 
 	// cached data for the compiled matcher
 	condition     ConditionType

v2/pkg/operators/operators.go

@@ -77,6 +77,9 @@ type Result struct {
 	DynamicValues map[string][]string
 	// PayloadValues contains payload values provided by user. (Optional)
 	PayloadValues map[string]interface{}
+
+	// Optional lineCounts for file protocol
+	LineCount string
 }
 
 // MakeDynamicValuesCallback takes an input dynamic values map and calls

v2/pkg/output/format_screen.go

@@ -2,6 +2,7 @@ package output
 
 import (
 	"bytes"
+	"strconv"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
@@ -64,6 +65,19 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 		builder.WriteString("]")
 	}
 
+	if len(output.LineCount) > 0 {
+		builder.WriteString(" [LN: ")
+
+		for i, line := range output.LineCount {
+			builder.WriteString(strconv.Itoa(line))
+
+			if i != len(output.LineCount)-1 {
+				builder.WriteString(",")
+			}
+		}
+		builder.WriteString("]")
+	}
+
 	// Write meta if any
 	if len(output.Metadata) > 0 {
 		builder.WriteString(" [")

v2/pkg/output/output.go

@@ -104,7 +104,10 @@ type ResultEvent struct {
 	// Only applicable if the report is for HTTP.
 	CURLCommand string `json:"curl-command,omitempty"`
 	// MatcherStatus is the status of the match
-	MatcherStatus       bool           `json:"matcher-status"`
+	MatcherStatus bool `json:"matcher-status"`
+	// LineCount is the line count for the specified match
+	LineCount []int `json:"matched-line"`
+
 	FileToIndexPosition map[string]int `json:"-"`
 }
 

v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go

@@ -24,6 +24,5 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out
 			}
 		}
 	}
-
 	return event
 }

v2/pkg/protocols/file/file.go

@@ -119,5 +119,5 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 // Requests returns the total number of requests the YAML rule will perform
 func (request *Request) Requests() int {
-	return 1
+	return 0
 }

v2/pkg/protocols/file/operators.go

@@ -94,6 +94,24 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []
 		return results
 	}
 
+	for _, result := range results {
+		lineWords := make(map[string]struct{})
+
+		if wrapped.OperatorsResult != nil {
+			for _, value := range wrapped.OperatorsResult.Matches {
+				for _, v := range value {
+					lineWords[v] = struct{}{}
+				}
+			}
+		}
+		if len(result.ExtractedResults) > 0 {
+			for _, v := range result.ExtractedResults {
+				lineWords[v] = struct{}{}
+			}
+		}
+		result.LineCount = calculateLineFunc(rawStr, lineWords)
+	}
+
 	// Identify the position of match in file using a dirty hack.
 	for _, result := range results {
 		for _, extraction := range result.ExtractedResults {

v2/pkg/protocols/file/request.go

@@ -4,6 +4,8 @@ import (
 	"encoding/hex"
 	"io/ioutil"
 	"os"
+	"sort"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/remeh/sizedwaitgroup"
@@ -29,6 +31,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
 
 	err := request.getInputPaths(input, func(data string) {
+		request.options.Progress.AddToTotal(1)
 		wg.Add()
 
 		go func(filePath string) {
@@ -69,6 +72,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 			dumpResponse(event, request.options, fileContent, filePath)
 
 			callback(event)
+			request.options.Progress.IncrementRequests()
 		}(data)
 	})
 	wg.Wait()
@@ -77,7 +81,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return errors.Wrap(err, "could not send file request")
 	}
-	request.options.Progress.IncrementRequests()
 	return nil
 }
 
@@ -93,3 +96,40 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
 		gologger.Debug().Msgf("[%s] Dumped file request for %s\n\n%s", requestOptions.TemplateID, filePath, highlightedResponse)
 	}
 }
+
+func getAllStringSubmatchIndex(content string, word string) []int {
+	indexes := []int{}
+
+	start := 0
+	for {
+		v := strings.Index(content[start:], word)
+		if v == -1 {
+			break
+		}
+		indexes = append(indexes, v+start)
+		start += len(word) + v
+	}
+	return indexes
+}
+
+func calculateLineFunc(contents string, words map[string]struct{}) []int {
+	var lines []int
+
+	for word := range words {
+		matches := getAllStringSubmatchIndex(contents, word)
+
+		for _, index := range matches {
+			lineCount := int(0)
+			for _, c := range contents[:index] {
+				if c == '\n' {
+					lineCount++
+				}
+			}
+			if lineCount > 0 {
+				lines = append(lines, lineCount+1)
+			}
+		}
+	}
+	sort.Ints(lines)
+	return lines
+}

v2/pkg/protocols/file/request_test.go

@@ -77,3 +77,17 @@ func TestFileExecuteWithResults(t *testing.T) {
 	require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
 	finalEvent = nil
 }
+
+func TestGenerateNewLineIndexes(t *testing.T) {
+	lines := calculateLineFunc(`aaa
+bbb
+ccc
+RequestDataTooBig
+dddd
+eeee
+RequestDataTooBig
+dd
+RequestDataTooBig3
+SuspiciousOperation`, map[string]struct{}{"SuspiciousOperation": {}, "RequestDataTooBig": {}})
+	require.ElementsMatch(t, []int{4, 7, 9, 10}, lines, "could not calculate correct lines")
+}