nuclei/pkg/protocols/offlinehttp/request.go

package offlinehttp

import (
	"fmt"
	"io"
	"net/http"
	"net/http/httputil"
	"os"

	"github.com/pkg/errors"

	"github.com/projectdiscovery/gologger"
	"github.com/projectdiscovery/nuclei/v3/pkg/output"
	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
	"github.com/projectdiscovery/utils/conversion"
	syncutil "github.com/projectdiscovery/utils/sync"
	unitutils "github.com/projectdiscovery/utils/unit"
)

var _ protocols.Request = &Request{}

const maxSize = 5 * unitutils.Mega

// Type returns the type of the protocol request
func (request *Request) Type() templateTypes.ProtocolType {
	return templateTypes.OfflineHTTPProtocol
}

// RawInputMode is a flag to indicate if the input is raw input
// rather than a file path
var RawInputMode = false

// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
	if RawInputMode {
		return request.executeRawInput(input.MetaInput.Input, "", input, callback)
	}

	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
	if err != nil {
		return err
	}

	err = request.getInputPaths(input.MetaInput.Input, func(data string) {
		wg.Add()

		go func(data string) {
			defer wg.Done()

			file, err := os.Open(data)
			if err != nil {
				gologger.Error().Msgf("Could not open file path %s: %s\n", data, err)
				return
			}
			defer func() {
				_ = file.Close()
			}()

			stat, err := file.Stat()
			if err != nil {
				gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err)
				return
			}
			if stat.Size() >= int64(maxSize) {
				gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data)
				return
			}

			buffer, err := io.ReadAll(file)
			if err != nil {
				gologger.Error().Msgf("Could not read file path %s: %s\n", data, err)
				return
			}
			dataStr := conversion.String(buffer)

			if err := request.executeRawInput(dataStr, data, input, callback); err != nil {
				gologger.Error().Msgf("Could not execute raw input %s: %s\n", data, err)
				return
			}
		}(data)
	})
	wg.Wait()
	if err != nil {
		request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, "file", err)
		request.options.Progress.IncrementFailedRequestsBy(1)
		return errors.Wrap(err, "could not send file request")
	}
	request.options.Progress.IncrementRequests()
	return nil
}

func (request *Request) executeRawInput(data, inputString string, input *contextargs.Context, callback protocols.OutputEventCallback) error {
	resp, err := readResponseFromString(data)
	if err != nil {
		return errors.Wrap(err, "could not read raw response")
	}

	if request.options.Options.Debug || request.options.Options.DebugRequests {
		gologger.Info().Msgf("[%s] Dumped offline-http request for %s", request.options.TemplateID, data)
		gologger.Print().Msgf("%s", data)
	}
	gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", request.options.TemplateID, data)

	dumpedResponse, err := httputil.DumpResponse(resp, true)
	if err != nil {
		return errors.Wrap(err, "could not dump raw http response")
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return errors.Wrap(err, "could not read raw http response body")
	}
	reqURL := inputString
	if inputString == "" {
		reqURL = getURLFromRequest(resp.Request)
	}

	outputEvent := request.responseToDSLMap(resp, data, reqURL, data, conversion.String(dumpedResponse), conversion.String(body), utils.HeadersToString(resp.Header), 0, nil)
	// add response fields to template context and merge templatectx variables to output event
	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), outputEvent)
	if request.options.HasTemplateCtx(input.MetaInput) {
		outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll())
	}
	outputEvent["ip"] = ""

	event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
	callback(event)
	return nil
}

func getURLFromRequest(req *http.Request) string {
	if req.URL.Scheme == "" {
		req.URL.Scheme = "https"
	}
	return fmt.Sprintf("%s://%s%s", req.URL.Scheme, req.Host, req.URL.Path)
}

// UpdateOptions replaces this request's options with a new copy
func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
	r.options = opts
}