diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index c580d43c4..c822480ed 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -23,8 +23,8 @@ builds: flags: - -trimpath -- main: cmd/cve-annotate/main.go - binary: cve-annotate +- main: cmd/tmc/main.go + binary: tmc id: annotate env: diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go deleted file mode 100644 index 127cd5c62..000000000 --- a/v2/cmd/cve-annotate/main.go +++ /dev/null @@ -1,650 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "io" - "log" - "net/http" - "net/url" - "os" - "path/filepath" - "reflect" - "regexp" - "sort" - "strconv" - "strings" - - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/projectdiscovery/nvd" - "github.com/projectdiscovery/retryablehttp-go" - sliceutil "github.com/projectdiscovery/utils/slice" - stringsutil "github.com/projectdiscovery/utils/strings" - "gopkg.in/yaml.v3" -) - -const ( - yamlIndentSpaces = 2 -) - -var cisaKnownExploitedVulnerabilities map[string]struct{} - -// allTagsRegex is a list of all tags in nuclei templates except id, info, and - -var allTagsRegex []*regexp.Regexp -var defaultOpts = types.DefaultOptions() - -func init() { - var tm templates.Template - t := reflect.TypeOf(tm) - for i := 0; i < t.NumField(); i++ { - tag := t.Field(i).Tag.Get("yaml") - if strings.Contains(tag, ",") { - tag = strings.Split(tag, ",")[0] - } - // ignore these tags - if tag == "id" || tag == "info" || tag == "" || tag == "-" { - continue - } - re := regexp.MustCompile(tag + `:\s*\n`) - allTagsRegex = append(allTagsRegex, re) - } - - defaultOpts := types.DefaultOptions() - // need to set headless to true for headless templates - defaultOpts.Headless = true - if err := protocolstate.Init(defaultOpts); err != nil { - gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) - } - if err := protocolinit.Init(defaultOpts); err != nil { - gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) - } - if err := fetchCISAKnownExploitedVulnerabilities(); err != nil { - panic(err) - } -} - -var ( - input = flag.String("i", "", "Templates to annotate") - verbose = flag.Bool("v", false, "show verbose output") -) - -func main() { - flag.Parse() - - if *input == "" { - log.Fatalf("invalid input, see -h\n") - } - if strings.HasPrefix(*input, "~/") { - home, err := os.UserHomeDir() - if err != nil { - log.Fatalf("Failed to read UserHomeDir: %v, provide absolute template path/directory\n", err) - } - *input = filepath.Join(home, (*input)[2:]) - } - gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) - if *verbose { - gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) - } - if err := process(); err != nil { - gologger.Error().Msgf("could not process: %s\n", err) - } -} - -func process() error { - tempDir, err := os.MkdirTemp("", "nuclei-nvd-%s") - if err != nil { - return err - } - defer os.RemoveAll(tempDir) - - client := nvd.NewClientV2() - templateCatalog := disk.NewCatalog(filepath.Dir(*input)) - paths, err := templateCatalog.GetTemplatePath(*input) - if err != nil { - return err - } - for _, path := range paths { - data, err := os.ReadFile(path) - if err != nil { - return err - } - dataString := string(data) - // try to fill max-requests - dataString, err = parseAndAddMaxRequests(templateCatalog, path, dataString) - if err != nil { - gologger.Error().Msgf("Could not compile max request %s: %s\n", path, err) - } - // try to resolve references to tags - dataString, err = parseAndAddReferenceBasedTags(path, dataString) - if err != nil { - gologger.Error().Msgf("Could not parse reference tags %s: %s\n", path, err) - continue - } - // try and fill CVE data - getCVEData(client, path, dataString) - } - return nil -} - -var ( - idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)") - severityRegex = regexp.MustCompile(`severity: ([a-z]+)`) -) - -const maxReferenceCount = 5 - -// dead sites to skip for references -var badRefs = []string{ - "osvdb.org/", - "securityfocus.com/", - "archives.neohapsis.com/", - "iss.net/", - "ntelbras.com/", - "andmp.com/", - "blacklanternsecurity.com/", - "pwnwiki.org/", - "0dayhack.net/", - "correkt.horse/", - "poc.wgpsec.org/", - "ctf-writeup.revers3c.com/", - "secunia.com/", -} - -func getCVEData(client *nvd.ClientV2, filePath, data string) { - matches := idRegex.FindAllStringSubmatch(data, 1) - if len(matches) == 0 { - return - } - cveName := matches[0][1] - - // Perform CISA Known-exploited-vulnerabilities tag annotation - // if we discover it has been exploited. - var err error - if cisaKnownExploitedVulnerabilities != nil { - _, ok := cisaKnownExploitedVulnerabilities[strings.ToLower(cveName)] - if ok { - data, err = parseAndAddCISAKevTagTemplate(filePath, data) - } - } - if err != nil { - gologger.Error().Msgf("Could not parse cisa data %s: %s\n", cveName, err) - return - } - - severityMatches := severityRegex.FindAllStringSubmatch(data, 1) - if len(severityMatches) == 0 { - return - } - severityValue := severityMatches[0][1] - - cveItem, err := client.FetchCVE(cveName) - if err != nil { - gologger.Error().Msgf("Could not fetch cve %s: %s\n", cveName, err) - return - } - var cweID []string - for _, weaknessData := range cveItem.Cve.Weaknesses { - for _, description := range weaknessData.Description { - cweID = append(cweID, description.Value) - } - } - cvssData, err := getPrimaryCVSSData(cveItem) - if err != nil { - gologger.Error().Msgf("Could not get CVSS data %s: %s\n", cveName, err) - return - } - cvssScore := cvssData.BaseScore - cvssMetrics := cvssData.VectorString - - // Perform some hacky string replacement to place the metadata in templates - infoBlockIndexData := data[strings.Index(data, "info:"):] - requestsIndex := strings.Index(infoBlockIndexData, "requests:") - networkIndex := strings.Index(infoBlockIndexData, "network:") - variablesIndex := strings.Index(infoBlockIndexData, "variables:") - if requestsIndex == -1 && networkIndex == -1 && variablesIndex == -1 { - return - } - if networkIndex != -1 { - requestsIndex = networkIndex - } - if variablesIndex != -1 { - requestsIndex = variablesIndex - } - infoBlockData := infoBlockIndexData[:requestsIndex] - infoBlockClean := strings.TrimRight(infoBlockData, "\n") - - infoBlock := InfoBlock{} - err = yaml.Unmarshal([]byte(data), &infoBlock) - if err != nil { - gologger.Warning().Msgf("Could not unmarshal info block: %s\n", err) - } - - var changed bool - if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" { - changed = true - infoBlock.Info.Severity = newSeverity - gologger.Info().Msgf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore) - } - isCvssEmpty := cvssScore == 0 || cvssMetrics == "" - hasCvssChanged := infoBlock.Info.Classification.CvssScore != cvssScore || cvssMetrics != infoBlock.Info.Classification.CvssMetrics - if !isCvssEmpty && hasCvssChanged { - changed = true - infoBlock.Info.Classification.CvssMetrics = cvssMetrics - infoBlock.Info.Classification.CvssScore = cvssScore - infoBlock.Info.Classification.CveId = cveName - if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") { - infoBlock.Info.Classification.CweId = strings.Join(cweID, ",") - } - } - // If there is no description field, fill the description from CVE information - enDescription, err := getEnglishLangString(cveItem.Cve.Descriptions) - hasDescriptionData := err != nil - isDescriptionEmpty := infoBlock.Info.Description == "" - if isDescriptionEmpty && hasDescriptionData { - changed = true - // removes all new lines - description := stringsutil.ReplaceAll(enDescription, "", "\n", "\\", "'", "\t") - description += "\n" - infoBlock.Info.Description = description - } - - // we are unmarshaling info block to have valid data - var referenceDataURLs []string - - // skip sites that are no longer alive - for _, reference := range cveItem.Cve.References { - if stringsutil.ContainsAny(reference.URL, badRefs...) { - continue - } - referenceDataURLs = append(referenceDataURLs, reference.URL) - } - hasReferenceData := len(cveItem.Cve.References) > 0 - areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs) - referencesCount := len(infoBlock.Info.Reference) - if hasReferenceData && !areCveReferencesContained { - changed = true - for _, ref := range referenceDataURLs { - referencesCount++ - if referencesCount >= maxReferenceCount { - break - } - infoBlock.Info.Reference = append(infoBlock.Info.Reference, ref) - } - infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference)) - } - - cpeSet := map[string]bool{} - for _, config := range cveItem.Cve.Configurations { - // Right now this covers only simple configurations. More complex configurations can have multiple CPEs - if len(config.Nodes) == 1 { - changed = true - node := config.Nodes[0] - for _, match := range node.CpeMatch { - cpeSet[extractVersionlessCpe(match.Criteria)] = true - } - } - } - uniqueCpes := make([]string, 0, len(cpeSet)) - for k := range cpeSet { - uniqueCpes = append(uniqueCpes, k) - } - if len(uniqueCpes) == 1 { - infoBlock.Info.Classification.Cpe = uniqueCpes[0] - } - - epss, err := fetchEpss(cveName) - if err != nil { - log.Printf("Could not fetch Epss score: %s\n", err) - return - } - hasEpssChanged := epss != infoBlock.Info.Classification.EpssScore - if hasEpssChanged { - changed = true - infoBlock.Info.Classification.EpssScore = epss - } - - var newInfoBlock bytes.Buffer - yamlEncoder := yaml.NewEncoder(&newInfoBlock) - yamlEncoder.SetIndent(yamlIndentSpaces) - err = yamlEncoder.Encode(infoBlock) - if err != nil { - gologger.Warning().Msgf("Could not marshal info block: %s\n", err) - return - } - newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n") - - newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlockData) - if changed { - _ = os.WriteFile(filePath, []byte(newTemplate), 0644) - gologger.Info().Msgf("Wrote updated template to %s\n", filePath) - } -} - -func getPrimaryCVSSData(vuln nvd.Vulnerability) (nvd.CvssData, error) { - for _, data := range vuln.Cve.Metrics.CvssMetricV31 { - if data.Type == "Primary" { - return data.CvssData, nil - } - } - for _, data := range vuln.Cve.Metrics.CvssMetricV3 { - if data.Type == "Primary" { - return data.CvssData, nil - } - } - return nvd.CvssData{}, fmt.Errorf("no primary cvss metric found") -} - -func getEnglishLangString(data []nvd.LangString) (string, error) { - for _, item := range data { - if item.Lang == "en" { - return item.Value, nil - } - } - return "", fmt.Errorf("no english item found") -} - -func isSeverityMatchingCvssScore(severity string, score float64) string { - if score == 0.0 { - return "" - } - var expected string - - if score >= 0.1 && score <= 3.9 { - expected = "low" - } else if score >= 4.0 && score <= 6.9 { - expected = "medium" - } else if score >= 7.0 && score <= 8.9 { - expected = "high" - } else if score >= 9.0 && score <= 10.0 { - expected = "critical" - } - if expected != "" && expected != severity { - return expected - } - return "" -} - -func extractVersionlessCpe(cpe string) string { - parts := strings.Split(cpe, ":") - versionlessPart := parts[0:5] - rest := strings.Split(strings.Repeat("*", len(parts)-len(versionlessPart)), "") - return strings.Join(append(versionlessPart, rest...), ":") -} - -type ApiFirstEpssResponse struct { - Status string `json:"status"` - StatusCode int `json:"status-code"` - Version string `json:"version"` - Access string `json:"access"` - Total int `json:"total"` - Offset int `json:"offset"` - Limit int `json:"limit"` - Data []struct { - Cve string `json:"cve"` - Epss string `json:"epss"` - Percentile string `json:"percentile"` - Date string `json:"date"` - } `json:"data"` -} - -func fetchEpss(cveId string) (float64, error) { - resp, err := http.Get(fmt.Sprintf("https://api.first.org/data/v1/epss?cve=%s", cveId)) - if err != nil { - return 0, fmt.Errorf("unable to fetch EPSS data from first.org: %v", err) - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return 0, fmt.Errorf("unable to read reponse body: %v", err) - } - var parsedResp ApiFirstEpssResponse - err = json.Unmarshal(body, &parsedResp) - if err != nil { - return 0, fmt.Errorf("error while parsing EPSS response: %v", err) - } - if len(parsedResp.Data) != 1 { - return 0, fmt.Errorf("unexpected number of results in EPSS response. Expecting exactly 1, got %v", len(parsedResp.Data)) - } - epss := parsedResp.Data[0].Epss - return strconv.ParseFloat(epss, 64) -} - -type cisaKEVData struct { - Vulnerabilities []struct { - CVEID string `json:"cveID"` - } -} - -// fetchCISAKnownExploitedVulnerabilities fetches CISA known exploited -// vulnerabilities catalog for template tag enrichment -func fetchCISAKnownExploitedVulnerabilities() error { - data := &cisaKEVData{} - - resp, err := retryablehttp.DefaultClient().Get("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json") - if err != nil { - return errors.Wrap(err, "could not get cisa kev catalog") - } - defer resp.Body.Close() - - if err := json.NewDecoder(resp.Body).Decode(data); err != nil { - return errors.Wrap(err, "could not decode cisa kev catalog json data") - } - cisaKnownExploitedVulnerabilities = make(map[string]struct{}) - for _, vuln := range data.Vulnerabilities { - cisaKnownExploitedVulnerabilities[strings.ToLower(vuln.CVEID)] = struct{}{} - } - return nil -} - -// parseAndAddCISAKevTagTemplate parses and adds `kev` tag to CISA KEV templates. -// also removes cisa tag if it exists -func parseAndAddCISAKevTagTemplate(path string, data string) (string, error) { - block := &InfoBlock{} - - if err := yaml.NewDecoder(strings.NewReader(data)).Decode(block); err != nil { - return "", errors.Wrap(err, "could not decode template yaml") - } - splitted := strings.Split(block.Info.Tags, ",") - if len(splitted) == 0 { - return data, nil - } - - var cisaIndex = -1 - for i, tag := range splitted { - // If we already have tag, return - if tag == "kev" { - return data, nil - } - if tag == "cisa" { - cisaIndex = i - } - } - // Remove CISA index tag element - if cisaIndex >= 0 { - splitted = append(splitted[:cisaIndex], splitted[cisaIndex+1:]...) - } - splitted = append(splitted, "kev") - replaced := strings.ReplaceAll(data, block.Info.Tags, strings.Join(splitted, ",")) - return replaced, os.WriteFile(path, []byte(replaced), os.ModePerm) -} - -// parseAndAddReferenceBasedTags parses and adds reference based tags to templates -func parseAndAddReferenceBasedTags(path string, data string) (string, error) { - block := &InfoBlock{} - if err := yaml.NewDecoder(strings.NewReader(data)).Decode(block); err != nil { - return "", errors.Wrap(err, "could not decode template yaml") - } - splitted := strings.Split(block.Info.Tags, ",") - if len(splitted) == 0 { - return data, nil - } - tagsCurrent := fmt.Sprintf("tags: %s", block.Info.Tags) - newTags := suggestTagsBasedOnReference(block.Info.Reference, splitted) - - if len(newTags) == len(splitted) { - return data, nil - } - replaced := strings.ReplaceAll(data, tagsCurrent, fmt.Sprintf("tags: %s", strings.Join(newTags, ","))) - return replaced, os.WriteFile(path, []byte(replaced), os.ModePerm) -} - -var referenceMapping = map[string]string{ - "huntr.dev": "huntr", - "hackerone.com": "hackerone", - "tenable.com": "tenable", - "packetstormsecurity.org": "packetstorm", - "seclists.org": "seclists", - "wpscan.com": "wpscan", - "packetstormsecurity.com": "packetstorm", - "exploit-db.com": "edb", - "https://github.com/rapid7/metasploit-framework/": "msf", - "https://github.com/vulhub/vulhub/": "vulhub", -} - -func suggestTagsBasedOnReference(references, currentTags []string) []string { - uniqueTags := make(map[string]struct{}) - for _, value := range currentTags { - uniqueTags[value] = struct{}{} - } - - for _, reference := range references { - parsed, err := url.Parse(reference) - if err != nil { - continue - } - hostname := parsed.Hostname() - - for value, tag := range referenceMapping { - if strings.HasSuffix(hostname, value) || strings.HasPrefix(reference, value) { - uniqueTags[tag] = struct{}{} - } - } - } - newTags := make([]string, 0, len(uniqueTags)) - for tag := range uniqueTags { - newTags = append(newTags, tag) - } - return newTags -} - -// InfoBlock Cloning struct from nuclei as we don't want any validation -type InfoBlock struct { - Info TemplateInfo `yaml:"info"` -} - -type TemplateClassification struct { - CvssMetrics string `yaml:"cvss-metrics,omitempty"` - CvssScore float64 `yaml:"cvss-score,omitempty"` - CveId string `yaml:"cve-id,omitempty"` - CweId string `yaml:"cwe-id,omitempty"` - Cpe string `yaml:"cpe,omitempty"` - EpssScore float64 `yaml:"epss-score,omitempty"` -} - -type TemplateInfo struct { - Name string `yaml:"name"` - Author string `yaml:"author"` - Severity string `yaml:"severity,omitempty"` - Description string `yaml:"description,omitempty"` - Reference []string `yaml:"reference,omitempty"` - Remediation string `yaml:"remediation,omitempty"` - Classification TemplateClassification `yaml:"classification,omitempty"` - Metadata map[string]interface{} `yaml:"metadata,omitempty"` - Tags string `yaml:"tags,omitempty"` -} - -// parseAndAddMaxRequests parses and adds max requests to templates -func parseAndAddMaxRequests(catalog catalog.Catalog, path, data string) (string, error) { - template, err := parseTemplate(catalog, path) - if err != nil { - gologger.Warning().Label("max-request").Msgf("Could not parse template: %s\n", err) - return data, err - } - - if template.TotalRequests < 1 { - return data, nil - } - // Marshal the updated info block back to YAML. - infoBlockStart, infoBlockEnd := getInfoStartEnd(data) - infoBlockOrig := data[infoBlockStart:infoBlockEnd] - infoBlockOrig = strings.TrimRight(infoBlockOrig, "\n") - - infoBlock := InfoBlock{} - err = yaml.Unmarshal([]byte(data), &infoBlock) - if err != nil { - gologger.Warning().Label("max-request").Msgf("Could not unmarshal info block: %s\n", err) - return data, err - } - // if metadata is nil, create a new map - if infoBlock.Info.Metadata == nil { - infoBlock.Info.Metadata = make(map[string]interface{}) - } - // do not update if it is already present and equal - if mr, ok := infoBlock.Info.Metadata["max-request"]; ok && mr.(int) == template.TotalRequests { - return data, nil - } - infoBlock.Info.Metadata["max-request"] = template.TotalRequests - - var newInfoBlock bytes.Buffer - yamlEncoder := yaml.NewEncoder(&newInfoBlock) - yamlEncoder.SetIndent(yamlIndentSpaces) - err = yamlEncoder.Encode(infoBlock) - if err != nil { - gologger.Warning().Msgf("Could not marshal info block: %s\n", err) - return data, err - } - newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n") - - // replace old info block with new info block - newTemplate := strings.ReplaceAll(data, infoBlockOrig, newInfoBlockData) - - err = os.WriteFile(path, []byte(newTemplate), 0644) - if err == nil { - gologger.Info().Label("max-request").Msgf("Wrote updated template to %s\n", path) - } - return newTemplate, err -} - -// parseTemplate parses a template and returns the template object -func parseTemplate(catalog catalog.Catalog, templatePath string) (*templates.Template, error) { - executorOpts := protocols.ExecutorOptions{ - Catalog: catalog, - Options: defaultOpts, - } - reader, err := executorOpts.Catalog.OpenFile(templatePath) - if err != nil { - return nil, err - } - template, err := templates.ParseTemplateFromReader(reader, nil, executorOpts) - if err != nil { - return nil, err - } - return template, nil -} - -// find the start and end of the info block -func getInfoStartEnd(data string) (int, int) { - info := strings.Index(data, "info:") - var indices []int - for _, re := range allTagsRegex { - // find the first occurrence of the label - match := re.FindStringIndex(data) - if match != nil { - indices = append(indices, match[0]) - } - } - // find the first one after info block - sort.Ints(indices) - return info, indices[0] - 1 -} diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go new file mode 100644 index 000000000..97f8d6122 --- /dev/null +++ b/v2/cmd/tmc/main.go @@ -0,0 +1,435 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" + "gopkg.in/yaml.v3" +) + +const ( + yamlIndentSpaces = 2 + // templateman api base url + tmBaseUrlDefault = "https://tm.nuclei.sh" +) + +var tmBaseUrl string + +func init() { + tmBaseUrl = os.Getenv("TEMPLATEMAN_SERVER") + if tmBaseUrl == "" { + tmBaseUrl = tmBaseUrlDefault + } +} + +// allTagsRegex is a list of all tags in nuclei templates except id, info, and - +var allTagsRegex []*regexp.Regexp +var defaultOpts = types.DefaultOptions() + +func init() { + var tm templates.Template + t := reflect.TypeOf(tm) + for i := 0; i < t.NumField(); i++ { + tag := t.Field(i).Tag.Get("yaml") + if strings.Contains(tag, ",") { + tag = strings.Split(tag, ",")[0] + } + // ignore these tags + if tag == "id" || tag == "info" || tag == "" || tag == "-" { + continue + } + re := regexp.MustCompile(tag + `:\s*\n`) + if t.Field(i).Type.Kind() == reflect.Bool { + re = regexp.MustCompile(tag + `:\s*(true|false)\s*\n`) + } + allTagsRegex = append(allTagsRegex, re) + } + + defaultOpts := types.DefaultOptions() + // need to set headless to true for headless templates + defaultOpts.Headless = true + if err := protocolstate.Init(defaultOpts); err != nil { + gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) + } + if err := protocolinit.Init(defaultOpts); err != nil { + gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) + } +} + +var idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)") + +type options struct { + input string + errorLogFile string + lint bool + validate bool + format bool + enhance bool + maxRequest bool + debug bool +} + +func main() { + opts := options{} + flagSet := goflags.NewFlagSet() + flagSet.SetDescription(`TemplateMan CLI is baisc utility built on the TemplateMan API to standardize nuclei templates.`) + + flagSet.CreateGroup("Input", "input", + flagSet.StringVarP(&opts.input, "input", "i", "", "Templates to annotate"), + ) + + flagSet.CreateGroup("Config", "config", + flagSet.BoolVarP(&opts.lint, "lint", "l", false, "lint given nuclei template"), + flagSet.BoolVarP(&opts.validate, "validate", "v", false, "validate given nuclei template"), + flagSet.BoolVarP(&opts.format, "format", "f", false, "format given nuclei template"), + flagSet.BoolVarP(&opts.enhance, "enhance", "e", false, "enhance given nuclei template"), + flagSet.BoolVarP(&opts.maxRequest, "max-request", "mr", false, "add / update max request counter"), + flagSet.StringVarP(&opts.errorLogFile, "error-log", "el", "", "file to write failed template update"), + flagSet.BoolVarP(&opts.debug, "debug", "d", false, "show debug message"), + ) + + if err := flagSet.Parse(); err != nil { + gologger.Fatal().Msgf("Error parsing flags: %s\n", err) + } + + if opts.input == "" { + gologger.Fatal().Msg("input template path/directory is required") + } + if strings.HasPrefix(opts.input, "~/") { + home, err := os.UserHomeDir() + if err != nil { + log.Fatalf("Failed to read UserHomeDir: %v, provide absolute template path/directory\n", err) + } + opts.input = filepath.Join(home, (opts.input)[2:]) + } + gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) + if opts.debug { + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) + } + if err := process(opts); err != nil { + gologger.Error().Msgf("could not process: %s\n", err) + } +} + +func process(opts options) error { + tempDir, err := os.MkdirTemp("", "nuclei-nvd-%s") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + var errFile *os.File + if opts.errorLogFile != "" { + errFile, err = os.OpenFile(opts.errorLogFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + gologger.Fatal().Msgf("could not open error log file: %s\n", err) + } + defer errFile.Close() + } + + templateCatalog := disk.NewCatalog(filepath.Dir(opts.input)) + paths, err := templateCatalog.GetTemplatePath(opts.input) + if err != nil { + return err + } + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + return err + } + dataString := string(data) + + if opts.maxRequest { + var updated bool // if max-requests is updated + dataString, updated, err = parseAndAddMaxRequests(templateCatalog, path, dataString) + if err != nil { + gologger.Info().Label("max-request").Msgf(logErrMsg(path, err, opts.debug, errFile)) + } else { + if updated { + gologger.Info().Label("max-request").Msgf("✅ updated template: %s\n", path) + } + // do not print if max-requests is not updated + } + } + + if opts.lint { + lint, err := lintTemplate(dataString) + if err != nil { + gologger.Info().Label("lint").Msg(logErrMsg(path, err, opts.debug, errFile)) + } + if lint { + gologger.Info().Label("lint").Msgf("✅ lint template: %s\n", path) + } + } + + if opts.validate { + validate, err := validateTemplate(dataString) + if err != nil { + gologger.Info().Label("validate").Msg(logErrMsg(path, err, opts.debug, errFile)) + } + if validate { + gologger.Info().Label("validate").Msgf("✅ validated template: %s\n", path) + } + } + + if opts.format { + formatedTemplateData, isFormated, err := formatTemplate(dataString) + if err != nil { + gologger.Info().Label("format").Msg(logErrMsg(path, err, opts.debug, errFile)) + } else { + if isFormated { + _ = os.WriteFile(path, []byte(formatedTemplateData), 0644) + dataString = formatedTemplateData + gologger.Info().Label("format").Msgf("✅ formated template: %s\n", path) + } + } + } + + if opts.enhance { + // currently enhance api only supports cve-id's + matches := idRegex.FindAllStringSubmatch(dataString, 1) + if len(matches) == 0 { + continue + } + enhancedTemplateData, isEnhanced, err := enhanceTemplate(dataString) + if err != nil { + gologger.Info().Label("enhance").Msg(logErrMsg(path, err, opts.debug, errFile)) + continue + } else { + if isEnhanced { + _ = os.WriteFile(path, []byte(enhancedTemplateData), 0644) + gologger.Info().Label("enhance").Msgf("✅ updated template: %s\n", path) + } + } + } + } + return nil +} + +func logErrMsg(path string, err error, debug bool, errFile *os.File) string { + msg := fmt.Sprintf("❌ template: %s\n", path) + if debug { + msg = fmt.Sprintf("❌ template: %s err: %s\n", path, err) + } + if errFile != nil { + _, _ = errFile.WriteString(fmt.Sprintf("❌ template: %s err: %s\n", path, err)) + } + return msg +} + +// enhanceTemplateData enhances template data using templateman +// ref: https://github.com/projectdiscovery/templateman/blob/main/templateman-rest-api/README.md#enhance-api +func enhanceTemplate(data string) (string, bool, error) { + resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/enhance", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) + if err != nil { + return data, false, err + } + if resp.StatusCode != 200 { + return data, false, errorutil.New("unexpected status code: %v", resp.Status) + } + var templateResp TemplateResp + if err := json.NewDecoder(resp.Body).Decode(&templateResp); err != nil { + return data, false, err + } + if strings.TrimSpace(templateResp.Enhanced) != "" { + return templateResp.Enhanced, templateResp.Enhance, nil + } + if templateResp.ValidateErrorCount > 0 { + if len(templateResp.ValidateError) > 0 { + return data, false, errorutil.NewWithTag("validate", templateResp.ValidateError[0].Message+": at line %v", templateResp.ValidateError[0].Mark.Line) + } + return data, false, errorutil.New("validation failed").WithTag("validate") + } + if templateResp.Error.Name != "" { + return data, false, errorutil.New(templateResp.Error.Name) + } + if strings.TrimSpace(templateResp.Enhanced) == "" && !templateResp.Lint { + if templateResp.LintError.Reason != "" { + return data, false, errorutil.NewWithTag("lint", templateResp.LintError.Reason+" : at line %v", templateResp.LintError.Mark.Line) + } + return data, false, errorutil.NewWithTag("lint", "at line: %v", templateResp.LintError.Mark.Line) + } + return data, false, errorutil.New("template enhance failed") +} + +// formatTemplateData formats template data using templateman format api +func formatTemplate(data string) (string, bool, error) { + resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/format", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) + if err != nil { + return data, false, err + } + if resp.StatusCode != 200 { + return data, false, errorutil.New("unexpected status code: %v", resp.Status) + } + var templateResp TemplateResp + if err := json.NewDecoder(resp.Body).Decode(&templateResp); err != nil { + return data, false, err + } + if strings.TrimSpace(templateResp.Updated) != "" { + return templateResp.Updated, templateResp.Format, nil + } + if templateResp.ValidateErrorCount > 0 { + if len(templateResp.ValidateError) > 0 { + return data, false, errorutil.NewWithTag("validate", templateResp.ValidateError[0].Message+": at line %v", templateResp.ValidateError[0].Mark.Line) + } + return data, false, errorutil.New("validation failed").WithTag("validate") + } + if templateResp.Error.Name != "" { + return data, false, errorutil.New(templateResp.Error.Name) + } + if strings.TrimSpace(templateResp.Updated) == "" && !templateResp.Lint { + if templateResp.LintError.Reason != "" { + return data, false, errorutil.NewWithTag("lint", templateResp.LintError.Reason+" : at line %v", templateResp.LintError.Mark.Line) + } + return data, false, errorutil.NewWithTag("lint", "at line: %v", templateResp.LintError.Mark.Line) + } + return data, false, errorutil.New("template format failed") +} + +// lintTemplateData lints template data using templateman lint api +func lintTemplate(data string) (bool, error) { + resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/lint", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) + if err != nil { + return false, err + } + if resp.StatusCode != 200 { + return false, errorutil.New("unexpected status code: %v", resp.Status) + } + var lintResp TemplateLintResp + if err := json.NewDecoder(resp.Body).Decode(&lintResp); err != nil { + return false, err + } + if lintResp.Lint { + return true, nil + } + if lintResp.LintError.Reason != "" { + return false, errorutil.NewWithTag("lint", lintResp.LintError.Reason+" : at line %v", lintResp.LintError.Mark.Line) + } + return false, errorutil.NewWithTag("lint", "at line: %v", lintResp.LintError.Mark.Line) +} + +// validateTemplate validates template data using templateman validate api +func validateTemplate(data string) (bool, error) { + resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/validate", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) + if err != nil { + return false, err + } + if resp.StatusCode != 200 { + return false, errorutil.New("unexpected status code: %v", resp.Status) + } + var validateResp TemplateResp + if err := json.NewDecoder(resp.Body).Decode(&validateResp); err != nil { + return false, err + } + if validateResp.Validate { + return true, nil + } + if validateResp.ValidateErrorCount > 0 { + if len(validateResp.ValidateError) > 0 { + return false, errorutil.NewWithTag("validate", validateResp.ValidateError[0].Message+": at line %v", validateResp.ValidateError[0].Mark.Line) + } + return false, errorutil.New("validation failed").WithTag("validate") + } + if validateResp.Error.Name != "" { + return false, errorutil.New(validateResp.Error.Name) + } + return false, errorutil.New("template validation failed") +} + +// parseAndAddMaxRequests parses and adds max requests to templates +func parseAndAddMaxRequests(catalog catalog.Catalog, path, data string) (string, bool, error) { + template, err := parseTemplate(catalog, path) + if err != nil { + return data, false, err + } + if template.TotalRequests < 1 { + return data, false, nil + } + // Marshal the updated info block back to YAML. + infoBlockStart, infoBlockEnd := getInfoStartEnd(data) + infoBlockOrig := data[infoBlockStart:infoBlockEnd] + infoBlockOrig = strings.TrimRight(infoBlockOrig, "\n") + infoBlock := InfoBlock{} + err = yaml.Unmarshal([]byte(data), &infoBlock) + if err != nil { + return data, false, err + } + // if metadata is nil, create a new map + if infoBlock.Info.Metadata == nil { + infoBlock.Info.Metadata = make(map[string]interface{}) + } + // do not update if it is already present and equal + if mr, ok := infoBlock.Info.Metadata["max-request"]; ok && mr.(int) == template.TotalRequests { + return data, false, nil + } + infoBlock.Info.Metadata["max-request"] = template.TotalRequests + + var newInfoBlock bytes.Buffer + yamlEncoder := yaml.NewEncoder(&newInfoBlock) + yamlEncoder.SetIndent(yamlIndentSpaces) + err = yamlEncoder.Encode(infoBlock) + if err != nil { + return data, false, err + } + newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n") + // replace old info block with new info block + newTemplate := strings.ReplaceAll(data, infoBlockOrig, newInfoBlockData) + err = os.WriteFile(path, []byte(newTemplate), 0644) + if err == nil { + return newTemplate, true, nil + } + return newTemplate, false, err +} + +// parseTemplate parses a template and returns the template object +func parseTemplate(catalog catalog.Catalog, templatePath string) (*templates.Template, error) { + executorOpts := protocols.ExecutorOptions{ + Catalog: catalog, + Options: defaultOpts, + } + reader, err := executorOpts.Catalog.OpenFile(templatePath) + if err != nil { + return nil, err + } + template, err := templates.ParseTemplateFromReader(reader, nil, executorOpts) + if err != nil { + return nil, err + } + return template, nil +} + +// find the start and end of the info block +func getInfoStartEnd(data string) (int, int) { + info := strings.Index(data, "info:") + var indices []int + for _, re := range allTagsRegex { + // find the first occurrence of the label + match := re.FindStringIndex(data) + if match != nil { + indices = append(indices, match[0]) + } + } + // find the first one after info block + sort.Ints(indices) + return info, indices[0] - 1 +} diff --git a/v2/cmd/tmc/types.go b/v2/cmd/tmc/types.go new file mode 100644 index 000000000..058746409 --- /dev/null +++ b/v2/cmd/tmc/types.go @@ -0,0 +1,80 @@ +package main + +type Mark struct { + Name string `json:"name,omitempty"` + Position int `json:"position,omitempty"` + Line int `json:"line,omitempty"` + Column int `json:"column,omitempty"` + Snippet string `json:"snippet,omitempty"` +} + +type Error struct { + Name string `json:"name"` + Mark Mark `json:"mark"` +} + +type LintError struct { + Name string `json:"name,omitempty"` + Reason string `json:"reason,omitempty"` + Mark Mark `json:"mark,omitempty"` +} + +type TemplateLintResp struct { + Input string `json:"template_input,omitempty"` + Lint bool `json:"template_lint,omitempty"` + LintError LintError `json:"lint_error,omitempty"` +} + +type ValidateError struct { + Location string `json:"location,omitempty"` + Message string `json:"message,omitempty"` + Name string `json:"name,omitempty"` + Argument interface{} `json:"argument,omitempty"` + Stack string `json:"stack,omitempty"` + Mark struct { + Line int `json:"line,omitempty"` + Column int `json:"column,omitempty"` + Pos int `json:"pos,omitempty"` + } `json:"mark,omitempty"` +} + +// TemplateResponse from templateman to be used for enhancing and formatting +type TemplateResp struct { + Input string `json:"template_input,omitempty"` + Format bool `json:"template_format,omitempty"` + Updated string `json:"updated_template,omitempty"` + Enhance bool `json:"template_enhance,omitempty"` + Enhanced string `json:"enhanced_template,omitempty"` + Lint bool `json:"template_lint,omitempty"` + LintError LintError `json:"lint_error,omitempty"` + Validate bool `json:"template_validate,omitempty"` + ValidateErrorCount int `json:"validate_error_count,omitempty"` + ValidateError []ValidateError `json:"validate_error,omitempty"` + Error Error `json:"error,omitempty"` +} + +// InfoBlock Cloning struct from nuclei as we don't want any validation +type InfoBlock struct { + Info TemplateInfo `yaml:"info"` +} + +type TemplateClassification struct { + CvssMetrics string `yaml:"cvss-metrics,omitempty"` + CvssScore float64 `yaml:"cvss-score,omitempty"` + CveId string `yaml:"cve-id,omitempty"` + CweId string `yaml:"cwe-id,omitempty"` + Cpe string `yaml:"cpe,omitempty"` + EpssScore float64 `yaml:"epss-score,omitempty"` +} + +type TemplateInfo struct { + Name string `yaml:"name"` + Author string `yaml:"author"` + Severity string `yaml:"severity,omitempty"` + Description string `yaml:"description,omitempty"` + Reference interface{} `yaml:"reference,omitempty"` + Remediation string `yaml:"remediation,omitempty"` + Classification TemplateClassification `yaml:"classification,omitempty"` + Metadata map[string]interface{} `yaml:"metadata,omitempty"` + Tags string `yaml:"tags,omitempty"` +} diff --git a/v2/go.mod b/v2/go.mod index 89369095b..c7d52f11c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -72,7 +72,6 @@ require ( github.com/projectdiscovery/gologger v1.1.10 github.com/projectdiscovery/httpx v1.3.0 github.com/projectdiscovery/mapcidr v1.1.2 - github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 github.com/projectdiscovery/ratelimit v0.0.8 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 diff --git a/v2/go.sum b/v2/go.sum index c38b6324e..0c4a0cc61 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -31,7 +31,6 @@ github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd/go.mod h1:AqtPw7WNT0O69k+AbPKWVGYeW94TqgMW/g+Ppc8AZr4= github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek= github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= -github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= @@ -54,7 +53,6 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= @@ -69,7 +67,6 @@ github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= @@ -355,7 +352,6 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= @@ -431,8 +427,6 @@ github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4 github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= -github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 h1:aDq18tNWbnN5ZM0ADQb+8KB4DEPIGZMXdDmcXyFUoNg= -github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8/go.mod h1:JiVXOIewstCBMPsO+ZnmI43UXMPJGEE1jwuFVz4ujKM= github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= github.com/projectdiscovery/rawhttp v0.1.13 h1:Xn3NY3SYIk0151K5Qfuvx3tayl2UOoxMuVyYvGT95BA= @@ -450,7 +444,6 @@ github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpW github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= github.com/projectdiscovery/utils v0.0.38 h1:EIAgaP3imfcQY+laxNOU9LXh7VZNAbmiwXsQN0mAxdQ= github.com/projectdiscovery/utils v0.0.38/go.mod h1:5+WAxSV7yGl6SDCtR1qiOyiEMCIo3jIff+A5OiYTCgM= github.com/projectdiscovery/wappalyzergo v0.0.94 h1:IVRskuU95MajWCKYgvH5L67+MXDOWJDWSeBD61OsS/A= @@ -468,7 +461,6 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sashabaranov/go-openai v1.9.1 h1:3N52HkJKo9Zlo/oe1AVv5ZkCOny0ra58/ACvAxkN3MM= @@ -599,7 +591,6 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= @@ -610,7 +601,6 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= @@ -636,7 +626,6 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -646,7 +635,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -660,13 +648,11 @@ golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -717,7 +703,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -730,7 +715,6 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= @@ -740,7 +724,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -755,7 +738,6 @@ golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=