External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:55:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

add mkdir support in headless screenshot (#3457)

Browse Source 
* add mkdir support in headless screenshot

* use filepath to join paths

* print info when screenshot is saved

* change version to v2.9.1-dev

* minor fixings on windows path

---------

Co-authored-by: Mzack9999 <mzack9999@protonmail.com>
This commit is contained in:
Tarun Koyalwar 2023-03-24 00:44:32 +05:30 committed by GitHub
parent 6659402042
commit f8c5a45966
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 76 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
v2/go.mod
View File

@ -30,7 +30,7 @@ require (
github.com/projectdiscovery/rawhttp v0.1.9
github.com/projectdiscovery/retryabledns v1.0.21
github.com/projectdiscovery/retryablehttp-go v1.0.13
github.com/projectdiscovery/stringsutil v0.0.2
github.com/projectdiscovery/stringsutil v0.0.2 // indirect
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.4.0
@ -246,7 +246,6 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/nwaples/rardecode v1.1.2 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/projectdiscovery/fileutil v0.0.3
github.com/projectdiscovery/iputil v0.0.2 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/src-d/gcfg v1.4.0 // indirect

6
v2/go.sum
View File

@ -58,7 +58,6 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@ -431,8 +430,6 @@ github.com/projectdiscovery/fastdialer v0.0.24 h1:yEyYALCmDQpPYWttZ4uo9AJseqt4mY
github.com/projectdiscovery/fastdialer v0.0.24/go.mod h1:X7zZy3BGdGoprR6CftHKeJyV86a3OjSAlJcNU7FL26E=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg=
github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ=
github.com/projectdiscovery/freeport v0.0.4 h1:H4VrK/7hUcC1zbg46zv9iSMBACBDpUqcHkV+FUyXISw=
github.com/projectdiscovery/freeport v0.0.4/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGTDSkUwex/g=
@ -688,7 +685,6 @@ golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@ -740,7 +736,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -752,7 +747,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

5
v2/internal/runner/options.go
View File

@ -20,8 +20,9 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/stringsutil"
fileutil "github.com/projectdiscovery/utils/file"
logutil "github.com/projectdiscovery/utils/log"
stringsutil "github.com/projectdiscovery/utils/strings"
)
func ConfigureOptions() error {
@ -250,7 +251,7 @@ func configureOutput(options *types.Options) {
}
// disable standard logger (ref: https://github.com/golang/go/issues/19895)
// logutil.DisableDefaultLogger()
logutil.DisableDefaultLogger()
}
// loadResolvers loads resolvers from both user provided flag and file

2
v2/pkg/catalog/config/config.go
View File

@ -32,7 +32,7 @@ type Config struct {
const nucleiConfigFilename = ".templates-config.json"
// Version is the current version of nuclei
const Version = `2.9.0`
const Version = `2.9.1-dev`
var customConfigDirectory string

2
v2/pkg/external/customtemplates/github.go vendored
View File

@ -9,8 +9,8 @@ import (
"github.com/go-git/go-git/v5"
"github.com/google/go-github/github"
"github.com/pkg/errors"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/gologger"
fileutil "github.com/projectdiscovery/utils/file"
"golang.org/x/oauth2"
"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
)

2
v2/pkg/external/customtemplates/s3.go vendored
View File

@ -11,7 +11,7 @@ import (
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/stringsutil"
stringsutil "github.com/projectdiscovery/utils/strings"
)
type customTemplateS3Bucket struct {

4
v2/pkg/output/output.go
View File

@ -24,6 +24,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
fileutil "github.com/projectdiscovery/utils/file"
osutils "github.com/projectdiscovery/utils/os"
)
// Writer is an interface which writes output to somewhere for nuclei events.
@ -322,6 +323,9 @@ func sanitizeFileName(fileName string) string {
fileName = strings.ReplaceAll(fileName, "\\", "_")
fileName = strings.ReplaceAll(fileName, "-", "_")
fileName = strings.ReplaceAll(fileName, ".", "_")
if osutils.IsWindows() {
fileName = strings.ReplaceAll(fileName, ":", "_")
}
fileName = strings.TrimPrefix(fileName, "__")
return fileName
}

2
v2/pkg/protocols/common/generators/validate.go
View File

@ -6,8 +6,8 @@ import (
"path/filepath"
"strings"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
fileutil "github.com/projectdiscovery/utils/file"
folderutil "github.com/projectdiscovery/utils/folder"
)

25
v2/pkg/protocols/headless/engine/page_actions.go
View File

@ -5,6 +5,7 @@ import (
"net"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
@ -16,7 +17,12 @@ import (
"github.com/go-rod/rod/lib/proto"
"github.com/go-rod/rod/lib/utils"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
folderutil "github.com/projectdiscovery/utils/folder"
stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/segmentio/ksuid"
)
@ -325,10 +331,27 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error {
if err != nil {
return errors.Wrap(err, "could not take screenshot")
}
err = os.WriteFile(to+".png", data, 0540)
if p.getActionArgWithDefaultValues(act, "mkdir") == "true" && stringsutil.ContainsAny(to, folderutil.UnixPathSeparator, folderutil.WindowsPathSeparator) {
// creates new directory if needed based on path `to`
// TODO: replace all permission bits with fileutil constants (https://github.com/projectdiscovery/utils/issues/113)
if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil {
return errorutil.NewWithErr(err).Msgf("failed to create directory while writing screenshot")
}
}
filePath := to
if !strings.HasSuffix(to, ".png") {
filePath += ".png"
}
if fileutil.FileExists(filePath) {
// return custom error as overwriting files is not supported
return errorutil.NewWithTag("screenshot", "failed to write screenshot, file %v already exists", filePath)
}
err = os.WriteFile(filePath, data, 0540)
if err != nil {
return errors.Wrap(err, "could not write screenshot")
}
gologger.Info().Msgf("Screenshot successfully saved at %v\n", filePath)
return nil
}

43
v2/pkg/protocols/headless/engine/page_actions_test.go
View File

@ -3,10 +3,13 @@ package engine
import (
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@ -190,18 +193,50 @@ func TestActionScreenshot(t *testing.T) {
<body>Nuclei Test Page</body>
</html>`
// filePath where screenshot is saved
filePath := filepath.Join(os.TempDir(), "test.png")
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": "test"}},
{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
el := page.Page()
require.FileExists(t, "test.png", el, "could not get screenshot file")
_ = os.Remove("test.png")
_ = page.Page()
require.FileExists(t, filePath, "could not find screenshot file %v", filePath)
if err := os.RemoveAll(filePath); err != nil {
t.Logf("got error %v while deleting temp file", err)
}
})
}
func TestActionScreenshotToDir(t *testing.T) {
response := `
<html>
<head>
<title>Nuclei Test Page</title>
</head>
<body>Nuclei Test Page</body>
</html>`
filePath := filepath.Join(os.TempDir(), "screenshot-"+strconv.Itoa(rand.Intn(1000)), "test.png")
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath, "mkdir": "true"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
_ = page.Page()
require.FileExists(t, filePath, "could not find screenshot file %v", filePath)
if err := os.RemoveAll(filePath); err != nil {
t.Logf("got error %v while deleting temp file", err)
}
})
}

2
v2/pkg/reporting/exporters/markdown/markdown.go
View File

@ -57,7 +57,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
filenameBuilder := &strings.Builder{}
filenameBuilder.WriteString(event.TemplateID)
filenameBuilder.WriteString("-")
filenameBuilder.WriteString(strings.ReplaceAll(strings.ReplaceAll(event.Matched, "/", "_"), ":", "_"))
filenameBuilder.WriteString(stringsutil.ReplaceAll(event.Matched, "_", "/", ":"))
var suffix string
if event.MatcherName != "" {