From 3e4ec90cea257ef7000001d37cea46a1d1793521 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto <25837540+dwisiswant0@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:19:49 +0700 Subject: [PATCH] ci: refactor workflows (#5818) * ci: refactor workflows Signed-off-by: Dwi Siswanto * chore: structured build outputs Signed-off-by: Dwi Siswanto * feat: use `go-ci` Signed-off-by: Dwi Siswanto * chore(make): misused var for `vet` cmd Signed-off-by: Dwi Siswanto --------- Signed-off-by: Dwi Siswanto --- .../{auto-merge.yml => auto-merge.yaml} | 4 +- .github/workflows/build-test.yml | 70 --------- .github/workflows/codeql-analysis.yml | 24 ---- .github/workflows/functional-test.yml | 31 ---- .github/workflows/generate-docs.yaml | 28 ++++ .github/workflows/lint-test.yml | 22 --- .github/workflows/perf-test.yaml | 25 ++++ .github/workflows/performance-test.yaml | 28 ---- .github/workflows/publish-docs.yaml | 48 ------- .github/workflows/release-test.yml | 24 ---- .../{release-binary.yml => release.yaml} | 9 +- .github/workflows/template-validate.yml | 21 --- .github/workflows/tests.yaml | 135 ++++++++++++++++++ .gitignore | 1 + Makefile | 73 +++++----- cmd/functional-test/main.go | 8 +- cmd/integration-test/integration-test.go | 10 +- lib/example_test.go | 4 +- 18 files changed, 248 insertions(+), 317 deletions(-) rename .github/workflows/{auto-merge.yml => auto-merge.yaml} (86%) delete mode 100644 .github/workflows/build-test.yml delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/functional-test.yml create mode 100644 .github/workflows/generate-docs.yaml delete mode 100644 .github/workflows/lint-test.yml create mode 100644 .github/workflows/perf-test.yaml delete mode 100644 .github/workflows/performance-test.yaml delete mode 100644 .github/workflows/publish-docs.yaml delete mode 100644 .github/workflows/release-test.yml rename .github/workflows/{release-binary.yml => release.yaml} (83%) delete mode 100644 .github/workflows/template-validate.yml create mode 100644 .github/workflows/tests.yaml diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yaml similarity index 86% rename from .github/workflows/auto-merge.yml rename to .github/workflows/auto-merge.yaml index 0e39bd2c0..0ff3098e6 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yaml @@ -1,6 +1,8 @@ name: πŸ€– Auto Merge on: + pull_request_review: + types: [submitted] workflow_run: workflows: ["♾️ Compatibility Check"] types: @@ -16,7 +18,7 @@ jobs: runs-on: ubuntu-latest if: github.actor == 'dependabot[bot]' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.DEPENDABOT_PAT }} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml deleted file mode 100644 index d3f52d37c..000000000 --- a/.github/workflows/build-test.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: πŸ”¨ Build Test - -on: - pull_request: - paths: - - '**.go' - - '**.mod' - workflow_dispatch: - -jobs: - build-test: - if: "! endsWith(github.actor, '[bot]')" - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - runs-on: ${{ matrix.os }} - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Go - uses: projectdiscovery/actions/setup/go@v1 - - # required for running python code in py-snippet.yaml integration test - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Verify Go modules - run: make verify - - - name: Build - run: go build . - working-directory: cmd/nuclei/ - - - name: Test - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}" - run: go test ./... - - - name: Integration Tests - timeout-minutes: 50 - env: - GH_ACTION: true - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}" - run: | - chmod +x run.sh - bash run.sh ${{ matrix.os }} - working-directory: integration_tests/ - - - name: Race Condition Tests - if: ${{ matrix.os != 'windows-latest' }} # known issue: https://github.com/golang/go/issues/46099 - run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version - working-directory: cmd/nuclei/ - - - name: Example SDK Simple - run: go run . - working-directory: examples/simple/ - - # Temporarily disabled very flaky in github actions - # - name: Example SDK Advanced - # run: go run . - # working-directory: examples/advanced/ - - - name: Example SDK with speed control - run: go run . - working-directory: examples/with_speed_control/ diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index e9f60acd1..000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: 🚨 CodeQL Analysis - -on: - pull_request: - paths: - - '**.go' - - '**.mod' - workflow_dispatch: - -jobs: - codeql-analysis: - if: "! endsWith(github.actor, '[bot]')" - runs-on: ubuntu-latest-16-cores - permissions: - actions: read - contents: read - security-events: write - steps: - - uses: actions/checkout@v4 - - uses: github/codeql-action/init@v2 - with: - languages: 'go' - - uses: github/codeql-action/autobuild@v2 - - uses: github/codeql-action/analyze@v2 \ No newline at end of file diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml deleted file mode 100644 index 002b2aa0e..000000000 --- a/.github/workflows/functional-test.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: πŸ§ͺ Functional Test - -on: - pull_request: - paths: - - '**.go' - workflow_dispatch: - - -jobs: - functional-test: - if: "! endsWith(github.actor, '[bot]')" - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - runs-on: ${{ matrix.os }} - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Go - uses: projectdiscovery/actions/setup/go@v1 - - - name: Functional Tests - env: - GH_ACTION: true - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - run: | - chmod +x run.sh - bash run.sh ${{ matrix.os }} - working-directory: cmd/functional-test diff --git a/.github/workflows/generate-docs.yaml b/.github/workflows/generate-docs.yaml new file mode 100644 index 000000000..074c95928 --- /dev/null +++ b/.github/workflows/generate-docs.yaml @@ -0,0 +1,28 @@ +name: ⏰ Generate Docs + +on: + push: + branches: + - dev + workflow_dispatch: + +jobs: + publish-docs: + if: "${{ !endsWith(github.actor, '[bot]') }}" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - uses: projectdiscovery/actions/setup/git@v1 + - run: make syntax-docs + - run: git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT + id: status + - uses: projectdiscovery/actions/commit@v1 + if: steps.status.outputs.CHANGES > 0 + with: + files: | + SYNTAX-REFERENCE.md + nuclei-jsonschema.json + message: 'docs: update syntax & JSON schema πŸ€–' + - run: git pull origin $GITHUB_REF --rebase + - run: git push origin $GITHUB_REF diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml deleted file mode 100644 index 185d78523..000000000 --- a/.github/workflows/lint-test.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: πŸ™πŸ» Lint Test - -on: - pull_request: - paths: - - '**.go' - - '**.mod' - workflow_dispatch: - -jobs: - lint-test: - if: "! endsWith(github.actor, '[bot]')" - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: projectdiscovery/actions/setup/go@v1 - - - name: Run golangci-lint - uses: projectdiscovery/actions/golangci-lint@v1 diff --git a/.github/workflows/perf-test.yaml b/.github/workflows/perf-test.yaml new file mode 100644 index 000000000..732e7a7de --- /dev/null +++ b/.github/workflows/perf-test.yaml @@ -0,0 +1,25 @@ +name: πŸ”¨ Performance Test + +on: + schedule: + - cron: '0 0 * * 0' # Weekly + workflow_dispatch: + +jobs: + perf-test: + strategy: + matrix: + count: [50, 100, 150] + runs-on: ubuntu-latest + if: github.repository == 'projectdiscovery/nuclei' + env: + LIST_FILE: "/tmp/targets-${{ matrix.count }}.txt" + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - run: make verify + - name: Generate list + run: for i in {1..${{ matrix.count }}}; do echo "https://scanme.sh/?_=${i}" >> "${LIST_FILE}"; done + - run: go run -race . -l "${LIST_FILE}" + working-directory: cmd/nuclei/ + diff --git a/.github/workflows/performance-test.yaml b/.github/workflows/performance-test.yaml deleted file mode 100644 index 210673a36..000000000 --- a/.github/workflows/performance-test.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: πŸ”¨ Performance Test - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * 0' # Weekly - -jobs: - perf-test: - strategy: - matrix: - os: [ubuntu-latest, macOS-latest] - runs-on: ${{ matrix.os }} - if: github.repository == 'projectdiscovery/nuclei' - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Go - uses: projectdiscovery/actions/setup/go@v1 - - - name: Verify Go modules - run: make verify - - # Max GH exection time 6H => timeout after that - - name: Running performance with big list - run: go run -race . -l ../functional-test/targets-150.txt - working-directory: cmd/nuclei/ \ No newline at end of file diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml deleted file mode 100644 index 92042fd88..000000000 --- a/.github/workflows/publish-docs.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: ⏰ Publish Docs - -on: - push: - branches: - - dev - workflow_dispatch: - -jobs: - publish-docs: - if: "! endsWith(github.actor, '[bot]')" - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: "Set up Go" - uses: projectdiscovery/actions/setup/go@v1 - - - name: "Set up Git" - uses: projectdiscovery/actions/setup/git@v1 - - - name: Generate YAML Syntax Documentation - id: generate-docs - run: | - if ! which dstdocgen > /dev/null; then - echo -e "Command dstdocgen not found! Installing\c" - go install github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen@main - fi - go generate pkg/templates/templates.go - go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go - ./cmd/docgen/docgen SYNTAX-REFERENCE.md nuclei-jsonschema.json - git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT - - - name: Commit files - if: steps.generate-docs.outputs.CHANGES > 0 - run: | - git add SYNTAX-REFERENCE.md nuclei-jsonschema.json - git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a - - - name: Push changes - if: steps.generate-docs.outputs.CHANGES > 0 - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml deleted file mode 100644 index ce598a8d5..000000000 --- a/.github/workflows/release-test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: πŸ”¨ Release Test - -on: - pull_request: - paths: - - '**.go' - - '**.mod' - workflow_dispatch: - -jobs: - release-test: - if: "! endsWith(github.actor, '[bot]')" - runs-on: ubuntu-latest-16-cores - steps: - - name: "Check out code" - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Go - uses: projectdiscovery/actions/setup/go@v1 - - - name: Release snapshot - uses: projectdiscovery/actions/goreleaser@v1 diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release.yaml similarity index 83% rename from .github/workflows/release-binary.yml rename to .github/workflows/release.yaml index b4827c8c2..66d45b01b 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -name: πŸŽ‰ Release Binary +name: πŸŽ‰ Release on: push: @@ -13,10 +13,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Set up Go - uses: projectdiscovery/actions/setup/go@v1 - + - uses: projectdiscovery/actions/setup/go@v1 - uses: projectdiscovery/actions/goreleaser@v1 with: release: true @@ -24,4 +21,4 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" - DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" diff --git a/.github/workflows/template-validate.yml b/.github/workflows/template-validate.yml deleted file mode 100644 index 5509d0257..000000000 --- a/.github/workflows/template-validate.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: πŸ›  Template Validate - -on: - pull_request: - paths: - - '**.go' - workflow_dispatch: - -jobs: - build: - if: "! endsWith(github.actor, '[bot]')" - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v4 - - uses: projectdiscovery/actions/setup/go@v1 - - name: Template Validation - run: | - go run . -ut - go run . -validate - go run . -validate -w workflows - working-directory: cmd/nuclei/ \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 000000000..db3b64b5e --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,135 @@ +name: πŸ”¨ Tests + +on: + push: + branches: ["dev"] + paths: + - '**.go' + - '**.mod' + pull_request: + paths: + - '**.go' + - '**.mod' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: "Lint" + if: "${{ !endsWith(github.actor, '[bot]') }}" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - uses: projectdiscovery/actions/golangci-lint@v1 + + tests: + name: "Tests" + needs: ["lint"] + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + runs-on: "${{ matrix.os }}" + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - run: make vet + - run: make build + - run: make test + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}" + - run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version + if: ${{ matrix.os != 'windows-latest' }} # known issue: https://github.com/golang/go/issues/46099 + working-directory: cmd/nuclei/ + + sdk: + name: "Run example SDK" + needs: ["tests"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - name: "Simple" + run: go run . + working-directory: examples/simple/ + # - run: go run . # Temporarily disabled very flaky in github actions + # working-directory: examples/advanced/ + - name: "with Speed Control" + run: go run . + working-directory: examples/with_speed_control/ + + integration: + name: "Integration tests" + needs: ["tests"] + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: bash run.sh "${{ matrix.os }}" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}" + timeout-minutes: 50 + working-directory: integration_tests/ + + functional: + name: "Functional tests" + needs: ["tests"] + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - run: bash run.sh "${{ matrix.os }}" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + working-directory: cmd/functional-test/ + + validate: + name: "Template validate" + needs: ["tests"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - run: make template-validate + + codeql: + name: "CodeQL analysis" + needs: ["tests"] + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + - uses: github/codeql-action/init@v2 + with: + languages: 'go' + - uses: github/codeql-action/autobuild@v2 + - uses: github/codeql-action/analyze@v2 + + release: + name: "Release test" + needs: ["tests"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - uses: projectdiscovery/actions/goreleaser@v1 diff --git a/.gitignore b/.gitignore index 614510334..3148d8bbd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .vscode # Binaries +/bin/* **/bindgen **/debug-* **/docgen diff --git a/Makefile b/Makefile index 8614ef82c..a35a4b61f 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,9 @@ ifneq ($(shell go env GOOS),darwin) endif .PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs -.PHONY: devtools-tsgen docs dsl-docs functional fuzzplayground go-build integration -.PHONY: jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test tidy ts verify +.PHONY: devtools-tsgen docs docgen dsl-docs functional fuzzplayground go-build syntax-docs +.PHONY: integration jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test +.PHONY: tidy ts verify download vet template-validate all: build @@ -28,42 +29,38 @@ go-build: $(GOBUILD) $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \ -o '${GOBUILD_OUTPUT}' $(GOBUILD_PACKAGES) -build: GOBUILD_OUTPUT = nuclei +build: GOBUILD_OUTPUT = ./bin/nuclei build: GOBUILD_PACKAGES = cmd/nuclei/main.go build: go-build -build-stats: GOBUILD_OUTPUT = nuclei-stats +build-stats: GOBUILD_OUTPUT = ./bin/nuclei-stats build-stats: GOBUILD_PACKAGES = cmd/nuclei/main.go build-stats: GOBUILD_ADDITIONAL_ARGS = -tags=stats build-stats: go-build -scan-charts: GOBUILD_OUTPUT = scan-charts +scan-charts: GOBUILD_OUTPUT = ./bin/scan-charts scan-charts: GOBUILD_PACKAGES = cmd/scan-charts/main.go scan-charts: go-build -docs: GOBUILD_OUTPUT = cmd/docgen/docgen -docs: GOBUILD_PACKAGES = cmd/docgen/docgen.go -docs: bin = dstdocgen -docs: +docgen: GOBUILD_OUTPUT = ./bin/docgen +docgen: GOBUILD_PACKAGES = cmd/docgen/docgen.go +docgen: bin = dstdocgen +docgen: @if ! which $(bin) >/dev/null; then \ - read -p "${bin} not found. Do you want to install it? (y/n) " answer; \ - if [ "$$answer" = "y" ]; then \ - echo "Installing ${bin}..."; \ - go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/$(bin); \ - go install -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/$(bin); \ - else \ - echo "Please install ${bin} manually."; \ - exit 1; \ - fi \ + echo "Command $(bin) not found! Installing..."; \ + go install -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/$(bin)@latest; \ fi - - # TODO: Handle the panic, so that we just need to run `go install $(bin)@latest` (line 51-52) + # TODO: FIX THIS PANIC $(GOCMD) generate pkg/templates/templates.go - $(GOBUILD) -o "${GOBUILD_OUTPUT}" $(GOBUILD_PACKAGES) - ./$(GOBUILD_OUTPUT) docs.md nuclei-jsonschema.json - git reset --hard # line 59 +docs: docgen +docs: + ./bin/docgen docs.md nuclei-jsonschema.json + +syntax-docs: docgen +syntax-docs: + ./bin/docgen SYNTAX-REFERENCE.md nuclei-jsonschema.json test: GOFLAGS = -race -v test: @@ -78,30 +75,36 @@ functional: tidy: $(GOMOD) tidy -verify: tidy +download: + $(GOMOD) download + +verify: download $(GOMOD) verify -devtools-bindgen: GOBUILD_OUTPUT = bindgen +vet: verify + $(GOCMD) vet ./... + +devtools-bindgen: GOBUILD_OUTPUT = ./bin/bindgen devtools-bindgen: GOBUILD_PACKAGES = pkg/js/devtools/bindgen/cmd/bindgen/main.go devtools-bindgen: go-build -devtools-tsgen: GOBUILD_OUTPUT = tsgen +devtools-tsgen: GOBUILD_OUTPUT = ./bin/tsgen devtools-tsgen: GOBUILD_PACKAGES = pkg/js/devtools/tsgen/cmd/tsgen/main.go devtools-tsgen: go-build -devtools-scrapefuncs: GOBUILD_OUTPUT = scrapefuncs +devtools-scrapefuncs: GOBUILD_OUTPUT = ./bin/scrapefuncs devtools-scrapefuncs: GOBUILD_PACKAGES = pkg/js/devtools/scrapefuncs/main.go devtools-scrapefuncs: go-build devtools-all: devtools-bindgen devtools-tsgen devtools-scrapefuncs -jsupdate-bindgen: GOBUILD_OUTPUT = bindgen +jsupdate-bindgen: GOBUILD_OUTPUT = ./bin/bindgen jsupdate-bindgen: GOBUILD_PACKAGES = pkg/js/devtools/bindgen/cmd/bindgen/main.go jsupdate-bindgen: go-build jsupdate-bindgen: ./$(GOBUILD_OUTPUT) -dir pkg/js/libs -out pkg/js/generated -jsupdate-tsgen: GOBUILD_OUTPUT = tsgen +jsupdate-tsgen: GOBUILD_OUTPUT = ./bin/tsgen jsupdate-tsgen: GOBUILD_PACKAGES = pkg/js/devtools/tsgen/cmd/tsgen/main.go jsupdate-tsgen: go-build jsupdate-tsgen: @@ -111,18 +114,24 @@ jsupdate-all: jsupdate-bindgen jsupdate-tsgen ts: jsupdate-tsgen -fuzzplayground: GOBUILD_OUTPUT = fuzzplayground +fuzzplayground: GOBUILD_OUTPUT = ./bin/fuzzplayground fuzzplayground: GOBUILD_PACKAGES = cmd/tools/fuzzplayground/main.go fuzzplayground: LDFLAGS = -s -w fuzzplayground: go-build -memogen: GOBUILD_OUTPUT = memogen +memogen: GOBUILD_OUTPUT = ./bin/memogen memogen: GOBUILD_PACKAGES = cmd/memogen/memogen.go memogen: go-build memogen: ./$(GOBUILD_OUTPUT) -src pkg/js/libs -tpl cmd/memogen/function.tpl -dsl-docs: GOBUILD_OUTPUT = scrapefuncs +dsl-docs: GOBUILD_OUTPUT = ./bin/scrapefuncs dsl-docs: GOBUILD_PACKAGES = pkg/js/devtools/scrapefuncs/main.go dsl-docs: ./$(GOBUILD_OUTPUT) -out dsl.md + +template-validate: build +template-validate: + ./bin/nuclei -ut + ./bin/nuclei -validate + ./bin/nuclei -validate -w workflows \ No newline at end of file diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go index 73308a310..8a5cebc59 100644 --- a/cmd/functional-test/main.go +++ b/cmd/functional-test/main.go @@ -8,6 +8,7 @@ import ( "os" "strings" + "github.com/kitabisa/go-ci" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -15,9 +16,8 @@ import ( ) var ( - success = aurora.Green("[βœ“]").String() - failed = aurora.Red("[✘]").String() - githubAction = os.Getenv("GH_ACTION") == "true" + success = aurora.Green("[βœ“]").String() + failed = aurora.Red("[✘]").String() mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary") devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary") @@ -45,7 +45,7 @@ func runFunctionalTests(debug bool) (error, bool) { errored, failedTestCases := runTestCases(file, debug) - if githubAction { + if ci.IsCI() { fmt.Println("::group::Failed tests with debug") for _, failedTestCase := range failedTestCases { _ = runTestCase(failedTestCase, true) diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 00726de5d..ca77533c4 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -7,6 +7,7 @@ import ( "runtime" "strings" + "github.com/kitabisa/go-ci" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" @@ -22,10 +23,9 @@ type TestCaseInfo struct { } var ( - debug = os.Getenv("DEBUG") == "true" - githubAction = os.Getenv("GH_ACTION") == "true" - customTests = os.Getenv("TESTS") - protocol = os.Getenv("PROTO") + debug = os.Getenv("DEBUG") == "true" + customTests = os.Getenv("TESTS") + protocol = os.Getenv("PROTO") success = aurora.Green("[βœ“]").String() failed = aurora.Red("[✘]").String() @@ -103,7 +103,7 @@ func main() { failedTestTemplatePaths := runTests(customTestsList) if len(failedTestTemplatePaths) > 0 { - if githubAction { + if ci.IsCI() { // run failed tests again assuming they are flaky // if they fail as well only then we assume that there is an actual issue fmt.Println("::group::Running failed tests again") diff --git a/lib/example_test.go b/lib/example_test.go index 1c705677a..1d82073e1 100644 --- a/lib/example_test.go +++ b/lib/example_test.go @@ -7,6 +7,7 @@ import ( "os" "testing" + "github.com/kitabisa/go-ci" nuclei "github.com/projectdiscovery/nuclei/v3/lib" "github.com/remeh/sizedwaitgroup" ) @@ -78,9 +79,10 @@ func ExampleThreadSafeNucleiEngine() { func TestMain(m *testing.M) { // this file only contains testtables examples https://go.dev/blog/examples // and actual functionality test are in sdk_test.go - if os.Getenv("GH_ACTION") != "" || os.Getenv("CI") != "" { + if ci.IsCI() { // no need to run this test on github actions return } + os.Exit(m.Run()) }