Ice3man fa56800fcc
Fuzzing layer enhancements + input-types support (#4477)
* feat: move fuzz package to root directory

* feat: added support for input providers like openapi,postman,etc

* feat: integration of new fuzzing logic in engine

* bugfix: use and instead of or

* fixed lint errors

* go mod tidy

* add new reqresp type + bump utils

* custom http request parser

* use new struct type RequestResponse

* introduce unified input/target provider

* abstract input formats via new inputprovider

* completed input provider refactor

* remove duplicated code

* add sdk method to load targets

* rename component url->path

* add new yaml format + remove duplicated code

* use gopkg.in/yaml.v3 for parsing

* update .gitignore

* refactor/move + docs fuzzing in http protocol

* fuzz: header + query integration test using fuzzplayground

* fix integration test runner in windows

* feat add support for filter in http fuzz

* rewrite header/query integration test with filter

* add replace regex rule

* support kv fuzzing + misc updates

* add path fuzzing example + misc improvements

* fix matchedURL + skip httpx on multi formats

* cookie fuzz integration test

* add json body + params body tests

* feat add multipart/form-data fuzzing support

* add all fuzz body integration test

* misc bug fixes + minor refactor

* add multipart form + body form unit tests

* only run fuzzing templates if -fuzz flag is given

* refactor/move fuzz playground server to pkg

* fix integration test + refactor

* add auth types and strategies

* add file auth provider

* start implementing auth logic in http

* add logic in http protocol

* static auth implemented for http

* default :80,:443 normalization

* feat: dynamic auth init

* feat: dynamic auth using templates

* validate targets count in openapi+swagger

* inputformats: add support to accept variables

* fix workflow integration test

* update lazy cred fetch logic

* fix unit test

* drop postman support

* domain related normalization

* update secrets.yaml file format + misc updates

* add auth prefetch option

* remove old secret files

* add fuzzing+auth related sdk options

* fix/support multiple mode in kv header fuzzing

* rename 'headers' -> 'header' in fuzzing rules

* fix deadlock due to merge conflict resolution

* misc update

* add bool type in parsed value

* add openapi validation+override+ new flags

* misc updates

* remove optional path parameters when unavailable

* fix swagger.yaml file

* misc updates

* update print msg

* multiple openapi validation enchancements + appMode

* add optional params in required_openapi_vars.yaml file

* improve warning/verbose msgs in format

* fix skip-format-validation not working

* use 'params/parameter' instead of 'variable' in openapi

* add retry support for falky tests

* fix nuclei loading ignored templates (#4849)

* fix tag include logic

* fix unit test

* remove quoting in extractor output

* remove quote in debug code command

* feat: issue tracker URLs in JSON + misc fixes (#4855)

* feat: issue tracker URLs in JSON + misc fixes

* misc changes

* feat: status update support for issues

* feat: report metadata generation hook support

* feat: added CLI summary of tickets created

* misc changes

* introduce `disable-unsigned-templates` flag (#4820)

* introduce `disable-unsigned-templates` flag

* minor

* skip instead of exit

* remove duplicate imports

* use stats package + misc enhancements

* force display warning + adjust skipped stats in unsigned count

* include unsigned skipped templates without -dut flag

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

* Purge cache on global callback set (#4840)

* purge cache on global callback set

* lint

* purging cache

* purge cache in runner after loading templates

* include internal cache from parsers + add global cache register/purge via config

* remove disable cache purge option

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

* misc update

* add application/octet-stream support

* openapi: support path specific params

* misc option + readme update

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Co-authored-by: Mzack9999 <mzack9999@protonmail.com>
2024-03-14 03:08:53 +05:30

291 lines
6.9 KiB
Go

package openapi
import (
"fmt"
"github.com/getkin/kin-openapi/openapi3"
"github.com/pkg/errors"
)
// From: https://github.com/danielgtaylor/apisprout/blob/master/example.go
func getSchemaExample(schema *openapi3.Schema) (interface{}, bool) {
if schema.Example != nil {
return schema.Example, true
}
if schema.Default != nil {
return schema.Default, true
}
if schema.Enum != nil && len(schema.Enum) > 0 {
return schema.Enum[0], true
}
return nil, false
}
// stringFormatExample returns an example string based on the given format.
// http://json-schema.org/latest/json-schema-validation.html#rfc.section.7.3
func stringFormatExample(format string) string {
switch format {
case "date":
// https://tools.ietf.org/html/rfc3339
return "2018-07-23"
case "date-time":
// This is the date/time of API Sprout's first commit! :-)
return "2018-07-23T22:58:00-07:00"
case "time":
return "22:58:00-07:00"
case "email":
return "email@example.com"
case "hostname":
// https://tools.ietf.org/html/rfc2606#page-2
return "example.com"
case "ipv4":
// https://tools.ietf.org/html/rfc5737
return "198.51.100.0"
case "ipv6":
// https://tools.ietf.org/html/rfc3849
return "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
case "uri":
return "https://tools.ietf.org/html/rfc3986"
case "uri-template":
// https://tools.ietf.org/html/rfc6570
return "http://example.com/dictionary/{term:1}/{term}"
case "json-pointer":
// https://tools.ietf.org/html/rfc6901
return "#/components/parameters/term"
case "regex":
// https://stackoverflow.com/q/3296050/164268
return "/^1?$|^(11+?)\\1+$/"
case "uuid":
// https://www.ietf.org/rfc/rfc4122.txt
return "f81d4fae-7dec-11d0-a765-00a0c91e6bf6"
case "password":
return "********"
case "binary":
return "sagefuzzertest"
}
return ""
}
// excludeFromMode will exclude a schema if the mode is request and the schema
// is read-only
func excludeFromMode(schema *openapi3.Schema) bool {
if schema == nil {
return true
}
if schema.ReadOnly {
return true
}
return false
}
// isRequired checks whether a key is actually required.
func isRequired(schema *openapi3.Schema, key string) bool {
for _, req := range schema.Required {
if req == key {
return true
}
}
return false
}
type cachedSchema struct {
pending bool
out interface{}
}
var (
// ErrRecursive is when a schema is impossible to represent because it infinitely recurses.
ErrRecursive = errors.New("Recursive schema")
// ErrNoExample is sent when no example was found for an operation.
ErrNoExample = errors.New("No example found")
)
func openAPIExample(schema *openapi3.Schema, cache map[*openapi3.Schema]*cachedSchema) (out interface{}, err error) {
if ex, ok := getSchemaExample(schema); ok {
return ex, nil
}
cached, ok := cache[schema]
if !ok {
cached = &cachedSchema{
pending: true,
}
cache[schema] = cached
} else if cached.pending {
return nil, ErrRecursive
} else {
return cached.out, nil
}
defer func() {
cached.pending = false
cached.out = out
}()
// Handle combining keywords
if len(schema.OneOf) > 0 {
var ex interface{}
var err error
for _, candidate := range schema.OneOf {
ex, err = openAPIExample(candidate.Value, cache)
if err == nil {
break
}
}
return ex, err
}
if len(schema.AnyOf) > 0 {
var ex interface{}
var err error
for _, candidate := range schema.AnyOf {
ex, err = openAPIExample(candidate.Value, cache)
if err == nil {
break
}
}
return ex, err
}
if len(schema.AllOf) > 0 {
example := map[string]interface{}{}
for _, allOf := range schema.AllOf {
candidate, err := openAPIExample(allOf.Value, cache)
if err != nil {
return nil, err
}
value, ok := candidate.(map[string]interface{})
if !ok {
return nil, ErrNoExample
}
for k, v := range value {
example[k] = v
}
}
return example, nil
}
switch {
case schema.Type == "boolean":
return true, nil
case schema.Type == "number", schema.Type == "integer":
value := 0.0
if schema.Min != nil && *schema.Min > value {
value = *schema.Min
if schema.ExclusiveMin {
if schema.Max != nil {
// Make the value half way.
value = (*schema.Min + *schema.Max) / 2.0
} else {
value++
}
}
}
if schema.Max != nil && *schema.Max < value {
value = *schema.Max
if schema.ExclusiveMax {
if schema.Min != nil {
// Make the value half way.
value = (*schema.Min + *schema.Max) / 2.0
} else {
value--
}
}
}
if schema.MultipleOf != nil && int(value)%int(*schema.MultipleOf) != 0 {
value += float64(int(*schema.MultipleOf) - (int(value) % int(*schema.MultipleOf)))
}
if schema.Type == "integer" {
return int(value), nil
}
return value, nil
case schema.Type == "string":
if ex := stringFormatExample(schema.Format); ex != "" {
return ex, nil
}
example := "string"
for schema.MinLength > uint64(len(example)) {
example += example
}
if schema.MaxLength != nil && *schema.MaxLength < uint64(len(example)) {
example = example[:*schema.MaxLength]
}
return example, nil
case schema.Type == "array", schema.Items != nil:
example := []interface{}{}
if schema.Items != nil && schema.Items.Value != nil {
ex, err := openAPIExample(schema.Items.Value, cache)
if err != nil {
return nil, fmt.Errorf("can't get example for array item: %+v", err)
}
example = append(example, ex)
for uint64(len(example)) < schema.MinItems {
example = append(example, ex)
}
}
return example, nil
case schema.Type == "object", len(schema.Properties) > 0:
example := map[string]interface{}{}
for k, v := range schema.Properties {
if excludeFromMode(v.Value) {
continue
}
ex, err := openAPIExample(v.Value, cache)
if err == ErrRecursive {
if isRequired(schema, k) {
return nil, fmt.Errorf("can't get example for '%s': %+v", k, err)
}
} else if err != nil {
return nil, fmt.Errorf("can't get example for '%s': %+v", k, err)
} else {
example[k] = ex
}
}
if schema.AdditionalProperties.Has != nil && schema.AdditionalProperties.Schema != nil {
addl := schema.AdditionalProperties.Schema.Value
if !excludeFromMode(addl) {
ex, err := openAPIExample(addl, cache)
if err == ErrRecursive {
// We just won't add this if it's recursive.
} else if err != nil {
return nil, fmt.Errorf("can't get example for additional properties: %+v", err)
} else {
example["additionalPropertyName"] = ex
}
}
}
return example, nil
}
return nil, ErrNoExample
}
// generateExampleFromSchema creates an example structure from an OpenAPI 3 schema
// object, which is an extended subset of JSON Schema.
//
// https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.1.md#schemaObject
func generateExampleFromSchema(schema *openapi3.Schema) (interface{}, error) {
return openAPIExample(schema, make(map[*openapi3.Schema]*cachedSchema)) // TODO: Use caching
}