8c76f6c847
* feat: update analyzer thresholds and add crawl configuration options * feat: update Helm chart version and add README for installation instructions * feat: update installation instructions in README and add Docker support * feat: update deployment manifests and configuration for improved service handling and analyzer settings * feat: add API endpoint for paginated IP retrieval and enhance dashboard visualization with category filters * feat: update configuration for Krawl service to use external config file * feat: refactor code for improved readability and consistency across multiple files * feat: remove Flake8, Pylint, and test steps from PR checks workflow
442 lines
8.7 KiB
YAML
442 lines
8.7 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: krawl-system
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: krawl-config
|
|
namespace: krawl-system
|
|
data:
|
|
config.yaml: |
|
|
# Krawl Honeypot Configuration
|
|
server:
|
|
port: 5000
|
|
delay: 100
|
|
timezone: null
|
|
links:
|
|
min_length: 5
|
|
max_length: 15
|
|
min_per_page: 10
|
|
max_per_page: 15
|
|
char_space: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
|
max_counter: 10
|
|
canary:
|
|
token_url: null
|
|
token_tries: 10
|
|
dashboard:
|
|
secret_path: null
|
|
api:
|
|
server_url: null
|
|
server_port: 8080
|
|
server_path: "/api/v2/users"
|
|
database:
|
|
path: "data/krawl.db"
|
|
retention_days: 30
|
|
behavior:
|
|
probability_error_codes: 0
|
|
analyzer:
|
|
http_risky_methods_threshold: 0.1
|
|
violated_robots_threshold: 0.1
|
|
uneven_request_timing_threshold: 0.5
|
|
uneven_request_timing_time_window_seconds: 300
|
|
user_agents_used_threshold: 2
|
|
attack_urls_threshold: 1
|
|
crawl:
|
|
infinite_pages_for_malicious: true
|
|
max_pages_limit: 250
|
|
ban_duration_seconds: 600
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: krawl-wordlists
|
|
namespace: krawl-system
|
|
data:
|
|
wordlists.json: |
|
|
{
|
|
"usernames": {
|
|
"prefixes": [
|
|
"admin",
|
|
"user",
|
|
"developer",
|
|
"root",
|
|
"system",
|
|
"db",
|
|
"api",
|
|
"service",
|
|
"deploy",
|
|
"test",
|
|
"prod",
|
|
"backup",
|
|
"monitor",
|
|
"jenkins",
|
|
"webapp"
|
|
],
|
|
"suffixes": [
|
|
"",
|
|
"_prod",
|
|
"_dev",
|
|
"_test",
|
|
"123",
|
|
"2024",
|
|
"_backup",
|
|
"_admin",
|
|
"01",
|
|
"02",
|
|
"_user",
|
|
"_service",
|
|
"_api"
|
|
]
|
|
},
|
|
"passwords": {
|
|
"prefixes": [
|
|
"P@ssw0rd",
|
|
"Passw0rd",
|
|
"Admin",
|
|
"Secret",
|
|
"Welcome",
|
|
"System",
|
|
"Database",
|
|
"Secure",
|
|
"Master",
|
|
"Root"
|
|
],
|
|
"simple": [
|
|
"test",
|
|
"demo",
|
|
"temp",
|
|
"change",
|
|
"password",
|
|
"admin",
|
|
"letmein",
|
|
"welcome",
|
|
"default",
|
|
"sample"
|
|
]
|
|
},
|
|
"emails": {
|
|
"domains": [
|
|
"example.com",
|
|
"company.com",
|
|
"localhost.com",
|
|
"test.com",
|
|
"domain.com",
|
|
"corporate.com",
|
|
"internal.net",
|
|
"enterprise.com",
|
|
"business.org"
|
|
]
|
|
},
|
|
"api_keys": {
|
|
"prefixes": [
|
|
"sk_live_",
|
|
"sk_test_",
|
|
"api_",
|
|
"key_",
|
|
"token_",
|
|
"access_",
|
|
"secret_",
|
|
"prod_",
|
|
""
|
|
]
|
|
},
|
|
"databases": {
|
|
"names": [
|
|
"production",
|
|
"prod_db",
|
|
"main_db",
|
|
"app_database",
|
|
"users_db",
|
|
"customer_data",
|
|
"analytics",
|
|
"staging_db",
|
|
"dev_database",
|
|
"wordpress",
|
|
"ecommerce",
|
|
"crm_db",
|
|
"inventory"
|
|
],
|
|
"hosts": [
|
|
"localhost",
|
|
"db.internal",
|
|
"mysql.local",
|
|
"postgres.internal",
|
|
"127.0.0.1",
|
|
"db-server-01",
|
|
"database.prod",
|
|
"sql.company.com"
|
|
]
|
|
},
|
|
"applications": {
|
|
"names": [
|
|
"WebApp",
|
|
"API Gateway",
|
|
"Dashboard",
|
|
"Admin Panel",
|
|
"CMS",
|
|
"Portal",
|
|
"Manager",
|
|
"Console",
|
|
"Control Panel",
|
|
"Backend"
|
|
]
|
|
},
|
|
"users": {
|
|
"roles": [
|
|
"Administrator",
|
|
"Developer",
|
|
"Manager",
|
|
"User",
|
|
"Guest",
|
|
"Moderator",
|
|
"Editor",
|
|
"Viewer",
|
|
"Analyst",
|
|
"Support"
|
|
]
|
|
},
|
|
"directory_listing": {
|
|
"files": [
|
|
"admin.txt",
|
|
"test.exe",
|
|
"backup.sql",
|
|
"database.sql",
|
|
"db_backup.sql",
|
|
"dump.sql",
|
|
"config.php",
|
|
"credentials.txt",
|
|
"passwords.txt",
|
|
"users.csv",
|
|
".env",
|
|
"id_rsa",
|
|
"id_rsa.pub",
|
|
"private_key.pem",
|
|
"api_keys.json",
|
|
"secrets.yaml",
|
|
"admin_notes.txt",
|
|
"settings.ini",
|
|
"database.yml",
|
|
"wp-config.php",
|
|
".htaccess",
|
|
"server.key",
|
|
"cert.pem",
|
|
"shadow.bak",
|
|
"passwd.old"
|
|
],
|
|
"directories": [
|
|
"uploads/",
|
|
"backups/",
|
|
"logs/",
|
|
"temp/",
|
|
"cache/",
|
|
"private/",
|
|
"config/",
|
|
"admin/",
|
|
"database/",
|
|
"backup/",
|
|
"old/",
|
|
"archive/",
|
|
".git/",
|
|
"keys/",
|
|
"credentials/"
|
|
]
|
|
},
|
|
"error_codes": [
|
|
400,
|
|
401,
|
|
403,
|
|
404,
|
|
500,
|
|
502,
|
|
503
|
|
],
|
|
"server_headers": [
|
|
"Apache/2.2.22 (Ubuntu)",
|
|
"nginx/1.18.0",
|
|
"Microsoft-IIS/10.0",
|
|
"LiteSpeed",
|
|
"Caddy",
|
|
"Gunicorn/20.0.4",
|
|
"uvicorn/0.13.4",
|
|
"Express",
|
|
"Flask/1.1.2",
|
|
"Django/3.1"
|
|
]
|
|
}
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: krawl-db
|
|
namespace: krawl-system
|
|
labels:
|
|
app: krawl-server
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: krawl-server
|
|
namespace: krawl-system
|
|
labels:
|
|
app: krawl-server
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: krawl-server
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: krawl-server
|
|
spec:
|
|
containers:
|
|
- name: krawl
|
|
image: ghcr.io/blessedrebus/krawl:latest
|
|
imagePullPolicy: Always
|
|
ports:
|
|
- containerPort: 5000
|
|
name: http
|
|
protocol: TCP
|
|
env:
|
|
- name: CONFIG_LOCATION
|
|
value: "config.yaml"
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /app/config.yaml
|
|
subPath: config.yaml
|
|
readOnly: true
|
|
- name: wordlists
|
|
mountPath: /app/wordlists.json
|
|
subPath: wordlists.json
|
|
readOnly: true
|
|
- name: database
|
|
mountPath: /app/data
|
|
resources:
|
|
requests:
|
|
memory: "64Mi"
|
|
cpu: "100m"
|
|
limits:
|
|
memory: "256Mi"
|
|
cpu: "500m"
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: krawl-config
|
|
- name: wordlists
|
|
configMap:
|
|
name: krawl-wordlists
|
|
- name: database
|
|
persistentVolumeClaim:
|
|
claimName: krawl-db
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: krawl-server
|
|
namespace: krawl-system
|
|
labels:
|
|
app: krawl-server
|
|
spec:
|
|
type: LoadBalancer
|
|
externalTrafficPolicy: Local
|
|
sessionAffinity: ClientIP
|
|
sessionAffinityConfig:
|
|
clientIP:
|
|
timeoutSeconds: 10800
|
|
ports:
|
|
- port: 5000
|
|
targetPort: 5000
|
|
protocol: TCP
|
|
name: http
|
|
selector:
|
|
app: krawl-server
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: krawl-ingress
|
|
namespace: krawl-system
|
|
spec:
|
|
ingressClassName: traefik
|
|
rules:
|
|
- host: krawl.example.com # Change to your domain
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: krawl-server
|
|
port:
|
|
number: 5000
|
|
# tls:
|
|
# - hosts:
|
|
# - krawl.example.com
|
|
# secretName: krawl-tls
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: krawl-network-policy
|
|
namespace: krawl-system
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app: krawl-server
|
|
policyTypes:
|
|
- Ingress
|
|
- Egress
|
|
ingress:
|
|
- from:
|
|
- podSelector: {}
|
|
- namespaceSelector: {}
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
ports:
|
|
- protocol: TCP
|
|
port: 5000
|
|
egress:
|
|
- to:
|
|
- namespaceSelector: {}
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
ports:
|
|
- protocol: TCP
|
|
- protocol: UDP
|
|
---
|
|
# Optional: HorizontalPodAutoscaler for auto-scaling
|
|
apiVersion: autoscaling/v2
|
|
kind: HorizontalPodAutoscaler
|
|
metadata:
|
|
name: krawl-hpa
|
|
namespace: krawl-system
|
|
spec:
|
|
scaleTargetRef:
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
name: krawl-server
|
|
minReplicas: 1
|
|
maxReplicas: 5
|
|
metrics:
|
|
- type: Resource
|
|
resource:
|
|
name: cpu
|
|
target:
|
|
type: Utilization
|
|
averageUtilization: 70
|
|
- type: Resource
|
|
resource:
|
|
name: memory
|
|
target:
|
|
type: Utilization
|
|
averageUtilization: 80
|