Malin/krawl.es
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat/sqlite3-storage

Browse Source
This commit is contained in:
Phillip Tarrant
2025-12-28 10:56:37 -06:00
parent f1c142c53d a1a5e406b0
commit 6487cb493d
12 changed files with 253 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -186,6 +186,7 @@ To customize the deception server installation several **environment variables**
| `DASHBOARD_SECRET_PATH` | Custom dashboard path | Auto-generated |
| `PROBABILITY_ERROR_CODES` | Error response probability (0-100%) | `0` |
| `SERVER_HEADER` | HTTP Server header for deception | `Apache/2.2.22 (Ubuntu)` |
| `TIMEZONE` | IANA timezone for logs and dashboard (e.g., `America/New_York`, `Europe/Rome`) | System timezone |
## robots.txt
The actual (juicy) robots.txt configuration is the following

44
deployment.yaml
View File

@@ -1,44 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: krawl-server
namespace: krawl
labels:
app: krawl-server
spec:
replicas: 1
selector:
matchLabels:
app: krawl-server
template:
metadata:
labels:
app: krawl-server
spec:
containers:
- name: krawl
image: ghcr.io/blessedrebus/krawl:latest
imagePullPolicy: Always
ports:
- containerPort: 5000
name: http
protocol: TCP
envFrom:
- configMapRef:
name: krawl-config
volumeMounts:
- name: wordlists
mountPath: /app/wordlists.json
subPath: wordlists.json
readOnly: true
resources:
requests:
memory: "64Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "500m"
volumes:
- name: wordlists
configMap:
name: krawl-wordlists

2
docker-compose.yaml
View File

@@ -25,6 +25,8 @@ services:
# - CANARY_TOKEN_URL=http://canarytokens.com/api/users/YOUR_TOKEN/passwords.txt
# Optional: Set custom dashboard path (auto-generated if not set)
# - DASHBOARD_SECRET_PATH=/my-secret-dashboard
# Optional: Set timezone for logs and dashboard (e.g., America/New_York, Europe/Rome)
# - TIMEZONE=UTC
restart: unless-stopped
healthcheck:
test: ["CMD", "python3", "-c", "import requests; requests.get('http://localhost:5000')"]

3
helm/templates/configmap.yaml
View File

@@ -16,3 +16,6 @@ data:
PROBABILITY_ERROR_CODES: {{ .Values.config.probabilityErrorCodes | quote }}
SERVER_HEADER: {{ .Values.config.serverHeader | quote }}
CANARY_TOKEN_URL: {{ .Values.config.canaryTokenUrl | quote }}
{{- if .Values.config.timezone }}
TIMEZONE: {{ .Values.config.timezone | quote }}
{{- end }}

1
helm/values.yaml
View File

@@ -75,6 +75,7 @@ config:
probabilityErrorCodes: 0
serverHeader: "Apache/2.2.22 (Ubuntu)"
# canaryTokenUrl: set-your-canary-token-url-here
# timezone: "UTC" # IANA timezone (e.g., "America/New_York", "Europe/Rome"). If not set, system timezone is used.
networkPolicy:
enabled: true

3
kubernetes/manifests/configmap.yaml
View File

@@ -14,4 +14,5 @@ data:
CANARY_TOKEN_TRIES: "10"
PROBABILITY_ERROR_CODES: "0"
SERVER_HEADER: "Apache/2.2.22 (Ubuntu)"
# CANARY_TOKEN_URL: set-your-canary-token-url-here
# CANARY_TOKEN_URL: set-your-canary-token-url-here
# TIMEZONE: "UTC" # IANA timezone (e.g., "America/New_York", "Europe/Rome")

41
src/config.py
View File

@@ -3,6 +3,8 @@
import os
from dataclasses import dataclass
from typing import Optional, Tuple
from zoneinfo import ZoneInfo
import time
@dataclass
@@ -25,6 +27,40 @@ class Config:
# Database settings
database_path: str = "data/krawl.db"
database_retention_days: int = 30
timezone: str = None # IANA timezone (e.g., 'America/New_York', 'Europe/Rome')
@staticmethod
# Try to fetch timezone before if not set
def get_system_timezone() -> str:
"""Get the system's default timezone"""
try:
if os.path.islink('/etc/localtime'):
tz_path = os.readlink('/etc/localtime')
if 'zoneinfo/' in tz_path:
return tz_path.split('zoneinfo/')[-1]
local_tz = time.tzname[time.daylight]
if local_tz and local_tz != 'UTC':
return local_tz
except Exception:
pass
# Default fallback to UTC
return 'UTC'
def get_timezone(self) -> ZoneInfo:
"""Get configured timezone as ZoneInfo object"""
if self.timezone:
try:
return ZoneInfo(self.timezone)
except Exception:
pass
system_tz = self.get_system_timezone()
try:
return ZoneInfo(system_tz)
except Exception:
return ZoneInfo('UTC')
@classmethod
def from_env(cls) -> 'Config':
@@ -48,8 +84,9 @@ class Config:
api_server_url=os.getenv('API_SERVER_URL'),
api_server_port=int(os.getenv('API_SERVER_PORT', 8080)),
api_server_path=os.getenv('API_SERVER_PATH', '/api/v2/users'),
probability_error_codes=int(os.getenv('PROBABILITY_ERROR_CODES', 5)),
server_header=os.getenv('SERVER_HEADER', 'Apache/2.2.22 (Ubuntu)'),
database_path=os.getenv('DATABASE_PATH', 'data/krawl.db'),
database_retention_days=int(os.getenv('DATABASE_RETENTION_DAYS', 30))
database_retention_days=int(os.getenv('DATABASE_RETENTION_DAYS', 30)),
probability_error_codes=int(os.getenv('PROBABILITY_ERROR_CODES', 0)),
timezone=os.getenv('TIMEZONE') # If not set, will use system timezone
)

33
src/logger.py
View File

@@ -8,6 +8,23 @@ Provides two loggers: app (application) and access (HTTP access logs).
import logging
import os
from logging.handlers import RotatingFileHandler
from typing import Optional
from zoneinfo import ZoneInfo
from datetime import datetime
class TimezoneFormatter(logging.Formatter):
"""Custom formatter that respects configured timezone"""
def __init__(self, fmt=None, datefmt=None, timezone: Optional[ZoneInfo] = None):
super().__init__(fmt, datefmt)
self.timezone = timezone or ZoneInfo('UTC')
def formatTime(self, record, datefmt=None):
"""Override formatTime to use configured timezone"""
dt = datetime.fromtimestamp(record.created, tz=self.timezone)
if datefmt:
return dt.strftime(datefmt)
return dt.isoformat()
class LoggerManager:
@@ -20,23 +37,27 @@ class LoggerManager:
cls._instance._initialized = False
return cls._instance
def initialize(self, log_dir: str = "logs") -> None:
def initialize(self, log_dir: str = "logs", timezone: Optional[ZoneInfo] = None) -> None:
"""
Initialize the logging system with rotating file handlers.
Args:
log_dir: Directory for log files (created if not exists)
timezone: ZoneInfo timezone for log timestamps (defaults to UTC)
"""
if self._initialized:
return
self.timezone = timezone or ZoneInfo('UTC')
# Create log directory if it doesn't exist
os.makedirs(log_dir, exist_ok=True)
# Common format for all loggers
log_format = logging.Formatter(
log_format = TimezoneFormatter(
"[%(asctime)s] %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
datefmt="%Y-%m-%d %H:%M:%S",
timezone=self.timezone
)
# Rotation settings: 1MB max, 5 backups
@@ -83,7 +104,7 @@ class LoggerManager:
self._credential_logger.handlers.clear()
# Credential logger uses a simple format: timestamp|ip|username|password|path
credential_format = logging.Formatter("%(message)s")
credential_format = TimezoneFormatter("%(message)s", timezone=self.timezone)
credential_file_handler = RotatingFileHandler(
os.path.join(log_dir, "credentials.log"),
@@ -136,6 +157,6 @@ def get_credential_logger() -> logging.Logger:
return _logger_manager.credentials
def initialize_logging(log_dir: str = "logs") -> None:
def initialize_logging(log_dir: str = "logs", timezone: Optional[ZoneInfo] = None) -> None:
"""Initialize the logging system."""
_logger_manager.initialize(log_dir)
_logger_manager.initialize(log_dir, timezone)

14
src/server.py
View File

@@ -36,6 +36,8 @@ def print_usage():
print(' SERVER_HEADER - HTTP Server header for deception (default: Apache/2.2.22 (Ubuntu))')
print(' DATABASE_PATH - Path to SQLite database (default: data/krawl.db)')
print(' DATABASE_RETENTION_DAYS - Days to retain database records (default: 30)')
print(' TIMEZONE - IANA timezone for logs/dashboard (e.g., America/New_York, Europe/Rome)')
print(' If not set, system timezone will be used')
def main():
@@ -44,8 +46,13 @@ def main():
print_usage()
exit(0)
# Initialize logging
initialize_logging()
config = Config.from_env()
# Get timezone configuration
tz = config.get_timezone()
# Initialize logging with timezone
initialize_logging(timezone=tz)
app_logger = get_app_logger()
access_logger = get_access_logger()
credential_logger = get_credential_logger()
@@ -59,7 +66,7 @@ def main():
except Exception as e:
app_logger.warning(f'Database initialization failed: {e}. Continuing with in-memory only.')
tracker = AccessTracker()
tracker = AccessTracker(timezone=tz)
Handler.config = config
Handler.tracker = tracker
@@ -81,6 +88,7 @@ def main():
try:
app_logger.info(f'Starting deception server on port {config.port}...')
app_logger.info(f'Timezone configured: {tz.key}')
app_logger.info(f'Dashboard available at: {config.dashboard_secret_path}')
if config.canary_token_url:
app_logger.info(f'Canary token will appear after {config.canary_token_tries} tries')

11
src/templates/dashboard_template.py
View File

@@ -6,7 +6,7 @@ Customize this template to change the dashboard appearance.
"""
import html
from datetime import datetime
def _escape(value) -> str:
"""Escape HTML special characters to prevent XSS attacks."""
@@ -14,6 +14,15 @@ def _escape(value) -> str:
return ""
return html.escape(str(value))
def format_timestamp(iso_timestamp: str) -> str:
"""Format ISO timestamp for display (YYYY-MM-DD HH:MM:SS)"""
try:
dt = datetime.fromisoformat(iso_timestamp)
return dt.strftime("%Y-%m-%d %H:%M:%S")
except Exception:
# Fallback for old format
return iso_timestamp.split("T")[1][:8] if "T" in iso_timestamp else iso_timestamp
def generate_dashboard(stats: dict) -> str:
"""Generate dashboard HTML with access statistics"""

12
src/tracker.py
View File

@@ -3,6 +3,7 @@
from typing import Dict, List, Tuple, Optional
from collections import defaultdict
from datetime import datetime
from zoneinfo import ZoneInfo
import re
import urllib.parse
@@ -16,7 +17,7 @@ class AccessTracker:
Maintains in-memory structures for fast dashboard access and
persists data to SQLite for long-term storage and analysis.
"""
def __init__(self, db_manager: Optional[DatabaseManager] = None):
def __init__(self, db_manager: Optional[DatabaseManager] = None, timezone: Optional[ZoneInfo] = None):
"""
Initialize the access tracker.
@@ -29,6 +30,7 @@ class AccessTracker:
self.user_agent_counts: Dict[str, int] = defaultdict(int)
self.access_log: List[Dict] = []
self.credential_attempts: List[Dict] = []
self.timezone = timezone or ZoneInfo('UTC')
self.suspicious_patterns = [
'bot', 'crawler', 'spider', 'scraper', 'curl', 'wget', 'python-requests',
'scanner', 'nikto', 'sqlmap', 'nmap', 'masscan', 'nessus', 'acunetix',
@@ -119,7 +121,7 @@ class AccessTracker:
'path': path,
'username': username,
'password': password,
'timestamp': datetime.now().isoformat()
'timestamp': datetime.now(self.timezone).isoformat()
})
# Persist to database
@@ -184,9 +186,9 @@ class AccessTracker:
'path': path,
'user_agent': user_agent,
'suspicious': is_suspicious,
'honeypot_triggered': is_honeypot,
'attack_types': attack_findings,
'timestamp': datetime.now().isoformat()
'honeypot_triggered': self.is_honeypot_path(path),
'attack_types':attack_findings,
'timestamp': datetime.now(self.timezone).isoformat()
})
# Persist to database

150
tests/test_credentials.sh Executable file
View File

@@ -0,0 +1,150 @@
#!/bin/bash
# This script sends various POST requests with credentials to the honeypot
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m'
# Configuration
HOST="localhost"
PORT="5000"
BASE_URL="http://${HOST}:${PORT}"
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Krawl Credential Logging Test Script${NC}"
echo -e "${BLUE}========================================${NC}\n"
# Check if server is running
echo -e "${YELLOW}Checking if server is running on ${BASE_URL}...${NC}"
if ! curl -s -f "${BASE_URL}/health" > /dev/null 2>&1; then
echo -e "${RED}❌ Server is not running. Please start the Krawl server first.${NC}"
echo -e "${YELLOW}Run: python3 src/server.py${NC}"
exit 1
fi
echo -e "${GREEN}✓ Server is running${NC}\n"
# Test 1: Simple login form POST
echo -e "${YELLOW}Test 1: POST to /login with form data${NC}"
curl -s -X POST "${BASE_URL}/login" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "username=admin&password=admin123" \
> /dev/null
echo -e "${GREEN}✓ Sent: admin / admin123${NC}\n"
sleep 1
# Test 2: Admin panel login
echo -e "${YELLOW}Test 2: POST to /admin with credentials${NC}"
curl -s -X POST "${BASE_URL}/admin" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "user=root&pass=toor&submit=Login" \
> /dev/null
echo -e "${GREEN}✓ Sent: root / toor${NC}\n"
sleep 1
# Test 3: WordPress login attempt
echo -e "${YELLOW}Test 3: POST to /wp-login.php${NC}"
curl -s -X POST "${BASE_URL}/wp-login.php" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "log=wpuser&pwd=Password1&wp-submit=Log+In" \
> /dev/null
echo -e "${GREEN}✓ Sent: wpuser / Password1${NC}\n"
sleep 1
# Test 4: JSON formatted credentials
echo -e "${YELLOW}Test 4: POST to /api/login with JSON${NC}"
curl -s -X POST "${BASE_URL}/api/login" \
-H "Content-Type: application/json" \
-d '{"username":"apiuser","password":"apipass123","remember":true}' \
> /dev/null
echo -e "${GREEN}✓ Sent: apiuser / apipass123${NC}\n"
sleep 1
# Test 5: SSH-style login
echo -e "${YELLOW}Test 5: POST to /ssh with credentials${NC}"
curl -s -X POST "${BASE_URL}/ssh" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "username=sshuser&password=P@ssw0rd!" \
> /dev/null
echo -e "${GREEN}✓ Sent: sshuser / P@ssw0rd!${NC}\n"
sleep 1
# Test 6: Database admin
echo -e "${YELLOW}Test 6: POST to /phpmyadmin with credentials${NC}"
curl -s -X POST "${BASE_URL}/phpmyadmin" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "pma_username=dbadmin&pma_password=dbpass123&server=1" \
> /dev/null
echo -e "${GREEN}✓ Sent: dbadmin / dbpass123${NC}\n"
sleep 1
# Test 7: Multiple fields with email
echo -e "${YELLOW}Test 7: POST to /register with email${NC}"
curl -s -X POST "${BASE_URL}/register" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "email=test@example.com&username=newuser&password=NewPass123&confirm_password=NewPass123" \
> /dev/null
echo -e "${GREEN}✓ Sent: newuser / NewPass123 (email: test@example.com)${NC}\n"
sleep 1
# Test 8: FTP credentials
echo -e "${YELLOW}Test 8: POST to /ftp/login${NC}"
curl -s -X POST "${BASE_URL}/ftp/login" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "ftpuser=ftpadmin&ftppass=ftp123456" \
> /dev/null
echo -e "${GREEN}✓ Sent: ftpadmin / ftp123456${NC}\n"
sleep 1
# Test 9: Common brute force attempt
echo -e "${YELLOW}Test 9: Multiple attempts (simulating brute force)${NC}"
for i in {1..3}; do
curl -s -X POST "${BASE_URL}/login" \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "username=admin&password=pass${i}" \
> /dev/null
echo -e "${GREEN}✓ Attempt $i: admin / pass${i}${NC}"
sleep 0.5
done
echo ""
sleep 1
# Test 10: Special characters in credentials
echo -e "${YELLOW}Test 10: POST with special characters${NC}"
curl -s -X POST "${BASE_URL}/login" \
-H "Content-Type: application/x-www-form-urlencoded" \
--data-urlencode "username=user@domain.com" \
--data-urlencode "password=P@\$\$w0rd!#%" \
> /dev/null
echo -e "${GREEN}✓ Sent: user@domain.com / P@\$\$w0rd!#%${NC}\n"
echo -e "${BLUE}========================================${NC}"
echo -e "${GREEN}✓ All credential tests completed!${NC}"
echo -e "${BLUE}========================================${NC}\n"
echo -e "${YELLOW}Check the results:${NC}"
echo -e " 1. View the log file: ${GREEN}cat src/logs/credentials.log${NC}"
echo -e " 2. View the dashboard: ${GREEN}${BASE_URL}/dashboard${NC}"
echo -e " 3. Check recent logs: ${GREEN}tail -20 src/logs/krawl.log${NC}\n"
# Display last 10 credential entries if log file exists
if [ -f "src/logs/credentials.log" ]; then
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Last 10 Captured Credentials:${NC}"
echo -e "${BLUE}========================================${NC}"
tail -10 src/logs/credentials.log
echo ""
fi
echo -e "${YELLOW}💡 Tip: Open ${BASE_URL}/dashboard in your browser to see the credentials in real-time!${NC}"