Malin/krawl.es
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed some print statements to leverage logging, pulled in most recent dev edits, added exports to gitignore

Browse Source
This commit is contained in:
Phillip Tarrant
2026-01-07 12:33:43 -06:00
parent 5fe1984365
commit edb288a271
3 changed files with 23 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -76,3 +76,6 @@ data/
# Personal canary tokens or sensitive configs # Personal canary tokens or sensitive configs
*canary*token*.yaml *canary*token*.yaml
personal-values.yaml personal-values.yaml
#exports dir (keeping .gitkeep so we have the dir)
/exports/*

18
src/analyzer.py
View File

@@ -8,10 +8,13 @@ from datetime import datetime, timedelta
import re import re
from wordlists import get_wordlists from wordlists import get_wordlists
from config import get_config from config import get_config
from logger import get_app_logger
""" """
Functions for user activity analysis Functions for user activity analysis
""" """
app_logger = get_app_logger()
class Analyzer: class Analyzer:
""" """
Analyzes users activity and produces aggregated insights Analyzes users activity and produces aggregated insights
@@ -56,7 +59,7 @@ class Analyzer:
attack_urls_threshold = config.attack_urls_threshold attack_urls_threshold = config.attack_urls_threshold
uneven_request_timing_time_window_seconds = config.uneven_request_timing_time_window_seconds uneven_request_timing_time_window_seconds = config.uneven_request_timing_time_window_seconds
print(f"http_risky_methods_threshold: {http_risky_methods_threshold}") app_logger.debug(f"http_risky_methods_threshold: {http_risky_methods_threshold}")
score = {} score = {}
score["attacker"] = {"risky_http_methods": False, "robots_violations": False, "uneven_request_timing": False, "different_user_agents": False, "attack_url": False} score["attacker"] = {"risky_http_methods": False, "robots_violations": False, "uneven_request_timing": False, "different_user_agents": False, "attack_url": False}
@@ -185,7 +188,7 @@ class Analyzer:
variance = sum((x - mean) ** 2 for x in time_diffs) / len(time_diffs) variance = sum((x - mean) ** 2 for x in time_diffs) / len(time_diffs)
std = variance ** 0.5 std = variance ** 0.5
cv = std/mean cv = std/mean
print(f"Mean: {mean} - Variance {variance} - Standard Deviation {std} - Coefficient of Variation: {cv}") app_logger.debug(f"Mean: {mean} - Variance {variance} - Standard Deviation {std} - Coefficient of Variation: {cv}")
if cv >= uneven_request_timing_threshold: if cv >= uneven_request_timing_threshold:
score["attacker"]["uneven_request_timing"] = True score["attacker"]["uneven_request_timing"] = True
@@ -268,10 +271,13 @@ class Analyzer:
regular_user_score = regular_user_score + score["regular_user"]["different_user_agents"] * weights["regular_user"]["different_user_agents"] regular_user_score = regular_user_score + score["regular_user"]["different_user_agents"] * weights["regular_user"]["different_user_agents"]
regular_user_score = regular_user_score + score["regular_user"]["attack_url"] * weights["regular_user"]["attack_url"] regular_user_score = regular_user_score + score["regular_user"]["attack_url"] * weights["regular_user"]["attack_url"]
print(f"Attacker score: {attacker_score}") score_details = f"""
print(f"Good Crawler score: {good_crawler_score}") Attacker score: {attacker_score}
print(f"Bad Crawler score: {bad_crawler_score}") Good Crawler score: {good_crawler_score}
print(f"Regular User score: {regular_user_score}") Bad Crawler score: {bad_crawler_score}
Regular User score: {regular_user_score}
"""
app_logger.debug(score_details)
analyzed_metrics = {"risky_http_methods": http_method_attacker_score, "robots_violations": violated_robots_ratio, "uneven_request_timing": mean, "different_user_agents": user_agents_used, "attack_url": attack_urls_found_list} analyzed_metrics = {"risky_http_methods": http_method_attacker_score, "robots_violations": violated_robots_ratio, "uneven_request_timing": mean, "different_user_agents": user_agents_used, "attack_url": attack_urls_found_list}
category_scores = {"attacker": attacker_score, "good_crawler": good_crawler_score, "bad_crawler": bad_crawler_score, "regular_user": regular_user_score} category_scores = {"attacker": attacker_score, "good_crawler": good_crawler_score, "bad_crawler": bad_crawler_score, "regular_user": regular_user_score}

12
src/database.py
View File

@@ -22,6 +22,9 @@ from sanitizer import (
sanitize_attack_pattern, sanitize_attack_pattern,
) )
from logger import get_app_logger
applogger = get_app_logger()
class DatabaseManager: class DatabaseManager:
""" """
@@ -154,7 +157,7 @@ class DatabaseManager:
except Exception as e: except Exception as e:
session.rollback() session.rollback()
# Log error but don't crash - database persistence is secondary to honeypot function # Log error but don't crash - database persistence is secondary to honeypot function
print(f"Database error persisting access: {e}") applogger.critical(f"Database error persisting access: {e}")
return None return None
finally: finally:
self.close_session() self.close_session()
@@ -193,7 +196,7 @@ class DatabaseManager:
except Exception as e: except Exception as e:
session.rollback() session.rollback()
print(f"Database error persisting credential: {e}") applogger.critical(f"Database error persisting credential: {e}")
return None return None
finally: finally:
self.close_session() self.close_session()
@@ -236,7 +239,8 @@ class DatabaseManager:
last_analysis: timestamp of last analysis last_analysis: timestamp of last analysis
""" """
print(f"Analyzed metrics {analyzed_metrics}, category {category}, category scores {category_scores}, last analysis {last_analysis}") applogger.debug(f"Analyzed metrics {analyzed_metrics}, category {category}, category scores {category_scores}, last analysis {last_analysis}")
applogger.info(f"IP: {ip} category has been updated to {category}")
session = self.session session = self.session
sanitized_ip = sanitize_ip(ip) sanitized_ip = sanitize_ip(ip)
@@ -295,7 +299,7 @@ class DatabaseManager:
session.commit() session.commit()
except Exception as e: except Exception as e:
session.rollback() session.rollback()
print(f"Error recording category change: {e}") applogger.error(f"Error recording category change: {e}")
def get_category_history(self, ip: str) -> List[Dict[str, Any]]: def get_category_history(self, ip: str) -> List[Dict[str, Any]]:
""" """