updated the architecture

app/utils/validators.py

@@ -0,0 +1,46 @@
+import ipaddress
+import socket
+from urllib.parse import urlparse
+
+from fastapi import HTTPException
+
+PRIVATE_NETWORKS = [
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("169.254.0.0/16"),
+    ipaddress.ip_network("0.0.0.0/8"),
+    ipaddress.ip_network("::1/128"),
+    ipaddress.ip_network("fc00::/7"),
+    ipaddress.ip_network("fe80::/10"),
+]
+
+
+def validate_url(url: str) -> str:
+    parsed = urlparse(url)
+
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="URL must use http or https scheme")
+
+    hostname = parsed.hostname
+    if not hostname:
+        raise HTTPException(status_code=400, detail="Invalid URL: no hostname found")
+
+    blocked_hostnames = {"localhost", "0.0.0.0"}
+    if hostname in blocked_hostnames:
+        raise HTTPException(status_code=400, detail="Scanning internal addresses is not allowed")
+
+    try:
+        resolved_ip = socket.gethostbyname(hostname)
+        ip = ipaddress.ip_address(resolved_ip)
+        for network in PRIVATE_NETWORKS:
+            if ip in network:
+                raise HTTPException(
+                    status_code=400,
+                    detail="Scanning internal/private IP addresses is not allowed",
+                )
+    except socket.gaierror:
+        raise HTTPException(status_code=400, detail=f"Could not resolve hostname: {hostname}")
+
+    return url