feat: Gemini AI assessment, Kit Digital detection, contact extraction

Kit Digital detection (enricher.py):
- Scans img src/alt/srcset for digitalizadores, kit-digital, fondos-europeos etc
- Scans page text for Kit Digital, Agente Digitalizador, Next Generation EU, PRTR
- Scans links for acelerapyme.es, red.es, kit-digital refs
- +20 score bonus for Kit Digital confirmed sites (proven IT buyers)

Contact extraction (enricher.py):
- Pulls mailto/tel/wa.me links from HTML
- Extracts email addresses via regex, phone numbers (ES format)
- Detects social media links (FB, IG, LinkedIn, Twitter, TikTok)
- Stored as JSON in contact_info column

Gemini via Replicate (replicate_ai.py):
- Assesses lead quality (HOT/WARM/COLD), Kit Digital confirmation
- Identifies best contact channel + actual value (email/phone/WA)
- Writes Spanish cold-call/email pitch angle
- Lists services likely needed + outreach notes
- 3 concurrent requests, 90s timeout, JSON output parsing

DB: migration adds kit_digital, kit_digital_signals, contact_info,
    ai_assessment, ai_lead_quality, ai_pitch, ai_contact_channel/value,
    ai_queue table

UI: Kit Digital 🏅 badge, AI quality pill (clickable modal with full
    assessment), contact chips (email/phone/WA/social), AI Assess button,
    Kit Digital only filter, AI queue status in enrichment tab

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-13 17:25:06 +02:00
parent 7acff12242
commit faca4b6e1a
7 changed files with 875 additions and 382 deletions

View File

@@ -26,7 +26,16 @@ CREATE TABLE IF NOT EXISTS enriched_domains (
server TEXT, server TEXT,
enriched_at TEXT, enriched_at TEXT,
error TEXT, error TEXT,
score INTEGER DEFAULT 0 score INTEGER DEFAULT 0,
kit_digital INTEGER DEFAULT 0,
kit_digital_signals TEXT,
contact_info TEXT,
ai_assessment TEXT,
ai_lead_quality TEXT,
ai_pitch TEXT,
ai_contact_channel TEXT,
ai_contact_value TEXT,
ai_assessed_at TEXT
); );
CREATE TABLE IF NOT EXISTS job_queue ( CREATE TABLE IF NOT EXISTS job_queue (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -37,6 +46,13 @@ CREATE TABLE IF NOT EXISTS job_queue (
completed_at TEXT, completed_at TEXT,
error TEXT error TEXT
); );
CREATE TABLE IF NOT EXISTS ai_queue (
domain TEXT PRIMARY KEY,
status TEXT DEFAULT 'pending',
created_at TEXT DEFAULT (datetime('now')),
completed_at TEXT,
error TEXT
);
CREATE TABLE IF NOT EXISTS scores ( CREATE TABLE IF NOT EXISTS scores (
domain TEXT PRIMARY KEY, domain TEXT PRIMARY KEY,
score INTEGER NOT NULL, score INTEGER NOT NULL,
@@ -44,6 +60,20 @@ CREATE TABLE IF NOT EXISTS scores (
); );
""" """
# Columns added after initial release — applied as migrations on existing DBs
_MIGRATIONS = [
"ALTER TABLE enriched_domains ADD COLUMN kit_digital INTEGER DEFAULT 0",
"ALTER TABLE enriched_domains ADD COLUMN kit_digital_signals TEXT",
"ALTER TABLE enriched_domains ADD COLUMN contact_info TEXT",
"ALTER TABLE enriched_domains ADD COLUMN ai_assessment TEXT",
"ALTER TABLE enriched_domains ADD COLUMN ai_lead_quality TEXT",
"ALTER TABLE enriched_domains ADD COLUMN ai_pitch TEXT",
"ALTER TABLE enriched_domains ADD COLUMN ai_contact_channel TEXT",
"ALTER TABLE enriched_domains ADD COLUMN ai_contact_value TEXT",
"ALTER TABLE enriched_domains ADD COLUMN ai_assessed_at TEXT",
"CREATE TABLE IF NOT EXISTS ai_queue (domain TEXT PRIMARY KEY, status TEXT DEFAULT 'pending', created_at TEXT DEFAULT (datetime('now')), completed_at TEXT, error TEXT)",
]
# Index build state # Index build state
_index_ready = False _index_ready = False
_index_building = False _index_building = False
@@ -57,6 +87,12 @@ _total_cache: int = 0
async def init_db(): async def init_db():
async with aiosqlite.connect(SQLITE_PATH) as db: async with aiosqlite.connect(SQLITE_PATH) as db:
await db.executescript(SCHEMA) await db.executescript(SCHEMA)
# Run migrations (safe to re-run — silently skips existing columns)
for sql in _MIGRATIONS:
try:
await db.execute(sql)
except Exception:
pass
await db.commit() await db.commit()
@@ -243,6 +279,8 @@ async def get_stats():
threshold = int(os.getenv("SCORE_THRESHOLD", "60")) threshold = int(os.getenv("SCORE_THRESHOLD", "60"))
async with db.execute("SELECT COUNT(*) FROM enriched_domains WHERE score >= ?", (threshold,)) as cur: async with db.execute("SELECT COUNT(*) FROM enriched_domains WHERE score >= ?", (threshold,)) as cur:
hot_leads = (await cur.fetchone())[0] hot_leads = (await cur.fetchone())[0]
async with db.execute("SELECT COUNT(*) FROM enriched_domains WHERE kit_digital=1") as cur:
kit_digital_count = (await cur.fetchone())[0]
async with db.execute("SELECT status, COUNT(*) FROM job_queue GROUP BY status") as cur: async with db.execute("SELECT status, COUNT(*) FROM job_queue GROUP BY status") as cur:
q = {r[0]: r[1] async for r in cur} q = {r[0]: r[1] async for r in cur}
@@ -250,6 +288,7 @@ async def get_stats():
"total_domains": _total_cache, "total_domains": _total_cache,
"enriched": enriched, "enriched": enriched,
"hot_leads": hot_leads, "hot_leads": hot_leads,
"kit_digital_count": kit_digital_count,
"tld_breakdown": _tld_cache, "tld_breakdown": _tld_cache,
"index_status": index_status(), "index_status": index_status(),
"queue": { "queue": {
@@ -263,7 +302,7 @@ async def get_stats():
# ── Enrichment helpers ─────────────────────────────────────────────────────── # ── Enrichment helpers ───────────────────────────────────────────────────────
async def get_enriched(min_score=0, cms=None, country=None, page=1, limit=100): async def get_enriched(min_score=0, cms=None, country=None, kit_digital=None, page=1, limit=100):
offset = (page - 1) * limit offset = (page - 1) * limit
conditions = ["score >= ?"] conditions = ["score >= ?"]
params: list = [min_score] params: list = [min_score]
@@ -273,6 +312,9 @@ async def get_enriched(min_score=0, cms=None, country=None, page=1, limit=100):
if country: if country:
conditions.append("ip_country = ?") conditions.append("ip_country = ?")
params.append(country) params.append(country)
if kit_digital is not None:
conditions.append("kit_digital = ?")
params.append(1 if kit_digital else 0)
where = "WHERE " + " AND ".join(conditions) where = "WHERE " + " AND ".join(conditions)
async with aiosqlite.connect(SQLITE_PATH) as db: async with aiosqlite.connect(SQLITE_PATH) as db:
db.row_factory = aiosqlite.Row db.row_factory = aiosqlite.Row
@@ -288,6 +330,52 @@ async def get_enriched(min_score=0, cms=None, country=None, page=1, limit=100):
return total, rows return total, rows
async def queue_ai(domains: list[str]):
async with aiosqlite.connect(SQLITE_PATH) as db:
await db.executemany(
"INSERT OR IGNORE INTO ai_queue (domain) VALUES (?)",
[(d,) for d in domains],
)
await db.commit()
async def get_ai_queue_status():
async with aiosqlite.connect(SQLITE_PATH) as db:
async with db.execute("SELECT status, COUNT(*) FROM ai_queue GROUP BY status") as cur:
rows = {r[0]: r[1] async for r in cur}
return {
"pending": rows.get("pending", 0),
"running": rows.get("running", 0),
"done": rows.get("done", 0),
"failed": rows.get("failed", 0),
"total": sum(rows.values()),
}
async def save_ai_assessment(domain: str, assessment: dict):
import json as _json
async with aiosqlite.connect(SQLITE_PATH) as db:
await db.execute(
"""UPDATE enriched_domains SET
ai_assessment=?, ai_lead_quality=?, ai_pitch=?,
ai_contact_channel=?, ai_contact_value=?, ai_assessed_at=datetime('now')
WHERE domain=?""",
(
_json.dumps(assessment),
assessment.get("lead_quality"),
assessment.get("pitch_angle"),
assessment.get("best_contact_channel"),
assessment.get("best_contact_value"),
domain,
),
)
await db.execute(
"UPDATE ai_queue SET status='done', completed_at=datetime('now') WHERE domain=?",
(domain,),
)
await db.commit()
async def queue_domains(domains: list[str]): async def queue_domains(domains: list[str]):
async with aiosqlite.connect(SQLITE_PATH) as db: async with aiosqlite.connect(SQLITE_PATH) as db:
await db.executemany( await db.executemany(

View File

@@ -1,5 +1,7 @@
import asyncio import asyncio
import json
import os import os
import re
import ssl import ssl
import socket import socket
import datetime import datetime
@@ -11,49 +13,149 @@ import dns.resolver
import aiosqlite import aiosqlite
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from app.db import SQLITE_PATH from app.db import SQLITE_PATH, queue_ai, save_ai_assessment, get_ai_queue_status
from app.scorer import score from app.scorer import score
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
CONCURRENCY_LIMIT = int(os.getenv("CONCURRENCY_LIMIT", "50")) CONCURRENCY_LIMIT = int(os.getenv("CONCURRENCY_LIMIT", "50"))
# ip-api.com free tier: 45 req/min → ~1.33/s. We use a separate slower semaphore. IP_API_RATE = 45 # req/min free tier
IP_API_SEMAPHORE: Optional[asyncio.Semaphore] = None
IP_API_RATE = 45 # per minute
_worker_task: Optional[asyncio.Task] = None _worker_task: Optional[asyncio.Task] = None
_ai_worker_task: Optional[asyncio.Task] = None
_paused = False _paused = False
_ip_sem: Optional[asyncio.Semaphore] = None
_ip_last_call = 0.0
def get_ip_semaphore(): def _get_ip_sem():
global IP_API_SEMAPHORE global _ip_sem
if IP_API_SEMAPHORE is None: if _ip_sem is None:
IP_API_SEMAPHORE = asyncio.Semaphore(1) _ip_sem = asyncio.Semaphore(1)
return IP_API_SEMAPHORE return _ip_sem
# ── CMS detection ────────────────────────────────────────────────────────────
CMS_SIGNATURES = { CMS_SIGNATURES = {
"wordpress": ["/wp-content/", "/wp-includes/", 'name="generator" content="WordPress'], "wordpress": ["/wp-content/", "/wp-includes/", 'content="WordPress'],
"joomla": ["/components/com_", "Joomla!", 'name="generator" content="Joomla'], "joomla": ["/components/com_", "Joomla!", 'content="Joomla'],
"drupal": ["/sites/default/files/", "Drupal.settings", 'name="generator" content="Drupal'], "drupal": ["/sites/default/files/", "Drupal.settings", 'content="Drupal'],
"wix": ["wix.com", "X-Wix-"], "wix": ["static.wixstatic.com", "X-Wix-"],
"squarespace": ["squarespace.com", "X-Squarespace-"], "squarespace": ["squarespace.com", "X-Squarespace-"],
"shopify": ["cdn.shopify.com", "Shopify.theme"], "shopify": ["cdn.shopify.com", "Shopify.theme"],
"prestashop": ["PrestaShop", "/modules/"], "prestashop": ["PrestaShop", "/modules/prestashop"],
"magento": ["Mage.Cookies", "X-Magento-"], "magento": ["Mage.Cookies", "X-Magento-"],
"typo3": ["typo3", "TYPO3 CMS"], "typo3": ["typo3temp", "TYPO3 CMS"],
"opencart": ["route=common/home", "OpenCart"], "opencart": ["route=common/home", "OpenCart"],
} }
def detect_cms(html: str, headers: dict) -> Optional[str]: def detect_cms(html: str, headers: dict) -> Optional[str]:
combined = html[:50000] + " ".join(f"{k}:{v}" for k, v in headers.items()) combined = html[:60000] + " ".join(f"{k}:{v}" for k, v in headers.items())
cl = combined.lower()
for cms, sigs in CMS_SIGNATURES.items(): for cms, sigs in CMS_SIGNATURES.items():
if any(sig.lower() in combined.lower() for sig in sigs): if any(s.lower() in cl for s in sigs):
return cms return cms
return None return None
# ── Kit Digital detection ────────────────────────────────────────────────────
KIT_IMG_PATS = [
"digitalizadores", "kit-digital", "kitdigital", "kit_digital",
"fondos-europeos", "fondos_europeos", "nextgeneration", "next-generation",
"prtr", "plan-recuperacion", "planderecuperacion",
"acelerapyme", "logo-ue", "recovery-eu", "cofinanciado",
]
KIT_TEXT_PATS = [
"kit digital", "agente digitalizador", "agentes digitalizadores",
"fondos europeos", "next generation eu", "nextgenerationeu",
"plan de recuperación", "plan de recuperacion",
"plan de digitalización", "digitalización pymes",
"prtr", "financiado por la unión europea",
"red.es/kit-digital", "acelerapyme.es",
]
KIT_LINK_PATS = ["acelerapyme", "red.es", "kit-digital", "kitdigital"]
def detect_kit_digital(soup, html: str) -> tuple[bool, list]:
signals = []
hl = html.lower()
for img in soup.find_all("img"):
combined = ((img.get("src") or "") + (img.get("alt") or "") + (img.get("srcset") or "")).lower()
for p in KIT_IMG_PATS:
if p in combined:
signals.append(f"img:{p}")
break
for p in KIT_TEXT_PATS:
if p in hl:
signals.append(f"text:{p}")
for a in soup.find_all("a", href=True):
href = a["href"].lower()
if any(p in href for p in KIT_LINK_PATS):
signals.append(f"link:{href[:60]}")
signals = list(dict.fromkeys(signals))[:15]
return len(signals) > 0, signals
# ── Contact extraction ────────────────────────────────────────────────────────
EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}")
PHONE_RE = re.compile(r"(?:\+34[\s\-]?)?(?:6|7|8|9)\d{2}[\s\-]?\d{3}[\s\-]?\d{3}")
SOCIAL_DOMAINS = ["facebook.com", "instagram.com", "linkedin.com", "twitter.com", "x.com", "tiktok.com"]
def extract_contacts(soup, html: str) -> dict:
contacts: dict = {"emails": [], "phones": [], "whatsapp": [], "social": []}
# mailto links
for a in soup.find_all("a", href=True):
href = a["href"]
if href.startswith("mailto:"):
em = href[7:].split("?")[0].strip()
if em and em not in contacts["emails"]:
contacts["emails"].append(em)
elif href.startswith("tel:"):
ph = re.sub(r"[^\d+]", "", href[4:])
if ph and ph not in contacts["phones"]:
contacts["phones"].append(ph)
elif "wa.me" in href or "api.whatsapp.com" in href:
if href not in contacts["whatsapp"]:
contacts["whatsapp"].append(href[:80])
else:
for sd in SOCIAL_DOMAINS:
if sd in href.lower():
clean = href.split("?")[0].rstrip("/")
if clean not in contacts["social"]:
contacts["social"].append(clean)
break
# Email regex in raw HTML (catches obfuscated ones)
for em in EMAIL_RE.findall(html[:100000]):
em = em.lower()
if em not in contacts["emails"] and not em.endswith((".png", ".jpg", ".css", ".js")):
contacts["emails"].append(em)
# Phone numbers in visible text
for ph in PHONE_RE.findall(soup.get_text()):
ph_clean = re.sub(r"[\s\-]", "", ph)
if ph_clean not in contacts["phones"]:
contacts["phones"].append(ph_clean)
# Dedupe + cap
for k in contacts:
contacts[k] = list(dict.fromkeys(contacts[k]))[:5]
return contacts
# ── SSL / MX / IP ─────────────────────────────────────────────────────────────
async def check_ssl(domain: str) -> tuple[bool, Optional[int]]: async def check_ssl(domain: str) -> tuple[bool, Optional[int]]:
try: try:
ctx = ssl.create_default_context() ctx = ssl.create_default_context()
@@ -63,11 +165,8 @@ async def check_ssl(domain: str) -> tuple[bool, Optional[int]]:
with socket.create_connection((domain, 443), timeout=5) as sock: with socket.create_connection((domain, 443), timeout=5) as sock:
with ctx.wrap_socket(sock, server_hostname=domain) as ssock: with ctx.wrap_socket(sock, server_hostname=domain) as ssock:
cert = ssock.getpeercert() cert = ssock.getpeercert()
expiry_str = cert.get("notAfter", "") expiry = datetime.datetime.strptime(cert["notAfter"], "%b %d %H:%M:%S %Y %Z")
expiry = datetime.datetime.strptime(expiry_str, "%b %d %H:%M:%S %Y %Z") return True, (expiry - datetime.datetime.utcnow()).days
days = (expiry - datetime.datetime.utcnow()).days
return True, days
return await loop.run_in_executor(None, _check) return await loop.run_in_executor(None, _check)
except Exception: except Exception:
return False, None return False, None
@@ -76,64 +175,53 @@ async def check_ssl(domain: str) -> tuple[bool, Optional[int]]:
async def check_mx(domain: str) -> bool: async def check_mx(domain: str) -> bool:
try: try:
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
def _check(): def _check():
try: try:
answers = dns.resolver.resolve(domain, "MX", lifetime=5) return len(dns.resolver.resolve(domain, "MX", lifetime=5)) > 0
return len(answers) > 0
except Exception: except Exception:
return False return False
return await loop.run_in_executor(None, _check) return await loop.run_in_executor(None, _check)
except Exception: except Exception:
return False return False
_ip_last_call = 0.0
_ip_lock = asyncio.Lock() if False else None # initialized lazily
async def get_ip_country(ip: str) -> Optional[str]: async def get_ip_country(ip: str) -> Optional[str]:
global _ip_last_call global _ip_last_call
# Enforce 45 req/min = 1 req per 1.33s async with _get_ip_sem():
async with get_ip_semaphore():
now = asyncio.get_event_loop().time() now = asyncio.get_event_loop().time()
wait = (1 / (IP_API_RATE / 60)) - (now - _ip_last_call) wait = (60 / IP_API_RATE) - (now - _ip_last_call)
if wait > 0: if wait > 0:
await asyncio.sleep(wait) await asyncio.sleep(wait)
_ip_last_call = asyncio.get_event_loop().time() _ip_last_call = asyncio.get_event_loop().time()
try: try:
async with httpx.AsyncClient(timeout=5) as client: async with httpx.AsyncClient(timeout=5) as client:
resp = await client.get(f"http://ip-api.com/json/{ip}?fields=countryCode") r = await client.get(f"http://ip-api.com/json/{ip}?fields=countryCode")
if resp.status_code == 200: if r.status_code == 200:
return resp.json().get("countryCode") return r.json().get("countryCode")
except Exception: except Exception:
pass pass
return None return None
# ── Main enrichment ───────────────────────────────────────────────────────────
async def enrich_domain(domain: str) -> dict: async def enrich_domain(domain: str) -> dict:
result = { result = {
"domain": domain, "domain": domain,
"is_live": False, "is_live": False, "status_code": None,
"status_code": None, "ssl_valid": False, "ssl_expiry_days": None,
"ssl_valid": False, "cms": None, "has_mx": False,
"ssl_expiry_days": None, "ip_country": None, "page_title": None,
"cms": None,
"has_mx": False,
"ip_country": None,
"page_title": None,
"server": None, "server": None,
"kit_digital": False, "kit_digital_signals": "[]",
"contact_info": "{}",
"enriched_at": datetime.datetime.utcnow().isoformat(), "enriched_at": datetime.datetime.utcnow().isoformat(),
"error": None, "error": None,
} }
try: try:
async with httpx.AsyncClient( async with httpx.AsyncClient(
timeout=10, timeout=12, follow_redirects=True, verify=False,
follow_redirects=True,
verify=False,
headers={"User-Agent": "Mozilla/5.0 (compatible; DomGod/1.0)"}, headers={"User-Agent": "Mozilla/5.0 (compatible; DomGod/1.0)"},
) as client: ) as client:
resp = await client.get(f"http://{domain}") resp = await client.get(f"http://{domain}")
@@ -143,11 +231,16 @@ async def enrich_domain(domain: str) -> dict:
html = resp.text html = resp.text
soup = BeautifulSoup(html, "html.parser") soup = BeautifulSoup(html, "html.parser")
title_tag = soup.find("title") title = soup.find("title")
result["page_title"] = title_tag.get_text(strip=True)[:500] if title_tag else None result["page_title"] = title.get_text(strip=True)[:500] if title else None
result["cms"] = detect_cms(html, dict(resp.headers)) result["cms"] = detect_cms(html, dict(resp.headers))
# Resolve IP for country lookup kit, signals = detect_kit_digital(soup, html)
result["kit_digital"] = kit
result["kit_digital_signals"] = json.dumps(signals)
result["contact_info"] = json.dumps(extract_contacts(soup, html))
try: try:
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
ip = await loop.run_in_executor(None, socket.gethostbyname, domain) ip = await loop.run_in_executor(None, socket.gethostbyname, domain)
@@ -158,15 +251,10 @@ async def enrich_domain(domain: str) -> dict:
except Exception as e: except Exception as e:
result["error"] = str(e)[:500] result["error"] = str(e)[:500]
# SSL check (independent of HTTP)
ssl_valid, ssl_days = await check_ssl(domain) ssl_valid, ssl_days = await check_ssl(domain)
result["ssl_valid"] = ssl_valid result["ssl_valid"] = ssl_valid
result["ssl_expiry_days"] = ssl_days result["ssl_expiry_days"] = ssl_days
# MX check
result["has_mx"] = await check_mx(domain) result["has_mx"] = await check_mx(domain)
# Score
result["score"] = score(result) result["score"] = score(result)
return result return result
@@ -177,19 +265,24 @@ async def save_enriched(data: dict):
await db.execute( await db.execute(
"""INSERT INTO enriched_domains """INSERT INTO enriched_domains
(domain, is_live, status_code, ssl_valid, ssl_expiry_days, cms, (domain, is_live, status_code, ssl_valid, ssl_expiry_days, cms,
has_mx, ip_country, page_title, server, enriched_at, error, score) has_mx, ip_country, page_title, server, enriched_at, error, score,
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) kit_digital, kit_digital_signals, contact_info)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON CONFLICT(domain) DO UPDATE SET ON CONFLICT(domain) DO UPDATE SET
is_live=excluded.is_live, status_code=excluded.status_code, is_live=excluded.is_live, status_code=excluded.status_code,
ssl_valid=excluded.ssl_valid, ssl_expiry_days=excluded.ssl_expiry_days, ssl_valid=excluded.ssl_valid, ssl_expiry_days=excluded.ssl_expiry_days,
cms=excluded.cms, has_mx=excluded.has_mx, ip_country=excluded.ip_country, cms=excluded.cms, has_mx=excluded.has_mx, ip_country=excluded.ip_country,
page_title=excluded.page_title, server=excluded.server, page_title=excluded.page_title, server=excluded.server,
enriched_at=excluded.enriched_at, error=excluded.error, score=excluded.score""", enriched_at=excluded.enriched_at, error=excluded.error, score=excluded.score,
kit_digital=excluded.kit_digital,
kit_digital_signals=excluded.kit_digital_signals,
contact_info=excluded.contact_info""",
( (
data["domain"], data["is_live"], data["status_code"], data["domain"], data["is_live"], data["status_code"],
data["ssl_valid"], data["ssl_expiry_days"], data["cms"], data["ssl_valid"], data["ssl_expiry_days"], data["cms"],
data["has_mx"], data["ip_country"], data["page_title"], data["has_mx"], data["ip_country"], data["page_title"],
data["server"], data["enriched_at"], data["error"], data["score"], data["server"], data["enriched_at"], data["error"], data["score"],
int(data["kit_digital"]), data["kit_digital_signals"], data["contact_info"],
), ),
) )
await db.execute( await db.execute(
@@ -205,18 +298,17 @@ async def mark_job(domain: str, status: str, error: str = None):
if status == "running": if status == "running":
await db.execute( await db.execute(
"UPDATE job_queue SET status=?, started_at=datetime('now') WHERE domain=?", "UPDATE job_queue SET status=?, started_at=datetime('now') WHERE domain=?",
(status, domain), (status, domain))
)
elif status in ("done", "failed"): elif status in ("done", "failed"):
await db.execute( await db.execute(
"UPDATE job_queue SET status=?, completed_at=datetime('now'), error=? WHERE domain=?", "UPDATE job_queue SET status=?, completed_at=datetime('now'), error=? WHERE domain=?",
(status, error, domain), (status, error, domain))
)
await db.commit() await db.commit()
# ── Enrichment worker ─────────────────────────────────────────────────────────
async def worker_loop(): async def worker_loop():
global _paused
sem = asyncio.Semaphore(CONCURRENCY_LIMIT) sem = asyncio.Semaphore(CONCURRENCY_LIMIT)
async def process(domain: str): async def process(domain: str):
@@ -233,26 +325,70 @@ async def worker_loop():
if _paused: if _paused:
await asyncio.sleep(1) await asyncio.sleep(1)
continue continue
async with aiosqlite.connect(SQLITE_PATH) as db: async with aiosqlite.connect(SQLITE_PATH) as db:
async with db.execute( async with db.execute(
"SELECT domain FROM job_queue WHERE status='pending' LIMIT 100" "SELECT domain FROM job_queue WHERE status='pending' LIMIT 100"
) as cur: ) as cur:
rows = await cur.fetchall() rows = await cur.fetchall()
if not rows: if not rows:
await asyncio.sleep(2) await asyncio.sleep(2)
continue continue
await asyncio.gather(*[asyncio.create_task(process(r[0])) for r in rows], return_exceptions=True)
tasks = [asyncio.create_task(process(r[0])) for r in rows]
await asyncio.gather(*tasks, return_exceptions=True) # ── AI assessment worker ──────────────────────────────────────────────────────
async def ai_worker_loop():
from app.replicate_ai import assess_domain as gemini_assess
while True:
async with aiosqlite.connect(SQLITE_PATH) as db:
async with db.execute(
"SELECT domain FROM ai_queue WHERE status='pending' LIMIT 20"
) as cur:
rows = await cur.fetchall()
# Mark as running
if rows:
await db.executemany(
"UPDATE ai_queue SET status='running', created_at=created_at WHERE domain=?",
[(r[0],) for r in rows],
)
await db.commit()
if not rows:
await asyncio.sleep(3)
continue
async def assess_one(domain: str):
try:
async with aiosqlite.connect(SQLITE_PATH) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"SELECT * FROM enriched_domains WHERE domain=?", (domain,)
) as cur:
row = await cur.fetchone()
if not row:
return
assessment = await gemini_assess(dict(row))
await save_ai_assessment(domain, assessment)
except Exception as e:
async with aiosqlite.connect(SQLITE_PATH) as db:
await db.execute(
"UPDATE ai_queue SET status='failed', completed_at=datetime('now') WHERE domain=?",
(domain,),
)
await db.commit()
logger.error("AI worker error %s: %s", domain, e)
await asyncio.gather(*[asyncio.create_task(assess_one(r[0])) for r in rows], return_exceptions=True)
def start_worker(): def start_worker():
global _worker_task global _worker_task, _ai_worker_task
if _worker_task is None or _worker_task.done(): if _worker_task is None or _worker_task.done():
_worker_task = asyncio.create_task(worker_loop()) _worker_task = asyncio.create_task(worker_loop())
_paused = False if _ai_worker_task is None or _ai_worker_task.done():
_ai_worker_task = asyncio.create_task(ai_worker_loop())
def pause_worker(): def pause_worker():

View File

@@ -6,6 +6,7 @@ from contextlib import asynccontextmanager
import httpx import httpx
import aiosqlite import aiosqlite
from typing import Optional
from fastapi import FastAPI, Query from fastapi import FastAPI, Query
from fastapi.responses import StreamingResponse, JSONResponse from fastapi.responses import StreamingResponse, JSONResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
@@ -17,6 +18,7 @@ from app.db import (
DATA_DIR, PARQUET_PATH, SQLITE_PATH, DATA_DIR, PARQUET_PATH, SQLITE_PATH,
init_db, get_stats, get_domains, get_enriched, init_db, get_stats, get_domains, get_enriched,
queue_domains, get_queue_status, build_duckdb_index, index_status, queue_domains, get_queue_status, build_duckdb_index, index_status,
queue_ai, get_ai_queue_status, save_ai_assessment,
) )
from app.enricher import start_worker, pause_worker, resume_worker, is_running from app.enricher import start_worker, pause_worker, resume_worker, is_running
from app.scorer import run_scoring from app.scorer import run_scoring
@@ -146,13 +148,53 @@ async def enriched(
min_score: int = Query(0, ge=0, le=100), min_score: int = Query(0, ge=0, le=100),
cms: str = Query(None), cms: str = Query(None),
country: str = Query(None), country: str = Query(None),
kit_digital: Optional[bool] = Query(None),
page: int = Query(1, ge=1), page: int = Query(1, ge=1),
limit: int = Query(100, ge=1, le=1000), limit: int = Query(100, ge=1, le=1000),
): ):
total, rows = await get_enriched(min_score=min_score, cms=cms, country=country, page=page, limit=limit) total, rows = await get_enriched(
min_score=min_score, cms=cms, country=country,
kit_digital=kit_digital, page=page, limit=limit,
)
return {"page": page, "limit": limit, "total": total, "results": rows} return {"page": page, "limit": limit, "total": total, "results": rows}
# ── AI assessment endpoints ───────────────────────────────────────────────────
@app.post("/api/ai/assess/batch")
async def ai_assess_batch(body: dict):
domains_list = body.get("domains", [])
if not domains_list:
return JSONResponse({"error": "no domains provided"}, status_code=400)
await queue_ai(domains_list)
return {"queued": len(domains_list)}
@app.get("/api/ai/status")
async def ai_status():
return await get_ai_queue_status()
@app.post("/api/ai/assess/single")
async def ai_assess_single(body: dict):
"""Immediate (blocking) AI assessment of a single domain."""
domain = body.get("domain")
if not domain:
return JSONResponse({"error": "no domain"}, status_code=400)
from app.replicate_ai import assess_domain as gemini_assess
async with aiosqlite.connect(SQLITE_PATH) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"SELECT * FROM enriched_domains WHERE domain=?", (domain,)
) as cur:
row = await cur.fetchone()
if not row:
return JSONResponse({"error": "domain not yet enriched"}, status_code=404)
assessment = await gemini_assess(dict(row))
await save_ai_assessment(domain, assessment)
return assessment
@app.get("/api/export") @app.get("/api/export")
async def export_csv( async def export_csv(
min_score: int = Query(0), min_score: int = Query(0),

142
app/replicate_ai.py Normal file
View File

@@ -0,0 +1,142 @@
"""Replicate / Gemini integration for domain lead assessment."""
import asyncio
import json
import logging
import os
import re
from typing import Optional
import httpx
logger = logging.getLogger(__name__)
REPLICATE_TOKEN = os.getenv("REPLICATE_API_TOKEN", "r8_6kV2NWMQyPVB9JILHJprrXJJh4vWazA22Osyj")
REPLICATE_MODEL = "https://api.replicate.com/v1/models/google/gemini-3-pro/predictions"
AI_CONCURRENCY = int(os.getenv("AI_CONCURRENCY", "3"))
_ai_sem: Optional[asyncio.Semaphore] = None
def _sem() -> asyncio.Semaphore:
global _ai_sem
if _ai_sem is None:
_ai_sem = asyncio.Semaphore(AI_CONCURRENCY)
return _ai_sem
def _build_prompt(row: dict) -> str:
kit_signals = row.get("kit_digital_signals") or "[]"
try:
sigs = json.loads(kit_signals)
kit_block = "\n".join(f" - {s}" for s in sigs) if sigs else " None detected"
except Exception:
kit_block = f" {kit_signals}"
contact_raw = row.get("contact_info") or "{}"
try:
contacts = json.loads(contact_raw)
except Exception:
contacts = {}
contact_block = []
if contacts.get("emails"):
contact_block.append(f" Emails: {', '.join(contacts['emails'][:3])}")
if contacts.get("phones"):
contact_block.append(f" Phones: {', '.join(contacts['phones'][:3])}")
if contacts.get("whatsapp"):
contact_block.append(f" WhatsApp: {', '.join(contacts['whatsapp'][:2])}")
if contacts.get("social"):
contact_block.append(f" Social: {', '.join(contacts['social'][:4])}")
contact_str = "\n".join(contact_block) if contact_block else " None found"
return f"""You are a sales intelligence analyst evaluating Spanish SME websites for IT services upsell.
DOMAIN DATA:
- Domain: {row.get("domain")}
- Page title: {row.get("page_title") or "N/A"}
- CMS: {row.get("cms") or "unknown"}
- Server: {row.get("server") or "unknown"}
- Country: {row.get("ip_country") or "unknown"}
- SSL valid: {row.get("ssl_valid")}, expires in {row.get("ssl_expiry_days") or "?"} days
- Has email (MX): {bool(row.get("has_mx"))}
- Is live: {bool(row.get("is_live"))}
- Kit Digital signals found on page:
{kit_block}
- Contact channels found on page:
{contact_str}
Kit Digital is a Spanish government program (up to €12k grants for SME digitalization). Sites that received it MUST display EU/digitalizadores logos. These businesses have proven they invest in IT services and may need follow-up: new website, SEO, hosting migration, security, maintenance contracts.
Assess this lead and respond ONLY with valid JSON (no markdown, no explanation outside the JSON):
{{
"is_local_sme": true/false,
"kit_digital_confirmed": true/false,
"kit_digital_reasoning": "1 sentence explaining why or why not",
"lead_quality": "HOT|WARM|COLD",
"lead_reasoning": "1-2 sentences on why this is a good/bad lead for IT services sales",
"best_contact_channel": "email|phone|whatsapp|social|web_form|unknown",
"best_contact_value": "the actual email/phone/URL to use, or empty string",
"pitch_angle": "One concrete opening sentence for a cold email or call in Spanish",
"services_likely_needed": ["service1", "service2"],
"outreach_notes": "Any useful context for the sales rep (language, business type, urgency)"
}}"""
def _parse_output(raw: str) -> dict:
"""Extract JSON from Gemini text output."""
text = re.sub(r"```(?:json)?", "", raw).strip().rstrip("`").strip()
m = re.search(r"\{[\s\S]+\}", text)
if m:
try:
return json.loads(m.group(0))
except json.JSONDecodeError:
pass
return {
"raw": raw[:500],
"lead_quality": "COLD",
"best_contact_channel": "unknown",
"best_contact_value": "",
"parse_error": True,
}
async def assess_domain(row: dict) -> dict:
"""Call Gemini via Replicate to assess a domain. Returns parsed assessment dict."""
async with _sem():
payload = {
"input": {
"prompt": _build_prompt(row),
"images": [],
"videos": [],
"top_p": 0.9,
"temperature": 0.2,
"thinking_level": "low",
"max_output_tokens": 1024,
}
}
try:
async with httpx.AsyncClient(timeout=90) as client:
resp = await client.post(
REPLICATE_MODEL,
headers={
"Authorization": f"Bearer {REPLICATE_TOKEN}",
"Content-Type": "application/json",
"Prefer": "wait",
},
json=payload,
)
resp.raise_for_status()
data = resp.json()
output = data.get("output", "")
if isinstance(output, list):
output = "".join(output)
result = _parse_output(output)
logger.info("AI %s%s / contact: %s",
row.get("domain"), result.get("lead_quality"), result.get("best_contact_channel"))
return result
except Exception as e:
logger.error("Replicate error %s: %s", row.get("domain"), e)
return {"error": str(e)[:300], "lead_quality": "COLD", "best_contact_channel": "unknown", "best_contact_value": ""}

View File

@@ -41,6 +41,9 @@ def score(domain_row: dict) -> int:
s += 10 s += 10
if local_biz_keywords(domain_row.get("page_title")): if local_biz_keywords(domain_row.get("page_title")):
s += 5 s += 5
# Kit Digital: proven buyer of IT services
if domain_row.get("kit_digital"):
s += 20
return min(s, 100) return min(s, 100)

View File

@@ -10,109 +10,139 @@
:root{ :root{
--bg:#0f1117;--surface:#1a1d27;--surface2:#222638;--border:#2e3250; --bg:#0f1117;--surface:#1a1d27;--surface2:#222638;--border:#2e3250;
--accent:#6c63ff;--accent2:#00d4aa;--danger:#ff4f6d;--warn:#ffb347; --accent:#6c63ff;--accent2:#00d4aa;--danger:#ff4f6d;--warn:#ffb347;
--text:#e8eaf0; --muted:#8891b0; --hot:#ff4f6d; --warm:#ffb347; --cold:#6c7aff; --text:#e8eaf0;--muted:#8891b0;--r:8px;
--r:8px; --kd:#f59e0b;
} }
*{box-sizing:border-box;margin:0;padding:0} *{box-sizing:border-box;margin:0;padding:0}
body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,sans-serif;font-size:14px} body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,sans-serif;font-size:14px}
a{color:var(--accent2);text-decoration:none} a{color:var(--accent2);text-decoration:none}
header{background:var(--surface);border-bottom:1px solid var(--border);padding:12px 20px;display:flex;align-items:center;gap:10px;position:sticky;top:0;z-index:100} header{background:var(--surface);border-bottom:1px solid var(--border);padding:11px 20px;display:flex;align-items:center;gap:8px;position:sticky;top:0;z-index:200}
header h1{font-size:20px;font-weight:800;letter-spacing:-1px} header h1{font-size:20px;font-weight:900;letter-spacing:-1px}
header h1 span{color:var(--accent)} header h1 span{color:var(--accent)}
.badge{background:var(--surface2);border:1px solid var(--border);color:var(--muted);font-size:11px;padding:2px 8px;border-radius:99px} .hbadge{background:var(--surface2);border:1px solid var(--border);color:var(--muted);font-size:11px;padding:2px 8px;border-radius:99px}
.index-pill{font-size:11px;padding:3px 10px;border-radius:99px;font-weight:600} .ipill{font-size:11px;padding:3px 9px;border-radius:99px;font-weight:700}
.index-building{background:#ffb34722;color:var(--warn)} .ip-ok{background:#00d4aa18;color:var(--accent2);border:1px solid #00d4aa33}
.index-ready{background:#00d4aa22;color:var(--accent2)} .ip-bld{background:#ffb34718;color:var(--warn);border:1px solid #ffb34733}
main{padding:16px 20px;display:flex;flex-direction:column;gap:16px;max-width:1440px;margin:0 auto;width:100%} main{padding:14px 20px;display:flex;flex-direction:column;gap:14px;max-width:1500px;margin:0 auto;width:100%}
.card{background:var(--surface);border:1px solid var(--border);border-radius:var(--r);padding:16px} .card{background:var(--surface);border:1px solid var(--border);border-radius:var(--r);padding:14px}
.card-title{font-size:11px;font-weight:700;color:var(--muted);text-transform:uppercase;letter-spacing:.6px;margin-bottom:12px} .ct{font-size:11px;font-weight:700;color:var(--muted);text-transform:uppercase;letter-spacing:.6px;margin-bottom:10px}
/* Stats */ /* Stats */
.stats-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px} .sg{display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:10px}
.stat{background:var(--surface2);border:1px solid var(--border);border-radius:var(--r);padding:12px 14px} .sb{background:var(--surface2);border:1px solid var(--border);border-radius:var(--r);padding:11px 13px}
.stat .lbl{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.4px} .sb .l{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.3px}
.stat .val{font-size:24px;font-weight:800;margin-top:2px} .sb .v{font-size:22px;font-weight:800;margin-top:2px}
.stat .sub{font-size:11px;color:var(--muted);margin-top:1px} .sb .s{font-size:11px;color:var(--muted);margin-top:1px}
.c-accent{color:var(--accent2)} .c-hot{color:var(--hot)} .c-warn{color:var(--warn)} .c-muted{color:var(--muted)} .c1{color:var(--accent2)} .c2{color:var(--danger)} .c3{color:var(--warn)} .c4{color:var(--muted)}
.ckd{color:var(--kd)}
/* Tabs */ /* Tabs */
.tabs{display:flex;gap:2px;padding:0 2px;border-bottom:1px solid var(--border)} .tabs{display:flex;gap:2px;border-bottom:1px solid var(--border)}
.tab{padding:8px 16px;cursor:pointer;font-size:13px;font-weight:500;color:var(--muted);border-radius:6px 6px 0 0;border:1px solid transparent;border-bottom:none;user-select:none} .tab{padding:8px 15px;cursor:pointer;font-size:13px;font-weight:500;color:var(--muted);border-radius:6px 6px 0 0;border:1px solid transparent;border-bottom:none;user-select:none}
.tab.active{background:var(--surface);color:var(--text);border-color:var(--border)} .tab.active{background:var(--surface);color:var(--text);border-color:var(--border)}
.tab:hover:not(.active){color:var(--text)} .tab:hover:not(.active){color:var(--text)}
/* Filters */ /* Filters */
.filters{display:flex;flex-wrap:wrap;gap:10px;align-items:flex-end;margin-bottom:12px} .frow{display:flex;flex-wrap:wrap;gap:8px;align-items:flex-end;margin-bottom:10px}
.field{display:flex;flex-direction:column;gap:3px} .field{display:flex;flex-direction:column;gap:3px}
.field label{font-size:11px;color:var(--muted);text-transform:uppercase;font-weight:600} .field label{font-size:11px;color:var(--muted);text-transform:uppercase;font-weight:600}
input[type=text],input[type=number],select{ input[type=text],input[type=number],select{
background:var(--surface2);border:1px solid var(--border);color:var(--text); background:var(--surface2);border:1px solid var(--border);color:var(--text);
padding:6px 10px;border-radius:6px;font-size:13px;outline:none padding:5px 9px;border-radius:6px;font-size:13px;outline:none
} }
input[type=text]:focus,select:focus{border-color:var(--accent)} input[type=text]:focus,select:focus{border-color:var(--accent)}
input[type=range]{accent-color:var(--accent);width:110px;cursor:pointer} input[type=range]{accent-color:var(--accent);width:100px;cursor:pointer}
.toggle{display:flex;align-items:center;gap:6px;cursor:pointer;padding:6px 0} .tog{display:flex;align-items:center;gap:5px;cursor:pointer;padding:5px 0;font-size:12px;color:var(--muted)}
.toggle input{accent-color:var(--accent);width:15px;height:15px;cursor:pointer} .tog input{accent-color:var(--accent);width:14px;height:14px;cursor:pointer}
.tog strong{color:var(--text)}
/* Buttons */ /* Buttons */
.btn{padding:6px 14px;border-radius:6px;font-size:13px;font-weight:600;cursor:pointer;border:none;transition:opacity .15s;white-space:nowrap} .btn{padding:6px 13px;border-radius:6px;font-size:12px;font-weight:700;cursor:pointer;border:none;transition:opacity .15s;white-space:nowrap}
.btn:hover:not(:disabled){opacity:.85} .btn:disabled{opacity:.35;cursor:not-allowed} .btn:hover:not(:disabled){opacity:.82} .btn:disabled{opacity:.35;cursor:not-allowed}
.btn-primary{background:var(--accent);color:#fff} .bp{background:var(--accent);color:#fff}
.btn-success{background:var(--accent2);color:#111} .bs{background:var(--accent2);color:#111}
.btn-danger{background:var(--danger);color:#fff} .bd{background:var(--danger);color:#fff}
.btn-warn{background:var(--warn);color:#111} .bw{background:var(--warn);color:#111}
.btn-ghost{background:var(--surface2);color:var(--text);border:1px solid var(--border)} .bg{background:var(--surface2);color:var(--text);border:1px solid var(--border)}
.btn-sm{padding:4px 10px;font-size:12px} .bkd{background:#f59e0b22;color:var(--kd);border:1px solid #f59e0b44}
.bai{background:#a855f722;color:#c084fc;border:1px solid #a855f744}
.sm{padding:4px 9px;font-size:11px}
/* Table */ /* Table */
.table-wrap{overflow-x:auto;border-radius:var(--r);border:1px solid var(--border)} .tw{overflow-x:auto;border-radius:var(--r);border:1px solid var(--border)}
table{width:100%;border-collapse:collapse;font-size:13px} table{width:100%;border-collapse:collapse;font-size:12px}
th{text-align:left;padding:8px 10px;font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.4px;background:var(--surface2);border-bottom:1px solid var(--border);white-space:nowrap} th{text-align:left;padding:7px 9px;font-size:10px;color:var(--muted);text-transform:uppercase;letter-spacing:.4px;background:var(--surface2);border-bottom:1px solid var(--border);white-space:nowrap}
td{padding:7px 10px;border-bottom:1px solid var(--border);vertical-align:middle} td{padding:6px 9px;border-bottom:1px solid var(--border);vertical-align:middle}
tr:last-child td{border-bottom:none} tr:last-child td{border-bottom:none}
tr:hover td{background:var(--surface2)} tr:hover td{background:rgba(255,255,255,.025)}
.pill{display:inline-block;padding:2px 7px;border-radius:99px;font-size:11px;font-weight:600} .pill{display:inline-block;padding:1px 6px;border-radius:99px;font-size:10px;font-weight:700}
.p-green{background:#00d4aa22;color:var(--accent2)} .p-red{background:#ff4f6d22;color:var(--danger)} .pg{background:#00d4aa18;color:var(--accent2)} .pr{background:#ff4f6d18;color:var(--danger)}
.p-grey{background:#ffffff11;color:var(--muted)} .p-cms{background:#6c63ff22;color:var(--accent)} .pp{background:#ffffff11;color:var(--muted)} .pc{background:#6c63ff18;color:var(--accent)}
.pkd{background:#f59e0b18;color:var(--kd);border:1px solid #f59e0b33}
/* AI quality pills */
.ai-hot{background:#ff4f6d18;color:var(--danger);border:1px solid #ff4f6d33}
.ai-warm{background:#ffb34718;color:var(--warn);border:1px solid #ffb34733}
.ai-cold{background:#6c7aff18;color:#6c7aff;border:1px solid #6c7aff33}
.ai-none{background:#ffffff08;color:var(--muted)}
/* Score badge */ .score{display:inline-block;padding:1px 6px;border-radius:5px;font-weight:800;font-size:11px;min-width:28px;text-align:center}
.score{display:inline-block;padding:2px 7px;border-radius:6px;font-weight:800;font-size:12px;min-width:32px;text-align:center}
/* Contact chips */
.contact-chips{display:flex;flex-wrap:wrap;gap:3px}
.chip{display:inline-flex;align-items:center;gap:3px;padding:1px 6px;border-radius:4px;font-size:10px;background:var(--surface2);border:1px solid var(--border);color:var(--muted);max-width:160px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.chip.email{border-color:#00d4aa33;color:var(--accent2)}
.chip.phone{border-color:#6c63ff33;color:var(--accent)}
.chip.wa{border-color:#22c55e33;color:#4ade80}
.chip.social{border-color:#f59e0b33;color:var(--kd)}
/* Tooltip */
[title]{cursor:help}
.pitch-cell{max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:11px;color:var(--muted);font-style:italic}
/* Pagination */ /* Pagination */
.pager{display:flex;align-items:center;gap:8px;margin-top:12px;flex-wrap:wrap} .pager{display:flex;align-items:center;gap:8px;margin-top:10px;flex-wrap:wrap}
.pager .info{font-size:12px;color:var(--muted)} .pager .inf{font-size:11px;color:var(--muted)}
/* Progress */ /* Progress */
.prog-wrap{background:var(--surface2);border-radius:99px;height:10px;overflow:hidden;margin:8px 0} .pw{background:var(--surface2);border-radius:99px;height:8px;overflow:hidden;margin:6px 0}
.prog-bar{height:100%;background:linear-gradient(90deg,var(--accent),var(--accent2));border-radius:99px;transition:width .5s} .pb{height:100%;background:linear-gradient(90deg,var(--accent),var(--accent2));border-radius:99px;transition:width .5s}
/* Pipeline */ /* Pipeline */
.pipeline{display:grid;grid-template-columns:1fr 1fr 1fr;gap:14px} .pipeline{display:grid;grid-template-columns:1fr 1fr 1fr;gap:12px}
.pipe-col{background:var(--surface2);border-radius:var(--r);border:1px solid var(--border);padding:14px;display:flex;flex-direction:column;gap:8px} .pcol{background:var(--surface2);border-radius:var(--r);border:1px solid var(--border);padding:12px;display:flex;flex-direction:column;gap:7px}
.pipe-col h3{font-size:15px;font-weight:700} .pcol h3{font-size:14px;font-weight:800}
.pipe-col .count{font-size:30px;font-weight:900;line-height:1} .pcol .cnt{font-size:28px;font-weight:900;line-height:1}
.samples{display:flex;flex-direction:column;gap:4px} .samples{display:flex;flex-direction:column;gap:3px}
.sample{font-size:12px;color:var(--muted);padding:4px 8px;background:var(--surface);border-radius:6px;display:flex;justify-content:space-between;align-items:center;gap:8px} .sample{font-size:11px;padding:4px 7px;background:var(--surface);border-radius:5px;display:flex;justify-content:space-between;align-items:center;gap:6px}
/* Enrich stats */ /* Enrich stats */
.eq-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(110px,1fr));gap:10px;margin-bottom:14px} .esg{display:grid;grid-template-columns:repeat(auto-fit,minmax(100px,1fr));gap:8px;margin-bottom:12px}
.eq-stat{background:var(--surface2);border-radius:var(--r);padding:10px;text-align:center} .esb{background:var(--surface2);border-radius:var(--r);padding:9px;text-align:center}
.eq-stat .v{font-size:22px;font-weight:800} .esb .ev{font-size:20px;font-weight:800}
.eq-stat .l{font-size:11px;color:var(--muted);margin-top:2px} .esb .el{font-size:10px;color:var(--muted);margin-top:1px}
/* Toast */ /* Toast */
.toast{position:fixed;bottom:24px;right:24px;background:var(--surface2);border:1px solid var(--border);border-radius:var(--r);padding:12px 18px;font-size:13px;font-weight:600;z-index:9999;display:flex;align-items:center;gap:10px;box-shadow:0 4px 24px #0008;transition:opacity .3s} .toast{position:fixed;bottom:20px;right:20px;background:var(--surface2);border:1px solid var(--border);border-radius:var(--r);padding:10px 16px;font-size:13px;font-weight:600;z-index:9999;transition:opacity .3s;box-shadow:0 4px 20px #0009}
.toast.hidden{opacity:0;pointer-events:none} .toast.hidden{opacity:0;pointer-events:none}
.toast.success{border-color:var(--accent2);color:var(--accent2)} .toast.success{border-color:#00d4aa55;color:var(--accent2)}
.toast.error{border-color:var(--danger);color:var(--danger)} .toast.error{border-color:#ff4f6d55;color:var(--danger)}
.toast.info{border-color:#6c63ff55;color:var(--accent)}
/* Chart */ /* Chart */
.chart-wrap{height:280px} .cw{height:260px}
@media(max-width:700px){.pipeline{grid-template-columns:1fr}.stats-grid{grid-template-columns:1fr 1fr}} /* AI detail modal */
.modal-bg{position:fixed;inset:0;background:#000a;z-index:300;display:flex;align-items:center;justify-content:center}
.modal{background:var(--surface);border:1px solid var(--border);border-radius:var(--r);padding:20px;max-width:500px;width:90%;max-height:80vh;overflow-y:auto}
.modal h2{font-size:16px;font-weight:800;margin-bottom:12px}
.modal .row{display:flex;gap:8px;margin-bottom:8px;font-size:13px}
.modal .label{color:var(--muted);min-width:110px;font-size:12px}
.modal .val{color:var(--text)}
@media(max-width:700px){.pipeline{grid-template-columns:1fr}.sg{grid-template-columns:1fr 1fr}}
</style> </style>
</head> </head>
<body x-data="app()" x-init="init()"> <body x-data="app()" x-init="init()">
@@ -120,56 +150,60 @@ tr:hover td{background:var(--surface2)}
<!-- Toast --> <!-- Toast -->
<div class="toast" :class="[toast.type, {hidden:!toast.show}]" x-text="toast.msg"></div> <div class="toast" :class="[toast.type, {hidden:!toast.show}]" x-text="toast.msg"></div>
<!-- AI Detail Modal -->
<div class="modal-bg" x-show="modal.open" @click.self="modal.open=false" x-cloak>
<div class="modal" @click.stop>
<h2>AI Assessment — <span style="color:var(--accent2)" x-text="modal.domain"></span></h2>
<div class="row"><span class="label">Lead quality</span><span class="val"><span class="pill" :class="aiPillClass(modal.data.lead_quality)" x-text="modal.data.lead_quality || '—'"></span></span></div>
<div class="row"><span class="label">Kit Digital</span><span class="val" x-text="modal.data.kit_digital_confirmed ? '✅ Confirmed' : '❌ Not confirmed'"></span></div>
<div class="row"><span class="label">KD reasoning</span><span class="val" x-text="modal.data.kit_digital_reasoning || '—'"></span></div>
<div class="row"><span class="label">Lead reasoning</span><span class="val" x-text="modal.data.lead_reasoning || '—'"></span></div>
<div class="row"><span class="label">Best channel</span><span class="val" x-text="(modal.data.best_contact_channel || '—') + (modal.data.best_contact_value ? ': ' + modal.data.best_contact_value : '')"></span></div>
<div class="row"><span class="label">Pitch</span><span class="val" style="font-style:italic;color:var(--accent2)" x-text="modal.data.pitch_angle || '—'"></span></div>
<div class="row"><span class="label">Services needed</span><span class="val" x-text="(modal.data.services_likely_needed || []).join(', ') || '—'"></span></div>
<div class="row"><span class="label">Outreach notes</span><span class="val" x-text="modal.data.outreach_notes || '—'"></span></div>
<button class="btn bg" style="margin-top:14px;width:100%" @click="modal.open=false">Close</button>
</div>
</div>
<header> <header>
<h1>Dom<span>God</span></h1> <h1>Dom<span>God</span></h1>
<span class="badge" x-text="stats.total_domains ? stats.total_domains.toLocaleString() + ' domains' : 'Loading…'"></span> <span class="hbadge" x-text="stats.total_domains ? stats.total_domains.toLocaleString()+' domains' : 'Loading…'"></span>
<span class="index-pill" <span class="ipill" :class="indexSt.ready ? 'ip-ok' : 'ip-bld'" x-text="indexSt.ready ? '⚡ Index ready' : '⏳ Building index…'"></span>
:class="indexSt.ready ? 'index-ready' : 'index-building'"
x-text="indexSt.ready ? '⚡ Index ready' : '⏳ Building index…'">
</span>
<span style="flex:1"></span> <span style="flex:1"></span>
<span style="font-size:12px;color:var(--muted)" x-text="stats.enriched ? stats.enriched.toLocaleString() + ' enriched' : ''"></span> <span style="font-size:11px;color:var(--muted)" x-text="aiSt.total ? aiSt.total + ' in AI queue' : ''"></span>
</header> </header>
<main> <main>
<!-- Stats bar --> <!-- Stats bar -->
<div class="card"> <div class="card">
<div class="card-title">Overview</div> <div class="ct">Overview</div>
<div class="stats-grid"> <div class="sg">
<div class="stat"><div class="lbl">Total Domains</div><div class="val c-accent" x-text="stats.total_domains?.toLocaleString() ?? '—'"></div><div class="sub">in dataset</div></div> <div class="sb"><div class="l">Total Domains</div><div class="v c1" x-text="stats.total_domains?.toLocaleString()??'—'"></div></div>
<div class="stat"><div class="lbl">Enriched</div><div class="val c-accent" x-text="stats.enriched?.toLocaleString() ?? '0'"></div><div class="sub" x-text="stats.total_domains ? ((stats.enriched/stats.total_domains*100).toFixed(3)+'%') : ''"></div></div> <div class="sb"><div class="l">Enriched</div><div class="v c1" x-text="stats.enriched?.toLocaleString()??'0'"></div><div class="s" x-text="stats.total_domains?((stats.enriched/stats.total_domains*100).toFixed(3)+'%'):''"></div></div>
<div class="stat"><div class="lbl">Hot Leads</div><div class="val c-hot" x-text="stats.hot_leads?.toLocaleString() ?? '0'"></div><div class="sub">score ≥ 60</div></div> <div class="sb"><div class="l">Hot Leads</div><div class="v c2" x-text="stats.hot_leads?.toLocaleString()??'0'"></div><div class="s">score ≥ 60</div></div>
<div class="stat"><div class="lbl">Queue Pending</div><div class="val c-warn" x-text="stats.queue?.pending?.toLocaleString() ?? '0'"></div><div class="sub" x-text="(stats.queue?.running ?? 0) + ' running'"></div></div> <div class="sb"><div class="l">Kit Digital</div><div class="v ckd" x-text="stats.kit_digital_count?.toLocaleString()??'0'"></div><div class="s">detected</div></div>
<div class="stat"><div class="lbl">Done / Failed</div><div class="val c-muted" x-text="stats.queue?.done?.toLocaleString() ?? '0'"></div><div class="sub" x-text="(stats.queue?.failed ?? 0) + ' failed'"></div></div> <div class="sb"><div class="l">Queue</div><div class="v c3" x-text="stats.queue?.pending?.toLocaleString()??'0'"></div><div class="s" x-text="(stats.queue?.running??0)+' running'"></div></div>
<div class="sb"><div class="l">AI Queue</div><div class="v" style="color:#c084fc" x-text="aiSt.pending??'0'"></div><div class="s" x-text="(aiSt.done??0)+' assessed'"></div></div>
</div> </div>
</div> </div>
<!-- Tabs --> <!-- Tabs -->
<div class="tabs"> <div class="tabs">
<div class="tab" :class="{active:tab==='browse'}" @click="tab='browse'">Browse & Filter</div> <div class="tab" :class="{active:tab==='browse'}" @click="tab='browse'">Browse & Filter</div>
<div class="tab" :class="{active:tab==='enrich'}" @click="tab='enrich'; loadQueue()">Enrichment Queue</div> <div class="tab" :class="{active:tab==='enrich'}" @click="tab='enrich';loadQueue()">Enrichment</div>
<div class="tab" :class="{active:tab==='pipeline'}" @click="tab='pipeline';loadPipeline()">Lead Pipeline</div> <div class="tab" :class="{active:tab==='pipeline'}" @click="tab='pipeline';loadPipeline()">Lead Pipeline</div>
<div class="tab" :class="{active:tab==='chart'}" @click="tab='chart';renderChart()">TLD Chart</div> <div class="tab" :class="{active:tab==='chart'}" @click="tab='chart';renderChart()">TLD Chart</div>
</div> </div>
<!-- ② Browse & Filter --> <!-- ② Browse & Filter -->
<div class="card" x-show="tab==='browse'"> <div class="card" x-show="tab==='browse'">
<div class="filters"> <div class="frow">
<div class="field"> <div class="field"><label>TLD</label><input type="text" x-model="f.tld" placeholder="es, com…" style="width:80px" @keydown.enter="search()"></div>
<label>TLD</label> <div class="field"><label>Keyword</label><input type="text" x-model="f.keyword" placeholder="hotel, dental…" style="width:130px" @keydown.enter="search()"></div>
<input type="text" x-model="f.tld" placeholder="es, com…" style="width:90px" @keydown.enter="search()"> <div class="field"><label>Min Score: <b x-text="f.min_score"></b></label><input type="range" x-model="f.min_score" min="0" max="100" step="5"></div>
</div> <div class="field"><label>CMS</label>
<div class="field">
<label>Keyword</label>
<input type="text" x-model="f.keyword" placeholder="hotel, dental…" style="width:130px" @keydown.enter="search()">
</div>
<div class="field">
<label>Min Score: <strong x-text="f.min_score"></strong></label>
<input type="range" x-model="f.min_score" min="0" max="100" step="5">
</div>
<div class="field">
<label>CMS</label>
<select x-model="f.cms" style="width:120px"> <select x-model="f.cms" style="width:120px">
<option value="">Any CMS</option> <option value="">Any CMS</option>
<option>wordpress</option><option>joomla</option><option>drupal</option> <option>wordpress</option><option>joomla</option><option>drupal</option>
@@ -177,184 +211,215 @@ tr:hover td{background:var(--surface2)}
<option>prestashop</option><option>magento</option><option>typo3</option><option>opencart</option> <option>prestashop</option><option>magento</option><option>typo3</option><option>opencart</option>
</select> </select>
</div> </div>
<div class="field"> <div class="field"><label>Per page</label>
<label>Per page</label> <select x-model="f.limit" style="width:75px">
<select x-model="f.limit" style="width:80px"> <option value="50">50</option><option value="100">100</option><option value="250">250</option>
<option value="50">50</option>
<option value="100">100</option>
<option value="250">250</option>
</select> </select>
</div> </div>
<label class="toggle field"><label>Live only</label><input type="checkbox" x-model="f.live_only"></label> <label class="tog"><input type="checkbox" x-model="f.live_only"><strong>Live only</strong></label>
<label class="toggle field"> <label class="tog"><input type="checkbox" x-model="f.alpha_only"><strong>Alpha only</strong> <span>(no hyphens/nums)</span></label>
<label>Alpha only</label> <label class="tog"><input type="checkbox" x-model="f.no_sld"><strong>No SLD</strong> <span>(skip com.es)</span></label>
<input type="checkbox" x-model="f.alpha_only"> <label class="tog"><input type="checkbox" x-model="f.kit_digital_only"><strong style="color:var(--kd)">🏅 Kit Digital only</strong></label>
<span style="font-size:11px;color:var(--muted)">(no hyphens/numbers)</span>
</label>
<label class="toggle field">
<label>No SLD</label>
<input type="checkbox" x-model="f.no_sld">
<span style="font-size:11px;color:var(--muted)">(skip com.es etc)</span>
</label>
</div> </div>
<div style="display:flex;gap:8px;margin-bottom:12px;flex-wrap:wrap;align-items:center"> <div style="display:flex;gap:6px;margin-bottom:10px;flex-wrap:wrap;align-items:center">
<button class="btn btn-primary" @click="search()">Search</button> <button class="btn bp" @click="search()">Search</button>
<button class="btn btn-ghost" @click="resetFilters()">Reset</button> <button class="btn bg" @click="resetFilters()">Reset</button>
<button class="btn btn-success" @click="enqueueSelected()" :disabled="selected.length===0"> <button class="btn bs" @click="enqueueSelected()" :disabled="selected.length===0">
+ Enrich (<span x-text="selected.length"></span>) selected + Enrich (<span x-text="selected.length"></span>)
</button> </button>
<button class="btn btn-ghost btn-sm" @click="selectAll()" x-show="domains.length > 0">Select all on page</button> <button class="btn bai" @click="aiAssessSelected()" :disabled="selected.length===0">
<button class="btn btn-ghost btn-sm" @click="selected=[]" x-show="selected.length > 0">Clear selection</button> 🤖 AI Assess (<span x-text="selected.length"></span>)
<span class="info" style="font-size:12px;color:var(--muted)" x-show="searchTotal > 0"> </button>
<strong x-text="searchTotal.toLocaleString()"></strong> matches <button class="btn bg sm" @click="selectAll()" x-show="domains.length>0">Select page</button>
</span> <button class="btn bg sm" @click="selected=[]" x-show="selected.length>0">Clear</button>
<span class="inf" x-show="searchTotal>0" x-text="searchTotal.toLocaleString()+' matches'"></span>
</div> </div>
<div class="table-wrap"> <div class="tw">
<table> <table>
<thead> <thead>
<tr> <tr>
<th style="width:32px"></th> <th></th><th>Domain</th><th>Score</th><th>KD</th><th>AI</th>
<th>Domain</th> <th>Contact</th><th>CMS</th><th>SSL days</th>
<th>Score</th> <th>Country</th><th>Live</th>
<th>CMS</th>
<th>SSL days</th>
<th>Country</th>
<th>Live</th>
<th>Server</th>
<th>Status</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<template x-if="loading"> <template x-if="loading">
<tr><td colspan="9" style="text-align:center;padding:28px;color:var(--muted)">Searching… (first query without index may take 30-60s)</td></tr> <tr><td colspan="10" style="text-align:center;padding:24px;color:var(--muted)">Searching… (first query may take 30-60s before index is ready)</td></tr>
</template> </template>
<template x-if="!loading && domains.length===0"> <template x-if="!loading && domains.length===0">
<tr><td colspan="9" style="text-align:center;padding:28px;color:var(--muted)">No results — enter a TLD or keyword and click Search</td></tr> <tr><td colspan="10" style="text-align:center;padding:24px;color:var(--muted)">No results — enter a TLD or keyword and click Search</td></tr>
</template> </template>
<template x-for="row in domains" :key="row.domain"> <template x-for="row in domains" :key="row.domain">
<tr> <tr>
<td><input type="checkbox" :value="row.domain" x-model="selected"></td> <td><input type="checkbox" :value="row.domain" x-model="selected"></td>
<td><a :href="'http://'+row.domain" target="_blank" rel="noopener" x-text="row.domain"></a></td> <td><a :href="'http://'+row.domain" target="_blank" rel="noopener" x-text="row.domain"></a></td>
<td> <td>
<template x-if="row.score != null"> <span x-show="row.score!=null" class="score" :style="scoreBg(row.score)" x-text="row.score"></span>
<span class="score" :style="scoreBg(row.score)" x-text="row.score"></span> <span x-show="row.score==null" class="pill pp"></span>
</td>
<!-- Kit Digital badge -->
<td>
<span x-show="row.kit_digital" class="pill pkd" :title="parseSignals(row.kit_digital_signals)">🏅 KD</span>
<span x-show="!row.kit_digital" style="color:var(--border)"></span>
</td>
<!-- AI quality -->
<td>
<span x-show="row.ai_lead_quality"
class="pill" :class="aiPillClass(row.ai_lead_quality)"
style="cursor:pointer"
@click="openModal(row)"
x-text="row.ai_lead_quality"></span>
<span x-show="!row.ai_lead_quality" class="pill ai-none" style="cursor:pointer" @click="openModal(row)" title="Click to assess"></span>
</td>
<!-- Contact info -->
<td>
<div class="contact-chips" x-data="{c: parseContacts(row.contact_info)}">
<template x-for="em in (c.emails||[]).slice(0,1)" :key="em">
<span class="chip email" :title="em"><span x-text="em"></span></span>
</template> </template>
<template x-if="row.score == null"> <template x-for="ph in (c.phones||[]).slice(0,1)" :key="ph">
<span class="pill p-grey"></span> <span class="chip phone" :title="ph">📞 <span x-text="ph"></span></span>
</template> </template>
<template x-for="wa in (c.whatsapp||[]).slice(0,1)" :key="wa">
<span class="chip wa" title="WhatsApp">💬 WA</span>
</template>
<template x-if="(c.social||[]).length>0">
<span class="chip social" :title="(c.social||[]).join(', ')">📲 <span x-text="(c.social||[]).length"></span></span>
</template>
</div>
</td> </td>
<td> <td>
<span x-show="row.cms" class="pill p-cms" x-text="row.cms"></span> <span x-show="row.cms" class="pill pc" x-text="row.cms"></span>
<span x-show="!row.cms" class="pill p-grey"></span> <span x-show="!row.cms" style="color:var(--border)"></span>
</td> </td>
<td x-text="row.ssl_expiry_days??'—'"></td> <td x-text="row.ssl_expiry_days??'—'"></td>
<td x-text="row.ip_country??'—'"></td> <td x-text="row.ip_country??'—'"></td>
<td><span class="pill" :class="row.is_live ? 'p-green' : 'p-grey'" x-text="row.is_live ? 'Yes' : '—'"></span></td> <td><span class="pill" :class="row.is_live?'pg':'pp'" x-text="row.is_live?'Yes':'—'"></span></td>
<td style="font-size:11px;color:var(--muted);max-width:120px;overflow:hidden;white-space:nowrap;text-overflow:ellipsis" x-text="row.server ?? '—'"></td>
<td x-text="row.status_code ?? '—'"></td>
</tr> </tr>
</template> </template>
</tbody> </tbody>
</table> </table>
</div> </div>
<!-- Pagination -->
<div class="pager"> <div class="pager">
<button class="btn btn-ghost btn-sm" @click="goPage(page-1)" :disabled="page<=1 || loading">← Prev</button> <button class="btn bg sm" @click="goPage(page-1)" :disabled="page<=1||loading">← Prev</button>
<span class="info">Page <strong x-text="page"></strong> <span class="inf">Page <b x-text="page"></b>
<template x-if="searchTotal > 0"> <span x-show="searchTotal>0" x-text="' of '+Math.ceil(searchTotal/Number(f.limit))"></span>
<span x-text="' of ' + Math.ceil(searchTotal / Number(f.limit))"></span>
</template>
</span> </span>
<button class="btn btn-ghost btn-sm" @click="goPage(page+1)" :disabled="loading || domains.length < Number(f.limit)">Next →</button> <button class="btn bg sm" @click="goPage(page+1)" :disabled="loading||domains.length<Number(f.limit)">Next →</button>
<span class="info" x-show="searchTotal > 0" x-text="searchTotal.toLocaleString() + ' total results'"></span> <span class="inf" x-show="searchTotal>0" x-text="searchTotal.toLocaleString()+' total'"></span>
</div> </div>
</div> </div>
<!-- ③ Enrichment Queue --> <!-- ③ Enrichment Queue -->
<div class="card" x-show="tab==='enrich'"> <div class="card" x-show="tab==='enrich'">
<div class="eq-grid"> <div class="ct">Enrichment</div>
<div class="eq-stat"><div class="v c-warn" x-text="qst.pending ?? '—'"></div><div class="l">Pending</div></div> <div class="esg">
<div class="eq-stat"><div class="v c-accent" x-text="qst.running ?? '—'"></div><div class="l">Running</div></div> <div class="esb"><div class="ev c3" x-text="qst.pending??'—'"></div><div class="el">Pending</div></div>
<div class="eq-stat"><div class="v c-accent" x-text="qst.done ?? '—'"></div><div class="l">Done</div></div> <div class="esb"><div class="ev c1" x-text="qst.running??'—'"></div><div class="el">Running</div></div>
<div class="eq-stat"><div class="v c-hot" x-text="qst.failed ?? '—'"></div><div class="l">Failed</div></div> <div class="esb"><div class="ev c1" x-text="qst.done??'—'"></div><div class="el">Done</div></div>
<div class="eq-stat"><div class="v c-muted" x-text="qst.eta_seconds ? Math.ceil(qst.eta_seconds/60)+'m' : '—'"></div><div class="l">ETA</div></div> <div class="esb"><div class="ev c2" x-text="qst.failed??'—'"></div><div class="el">Failed</div></div>
<div class="esb"><div class="ev c4" x-text="qst.eta_seconds?Math.ceil(qst.eta_seconds/60)+'m':'—'"></div><div class="el">ETA</div></div>
</div>
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:6px;flex-wrap:wrap;gap:6px">
<span style="font-size:11px;color:var(--muted)" x-text="qLabel()"></span>
<div style="display:flex;gap:6px;flex-wrap:wrap">
<button class="btn bs" x-show="!qst.worker_running" @click="startEnrich()">▶ Start</button>
<button class="btn bw" x-show="qst.worker_running" @click="pauseEnrich()">⏸ Pause</button>
<button class="btn bg" @click="retryFailed()">↺ Retry Failed</button>
<button class="btn bg" @click="runScoring()">★ Re-score All</button>
</div>
</div>
<div class="pw"><div class="pb" :style="'width:'+qPct()+'%'"></div></div>
<div style="font-size:10px;color:var(--muted);margin-top:3px" x-text="qPct().toFixed(1)+'% complete'"></div>
<!-- AI Queue section -->
<div style="margin-top:18px;padding-top:14px;border-top:1px solid var(--border)">
<div class="ct" style="margin-bottom:8px">AI Assessment Queue (Gemini via Replicate)</div>
<div class="esg">
<div class="esb"><div class="ev" style="color:#c084fc" x-text="aiSt.pending??'0'"></div><div class="el">Pending</div></div>
<div class="esb"><div class="ev" style="color:#c084fc" x-text="aiSt.running??'0'"></div><div class="el">Running</div></div>
<div class="esb"><div class="ev c1" x-text="aiSt.done??'0'"></div><div class="el">Done</div></div>
<div class="esb"><div class="ev c2" x-text="aiSt.failed??'0'"></div><div class="el">Failed</div></div>
</div>
<div style="font-size:12px;color:var(--muted);margin-bottom:8px">
Auto-assesses enriched domains via Gemini. Detects Kit Digital confirmation, extracts best contact channel, writes pitch.
</div>
<button class="btn bai" @click="aiAssessAllKD()">🤖 AI Assess all Kit Digital domains</button>
</div> </div>
<div> <div style="margin-top:18px;padding-top:14px;border-top:1px solid var(--border)">
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:6px;flex-wrap:wrap;gap:8px"> <div class="ct" style="margin-bottom:6px">Queue custom domains</div>
<span style="font-size:12px;color:var(--muted)" x-text="qLabel()"></span>
<div style="display:flex;gap:8px">
<button class="btn btn-success" x-show="!qst.worker_running" @click="startEnrich()">▶ Start</button>
<button class="btn btn-warn" x-show="qst.worker_running" @click="pauseEnrich()">⏸ Pause</button>
<button class="btn btn-ghost" @click="retryFailed()">↺ Retry Failed</button>
<button class="btn btn-ghost" @click="runScoring()">★ Score All</button>
</div>
</div>
<div class="prog-wrap"><div class="prog-bar" :style="'width:'+qPct()+'%'"></div></div>
<div style="font-size:11px;color:var(--muted);margin-top:4px" x-text="qPct().toFixed(1)+'% complete'"></div>
</div>
<div style="margin-top:20px">
<div class="card-title">Queue custom domains</div>
<div style="display:flex;gap:8px;align-items:flex-end"> <div style="display:flex;gap:8px;align-items:flex-end">
<div style="flex:1">
<textarea x-model="customDomains" placeholder="example.com&#10;another.es" <textarea x-model="customDomains" placeholder="example.com&#10;another.es"
style="width:100%;background:var(--surface2);border:1px solid var(--border);color:var(--text);border-radius:6px;padding:8px;min-height:80px;font-size:12px;resize:vertical"></textarea> style="flex:1;background:var(--surface2);border:1px solid var(--border);color:var(--text);border-radius:6px;padding:7px;min-height:70px;font-size:12px;resize:vertical"></textarea>
</div> <button class="btn bp" @click="enqueueCustom()">Queue</button>
<button class="btn btn-primary" @click="enqueueCustom()">Queue</button>
</div> </div>
</div> </div>
</div> </div>
<!-- ④ Lead Pipeline --> <!-- ④ Lead Pipeline -->
<div class="card" x-show="tab==='pipeline'"> <div class="card" x-show="tab==='pipeline'">
<div style="display:flex;justify-content:flex-end;margin-bottom:12px"> <div style="display:flex;justify-content:flex-end;margin-bottom:10px;gap:8px">
<button class="btn btn-ghost btn-sm" @click="loadPipeline()">↻ Refresh</button> <button class="btn bg sm" @click="loadPipeline()">↻ Refresh</button>
<button class="btn bg sm" @click="exportTier('all')">⬇ Export All CSV</button>
</div> </div>
<div class="pipeline"> <div class="pipeline">
<div class="pipe-col" style="border-top:3px solid var(--hot)"> <div class="pcol" style="border-top:3px solid var(--danger)">
<h3>🔥 Hot</h3> <h3>🔥 Hot</h3><div style="font-size:11px;color:var(--muted)">Score 80100</div>
<div style="font-size:12px;color:var(--muted)">Score 80100</div> <div class="cnt c2" x-text="pipeline.hot.count.toLocaleString()"></div>
<div class="count c-hot" x-text="pipeline.hot.count.toLocaleString()"></div>
<div class="samples"> <div class="samples">
<template x-for="d in pipeline.hot.samples" :key="d.domain"> <template x-for="d in pipeline.hot.samples" :key="d.domain">
<div class="sample"><a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a><span class="score" :style="scoreBg(d.score)" x-text="d.score"></span></div> <div class="sample">
<div style="display:flex;flex-direction:column;gap:2px;min-width:0">
<a :href="'http://'+d.domain" target="_blank" x-text="d.domain" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap"></a>
<span x-show="d.kit_digital" class="pill pkd" style="font-size:9px;width:fit-content">🏅 Kit Digital</span>
</div>
<span class="score" :style="scoreBg(d.score)" x-text="d.score"></span>
</div>
</template> </template>
</div> </div>
<button class="btn btn-danger btn-sm" style="margin-top:auto" @click="exportTier('hot')">⬇ Export Hot CSV</button> <button class="btn bd sm" style="margin-top:auto" @click="exportTier('hot')">⬇ Export Hot</button>
</div> </div>
<div class="pipe-col" style="border-top:3px solid var(--warm)"> <div class="pcol" style="border-top:3px solid var(--warn)">
<h3>♨️ Warm</h3> <h3>♨️ Warm</h3><div style="font-size:11px;color:var(--muted)">Score 5079</div>
<div style="font-size:12px;color:var(--muted)">Score 5079</div> <div class="cnt c3" x-text="pipeline.warm.count.toLocaleString()"></div>
<div class="count c-warn" x-text="pipeline.warm.count.toLocaleString()"></div>
<div class="samples"> <div class="samples">
<template x-for="d in pipeline.warm.samples" :key="d.domain"> <template x-for="d in pipeline.warm.samples" :key="d.domain">
<div class="sample"><a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a><span class="score" :style="scoreBg(d.score)" x-text="d.score"></span></div> <div class="sample">
<div style="display:flex;flex-direction:column;gap:2px;min-width:0">
<a :href="'http://'+d.domain" target="_blank" x-text="d.domain" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap"></a>
<span x-show="d.kit_digital" class="pill pkd" style="font-size:9px;width:fit-content">🏅 Kit Digital</span>
</div>
<span class="score" :style="scoreBg(d.score)" x-text="d.score"></span>
</div>
</template> </template>
</div> </div>
<button class="btn btn-warn btn-sm" style="margin-top:auto" @click="exportTier('warm')">⬇ Export Warm CSV</button> <button class="btn bw sm" style="margin-top:auto" @click="exportTier('warm')">⬇ Export Warm</button>
</div> </div>
<div class="pipe-col" style="border-top:3px solid var(--cold)"> <div class="pcol" style="border-top:3px solid #6c7aff">
<h3>🧊 Cold</h3> <h3>🧊 Cold</h3><div style="font-size:11px;color:var(--muted)">Score &lt; 50</div>
<div style="font-size:12px;color:var(--muted)">Score &lt; 50</div> <div class="cnt" style="color:#6c7aff" x-text="pipeline.cold.count.toLocaleString()"></div>
<div class="count" style="color:var(--cold)" x-text="pipeline.cold.count.toLocaleString()"></div>
<div class="samples"> <div class="samples">
<template x-for="d in pipeline.cold.samples" :key="d.domain"> <template x-for="d in pipeline.cold.samples" :key="d.domain">
<div class="sample"><a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a><span class="score" :style="scoreBg(d.score)" x-text="d.score"></span></div> <div class="sample">
<a :href="'http://'+d.domain" target="_blank" x-text="d.domain" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;flex:1"></a>
<span class="score" :style="scoreBg(d.score)" x-text="d.score"></span>
</div>
</template> </template>
</div> </div>
<button class="btn btn-ghost btn-sm" style="margin-top:auto" @click="exportTier('cold')">⬇ Export Cold CSV</button> <button class="btn bg sm" style="margin-top:auto" @click="exportTier('cold')">⬇ Export Cold</button>
</div> </div>
</div> </div>
</div> </div>
<!-- ⑤ TLD Chart --> <!-- ⑤ TLD Chart -->
<div class="card" x-show="tab==='chart'"> <div class="card" x-show="tab==='chart'">
<div class="card-title">Top 20 TLDs in dataset</div> <div class="ct">Top 20 TLDs</div>
<div class="chart-wrap"><canvas id="tldChart"></canvas></div> <div class="cw"><canvas id="tldChart"></canvas></div>
</div> </div>
</main> </main>
@@ -363,52 +428,45 @@ tr:hover td{background:var(--surface2)}
function app() { function app() {
return { return {
tab: 'browse', tab: 'browse',
stats: {}, stats: {}, indexSt: {ready:false,building:false,total:0},
indexSt: { ready: false, building: false, total: 0 }, aiSt: {pending:0,running:0,done:0,failed:0,total:0},
domains: [], domains: [], selected: [],
selected: [], loading: false, page: 1, searchTotal: 0,
loading: false, f: {tld:'',keyword:'',min_score:0,cms:'',live_only:false,alpha_only:false,no_sld:false,kit_digital_only:false,limit:'100'},
page: 1, qst: {}, customDomains: '',
searchTotal: 0,
f: { tld:'', keyword:'', min_score:0, cms:'', live_only:false, alpha_only:false, no_sld:false, limit:'100' },
qst: {},
customDomains: '',
pipeline: {hot:{count:0,samples:[]},warm:{count:0,samples:[]},cold:{count:0,samples:[]}}, pipeline: {hot:{count:0,samples:[]},warm:{count:0,samples:[]},cold:{count:0,samples:[]}},
toast: {show:false,msg:'',type:'success'}, toast: {show:false,msg:'',type:'success'},
_chart: null, modal: {open:false,domain:'',data:{}},
_poll: null, _chart: null, _poll: null, _toastTimer: null,
_toastTimer: null,
async init() { async init() {
await this.loadStats(); await Promise.all([this.loadStats(), this.pollIndex(), this.loadAiStatus()]);
this._poll = setInterval(()=>{ this._poll = setInterval(()=>{
this.loadStats(); this.loadStats(); this.pollIndex(); this.loadAiStatus();
this.pollIndex();
if(this.tab==='enrich') this.loadQueue(); if(this.tab==='enrich') this.loadQueue();
if(this.tab==='pipeline') this.loadPipeline(); if(this.tab==='pipeline') this.loadPipeline();
}, 3000); }, 3000);
}, },
async loadStats() { async loadStats() {
try { this.stats = await fetch('/api/stats').then(r=>r.json()); } catch(e){} try {
const s = await fetch('/api/stats').then(r=>r.json());
this.stats = s;
} catch(e){}
}, },
async pollIndex() { async pollIndex() {
try { this.indexSt = await fetch('/api/index/status').then(r=>r.json()); } catch(e){} try { this.indexSt = await fetch('/api/index/status').then(r=>r.json()); } catch(e){}
}, },
async search() { async loadAiStatus() {
this.page = 1; try { this.aiSt = await fetch('/api/ai/status').then(r=>r.json()); } catch(e){}
await this._fetchDomains();
}, },
async goPage(p) { async search() { this.page=1; await this._fetch(); },
if (p < 1) return; async goPage(p) { if(p<1) return; this.page=p; await this._fetch(); },
this.page = p;
await this._fetchDomains();
},
async _fetchDomains() { async _fetch() {
this.loading = true; this.loading = true;
const p = new URLSearchParams({page:this.page, limit:this.f.limit}); const p = new URLSearchParams({page:this.page, limit:this.f.limit});
if(this.f.tld) p.set('tld', this.f.tld.trim()); if(this.f.tld) p.set('tld', this.f.tld.trim());
@@ -419,12 +477,11 @@ function app() {
try { try {
const data = await fetch('/api/domains?'+p).then(r=>r.json()); const data = await fetch('/api/domains?'+p).then(r=>r.json());
this.searchTotal = data.total ?? 0; this.searchTotal = data.total ?? 0;
// Client-side score/cms filter (only applies to already-enriched rows) let rows = data.results;
this.domains = data.results.filter(row => { if(this.f.min_score>0) rows = rows.filter(r=> r.score==null || r.score>=Number(this.f.min_score));
if (this.f.min_score > 0 && row.score != null && row.score < Number(this.f.min_score)) return false; if(this.f.cms) rows = rows.filter(r=> r.cms===this.f.cms);
if (this.f.cms && row.cms !== this.f.cms) return false; if(this.f.kit_digital_only) rows = rows.filter(r=> r.kit_digital);
return true; this.domains = rows;
});
} catch(e) { } catch(e) {
this.domains = []; this.domains = [];
this.notify('Search failed: '+e.message,'error'); this.notify('Search failed: '+e.message,'error');
@@ -433,36 +490,43 @@ function app() {
}, },
selectAll() { this.selected = this.domains.map(d=>d.domain); }, selectAll() { this.selected = this.domains.map(d=>d.domain); },
resetFilters() { this.f = { tld:'', keyword:'', min_score:0, cms:'', live_only:false, alpha_only:false, no_sld:false, limit:'100' }; }, resetFilters() { this.f={tld:'',keyword:'',min_score:0,cms:'',live_only:false,alpha_only:false,no_sld:false,kit_digital_only:false,limit:'100'}; },
async enqueueSelected() { async enqueueSelected() {
if(!this.selected.length) return; if(!this.selected.length) return;
try { try {
const r = await fetch('/api/enrich/batch', { const r = await fetch('/api/enrich/batch',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({domains:this.selected})});
method:'POST', headers:{'Content-Type':'application/json'}, const d = await r.json();
body: JSON.stringify({ domains: this.selected }), r.ok ? this.notify(`Queued ${d.queued} for enrichment`,'success') : this.notify('Error: '+(d.error||r.statusText),'error');
});
const data = await r.json();
if (r.ok) {
this.notify(`Queued ${data.queued} domain(s) for enrichment`, 'success');
this.selected = []; this.selected = [];
} else {
this.notify('Error: ' + (data.error || r.statusText), 'error');
}
} catch(e) { this.notify('Request failed: '+e.message,'error'); } } catch(e) { this.notify('Request failed: '+e.message,'error'); }
}, },
async aiAssessSelected() {
if(!this.selected.length) return;
const r = await fetch('/api/ai/assess/batch',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({domains:this.selected})});
const d = await r.json();
r.ok ? this.notify(`Queued ${d.queued} for AI assessment`,'info') : this.notify('Error: '+d.error,'error');
this.selected = [];
},
async aiAssessAllKD() {
// Get all Kit Digital domains and queue them
const r = await fetch('/api/enriched?kit_digital=true&limit=500').then(r=>r.json());
const domains = r.results.map(d=>d.domain);
if(!domains.length) { this.notify('No Kit Digital domains enriched yet','info'); return; }
const r2 = await fetch('/api/ai/assess/batch',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({domains})});
const d2 = await r2.json();
this.notify(`Queued ${d2.queued} Kit Digital domains for AI assessment`,'info');
},
async enqueueCustom() { async enqueueCustom() {
const domains = this.customDomains.split('\n').map(d=>d.trim()).filter(Boolean); const domains = this.customDomains.split('\n').map(d=>d.trim()).filter(Boolean);
if(!domains.length) return; if(!domains.length) return;
const r = await fetch('/api/enrich/batch', { const r = await fetch('/api/enrich/batch',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({domains})});
method:'POST', headers:{'Content-Type':'application/json'}, const d = await r.json();
body: JSON.stringify({ domains }), r.ok ? this.notify(`Queued ${d.queued} domains`,'success') : this.notify('Error: '+d.error,'error');
}); this.customDomains = ''; await this.loadQueue();
const data = await r.json();
if (r.ok) { this.notify(`Queued ${data.queued} domains`, 'success'); this.customDomains = ''; }
else { this.notify('Error: ' + data.error, 'error'); }
await this.loadQueue();
}, },
async loadQueue() { async loadQueue() {
@@ -471,33 +535,54 @@ function app() {
async startEnrich() { await fetch('/api/enrich/resume',{method:'POST'}); this.notify('Worker started','success'); await this.loadQueue(); }, async startEnrich() { await fetch('/api/enrich/resume',{method:'POST'}); this.notify('Worker started','success'); await this.loadQueue(); },
async pauseEnrich() { await fetch('/api/enrich/pause',{method:'POST'}); this.notify('Worker paused','success'); await this.loadQueue(); }, async pauseEnrich() { await fetch('/api/enrich/pause',{method:'POST'}); this.notify('Worker paused','success'); await this.loadQueue(); },
async retryFailed() { await fetch('/api/enrich/retry',{method:'POST'}); this.notify('Retrying failed jobs','success'); await this.loadQueue(); }, async retryFailed() { await fetch('/api/enrich/retry',{method:'POST'}); this.notify('Retrying failed','success'); await this.loadQueue(); },
async runScoring() { const r = await fetch('/api/score/run',{method:'POST'}); const d=await r.json(); this.notify(`Scored ${d.scored} domains`,'success'); }, async runScoring() { const r = await fetch('/api/score/run',{method:'POST'}); const d=await r.json(); this.notify(`Scored ${d.scored} domains`,'success'); },
qPct() { const q=this.qst; if(!q||!q.total) return 0; return (q.done/q.total)*100; }, qPct() { const q=this.qst; return (!q||!q.total)?0:(q.done/q.total*100); },
qLabel() { const q=this.qst; return `${q.done??0} done · ${q.pending??0} pending · ${q.running??0} running · ${q.failed??0} failed`; }, qLabel() { const q=this.qst; return `${q.done??0} done · ${q.pending??0} pending · ${q.running??0} running · ${q.failed??0} failed`; },
async loadPipeline() { async loadPipeline() {
const [hot, warm, cold, hotC, warmC, coldC] = await Promise.all([ const [hot,warm,cold] = await Promise.all([
fetch('/api/enriched?min_score=80&limit=5').then(r=>r.json()), fetch('/api/enriched?min_score=80&limit=5').then(r=>r.json()),
fetch('/api/enriched?min_score=50&limit=6').then(r=>r.json()), fetch('/api/enriched?min_score=50&limit=5').then(r=>r.json()),
fetch('/api/enriched?min_score=0&limit=6').then(r=>r.json()), fetch('/api/enriched?min_score=0&limit=5').then(r=>r.json()),
fetch('/api/enriched?min_score=80&limit=1000').then(r=>r.json()),
fetch('/api/enriched?min_score=50&limit=1000').then(r=>r.json()),
fetch('/api/enriched?min_score=0&limit=1000').then(r=>r.json()),
]); ]);
this.pipeline.hot = { count: hotC.total ?? hot.results.length, samples: hot.results.slice(0,5) }; this.pipeline.hot = {count:hot.total??0, samples:hot.results.slice(0,5)};
this.pipeline.warm = { count: (warmC.total ?? warm.results.length) - (hotC.total ?? 0), samples: warm.results.filter(d=>d.score<80).slice(0,5) }; this.pipeline.warm = {count:Math.max(0,(warm.total??0)-(hot.total??0)), samples:warm.results.filter(d=>d.score<80).slice(0,5)};
this.pipeline.cold = { count: (coldC.total ?? cold.results.length) - (warmC.total ?? 0), samples: cold.results.filter(d=>d.score<50).slice(0,5) }; this.pipeline.cold = {count:Math.max(0,(cold.total??0)-(warm.total??0)), samples:cold.results.filter(d=>d.score<50).slice(0,5)};
}, },
exportTier(tier) { window.location = `/api/export?tier=${tier}`; }, exportTier(tier) { window.location = `/api/export?tier=${tier}`; },
openModal(row) {
this.modal.domain = row.domain;
try { this.modal.data = row.ai_assessment ? JSON.parse(row.ai_assessment) : {}; }
catch(e) { this.modal.data = {}; }
this.modal.open = true;
},
scoreBg(s) { scoreBg(s) {
if (s == null) return 'background:#333;color:#aaa'; if(s==null) return 'background:#333;color:#888';
if (s >= 80) return 'background:#ff4f6d33;color:#ff4f6d'; if(s>=80) return 'background:#ff4f6d22;color:#ff4f6d';
if (s >= 50) return 'background:#ffb34733;color:#ffb347'; if(s>=50) return 'background:#ffb34722;color:#ffb347';
return 'background:#6c7aff33;color:#6c7aff'; return 'background:#6c7aff22;color:#6c7aff';
},
aiPillClass(q) {
if(!q) return 'ai-none';
if(q==='HOT') return 'ai-hot';
if(q==='WARM') return 'ai-warm';
return 'ai-cold';
},
parseContacts(raw) {
if(!raw) return {};
try { return JSON.parse(raw); } catch(e) { return {}; }
},
parseSignals(raw) {
if(!raw) return 'No signals';
try { return JSON.parse(raw).join('\n'); } catch(e) { return raw; }
}, },
notify(msg, type='success') { notify(msg, type='success') {
@@ -516,17 +601,12 @@ function app() {
type:'bar', type:'bar',
data:{ data:{
labels:tlds.map(t=>'.'+t.tld), labels:tlds.map(t=>'.'+t.tld),
datasets:[{ label:'Domains', data:tlds.map(t=>t.count), datasets:[{label:'Domains',data:tlds.map(t=>t.count),backgroundColor:'rgba(108,99,255,.7)',borderColor:'rgba(108,99,255,1)',borderWidth:1,borderRadius:4}]
backgroundColor:'rgba(108,99,255,0.7)', borderColor:'rgba(108,99,255,1)',
borderWidth:1, borderRadius:4 }]
}, },
options:{ options:{
responsive:true,maintainAspectRatio:false, responsive:true,maintainAspectRatio:false,
plugins:{legend:{display:false}}, plugins:{legend:{display:false}},
scales:{ scales:{x:{ticks:{color:'#8891b0'},grid:{color:'#2e3250'}},y:{ticks:{color:'#8891b0'},grid:{color:'#2e3250'}}}
x:{ticks:{color:'#8891b0'},grid:{color:'#2e3250'}},
y:{ticks:{color:'#8891b0'},grid:{color:'#2e3250'}},
}
} }
}); });
}, },

View File

@@ -13,4 +13,6 @@ services:
- SCORE_THRESHOLD=60 - SCORE_THRESHOLD=60
- TARGET_TLDS=es,com,net - TARGET_TLDS=es,com,net
- TARGET_COUNTRIES=ES,GB,DE,FR,RO,PT,AD,IT - TARGET_COUNTRIES=ES,GB,DE,FR,RO,PT,AD,IT
- REPLICATE_API_TOKEN=r8_6kV2NWMQyPVB9JILHJprrXJJh4vWazA22Osyj
- AI_CONCURRENCY=3
restart: unless-stopped restart: unless-stopped