feat: persistent DuckDB index, new filters, pagination fix, enrich UX
- Build /data/domains.duckdb on first run (tld+parts columns + ART index) → TLD filter goes from ~60s full scan to <100ms index lookup → System still works (slower) while index builds in background - New /api/domains params: alpha_only, no_sld, keyword → alpha_only: domains with only letters (no hyphens/numbers) → no_sld: parts=2, excludes com.es / net.es patterns → keyword: LIKE '%term%' niche search - /api/domains and /api/enriched now return total count for pagination - Pagination: shows total matches, page X of Y, Next disabled at last page - Enrich button: toast notifications instead of alert(), error handling - Select all on page button, clear selection button - Stats/TLD breakdown cached after first load (no repeat full scan) - Header shows index build status (building → ready) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
330
app/db.py
330
app/db.py
@@ -1,10 +1,15 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import aiosqlite
|
||||
import duckdb
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DATA_DIR = Path(os.getenv("DATA_DIR", "/data"))
|
||||
PARQUET_PATH = DATA_DIR / "domains.parquet"
|
||||
DUCKDB_PATH = DATA_DIR / "domains.duckdb"
|
||||
SQLITE_PATH = DATA_DIR / "enrichment.db"
|
||||
|
||||
SCHEMA = """
|
||||
@@ -23,7 +28,6 @@ CREATE TABLE IF NOT EXISTS enriched_domains (
|
||||
error TEXT,
|
||||
score INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_queue (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
domain TEXT UNIQUE NOT NULL,
|
||||
@@ -33,7 +37,6 @@ CREATE TABLE IF NOT EXISTS job_queue (
|
||||
completed_at TEXT,
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scores (
|
||||
domain TEXT PRIMARY KEY,
|
||||
score INTEGER NOT NULL,
|
||||
@@ -41,6 +44,15 @@ CREATE TABLE IF NOT EXISTS scores (
|
||||
);
|
||||
"""
|
||||
|
||||
# Index build state
|
||||
_index_ready = False
|
||||
_index_building = False
|
||||
_index_total = 0
|
||||
|
||||
# Cached stats (TLD breakdown is expensive — compute once)
|
||||
_tld_cache: list = []
|
||||
_total_cache: int = 0
|
||||
|
||||
|
||||
async def init_db():
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
@@ -48,142 +60,219 @@ async def init_db():
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def get_db():
|
||||
return await aiosqlite.connect(SQLITE_PATH)
|
||||
# ── DuckDB persistent index ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def duckdb_query(sql: str, params=None):
|
||||
conn = duckdb.connect(database=":memory:", read_only=False)
|
||||
conn.execute(f"SET threads=4")
|
||||
if params:
|
||||
result = conn.execute(sql, params).fetchall()
|
||||
else:
|
||||
result = conn.execute(sql).fetchall()
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
|
||||
def duckdb_query_df(sql: str, params=None):
|
||||
conn = duckdb.connect(database=":memory:", read_only=False)
|
||||
def _build_index_sync():
|
||||
global _index_ready, _index_building, _index_total
|
||||
_index_building = True
|
||||
try:
|
||||
conn = duckdb.connect(str(DUCKDB_PATH))
|
||||
conn.execute("SET threads=4")
|
||||
if params:
|
||||
result = conn.execute(sql, params).df()
|
||||
else:
|
||||
result = conn.execute(sql).df()
|
||||
conn.execute("SET memory_limit='2GB'")
|
||||
|
||||
# Check if already built
|
||||
try:
|
||||
n = conn.execute("SELECT COUNT(*) FROM domains").fetchone()[0]
|
||||
if n > 0:
|
||||
_index_total = n
|
||||
_index_ready = True
|
||||
_index_building = False
|
||||
logger.info("DuckDB index already ready (%d rows)", n)
|
||||
conn.close()
|
||||
return result
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info("Building DuckDB index from parquet (one-time ~2-3 min)...")
|
||||
conn.execute("""
|
||||
CREATE OR REPLACE TABLE domains AS
|
||||
SELECT
|
||||
domain,
|
||||
lower(regexp_extract(domain, '\\.([^.]+)$', 1)) AS tld,
|
||||
len(string_split(domain, '.')) AS parts
|
||||
FROM read_parquet(?)
|
||||
""", [str(PARQUET_PATH)])
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_tld ON domains(tld)")
|
||||
_index_total = conn.execute("SELECT COUNT(*) FROM domains").fetchone()[0]
|
||||
conn.close()
|
||||
_index_ready = True
|
||||
logger.info("DuckDB index built: %d rows", _index_total)
|
||||
except Exception as e:
|
||||
logger.error("DuckDB index build failed: %s", e)
|
||||
finally:
|
||||
_index_building = False
|
||||
|
||||
|
||||
async def build_duckdb_index():
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, _build_index_sync)
|
||||
|
||||
|
||||
def index_status() -> dict:
|
||||
return {
|
||||
"ready": _index_ready,
|
||||
"building": _index_building,
|
||||
"total": _index_total,
|
||||
}
|
||||
|
||||
|
||||
# ── Domain queries ───────────────────────────────────────────────────────────
|
||||
|
||||
def _domains_sync(tld, page, limit, alpha_only, no_sld, keyword):
|
||||
conditions = []
|
||||
params_count = []
|
||||
params_data = []
|
||||
|
||||
if _index_ready:
|
||||
source = "domains"
|
||||
|
||||
def _add(clause, val=None):
|
||||
conditions.append(clause)
|
||||
if val is not None:
|
||||
params_count.append(val)
|
||||
params_data.append(val)
|
||||
else:
|
||||
source = f"read_parquet('{PARQUET_PATH}')"
|
||||
|
||||
def _add(clause, val=None):
|
||||
conditions.append(clause)
|
||||
if val is not None:
|
||||
params_count.append(val)
|
||||
params_data.append(val)
|
||||
|
||||
if tld:
|
||||
if _index_ready:
|
||||
_add("tld = ?", tld.lower().lstrip("."))
|
||||
else:
|
||||
_add("lower(regexp_extract(domain, '\\.([^.]+)$', 1)) = ?", tld.lower().lstrip("."))
|
||||
|
||||
if no_sld:
|
||||
if _index_ready:
|
||||
_add("parts = 2")
|
||||
else:
|
||||
_add("len(string_split(domain, '.')) = 2")
|
||||
|
||||
if alpha_only:
|
||||
_add("NOT regexp_matches(domain, '[^a-zA-Z.]')")
|
||||
|
||||
if keyword:
|
||||
_add("domain LIKE ?", f"%{keyword.lower()}%")
|
||||
|
||||
where = ("WHERE " + " AND ".join(conditions)) if conditions else ""
|
||||
offset = (page - 1) * limit
|
||||
|
||||
if _index_ready:
|
||||
conn = duckdb.connect(str(DUCKDB_PATH), read_only=True)
|
||||
else:
|
||||
conn = duckdb.connect(":memory:")
|
||||
conn.execute("SET threads=4")
|
||||
|
||||
total = conn.execute(f"SELECT COUNT(*) FROM {source} {where}", params_count).fetchone()[0]
|
||||
rows = conn.execute(
|
||||
f"SELECT domain FROM {source} {where} LIMIT {limit} OFFSET {offset}", params_data
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return total, [r[0] for r in rows]
|
||||
|
||||
|
||||
async def get_domains(tld=None, page=1, limit=100, alpha_only=False, no_sld=False, keyword=None, live_only=False):
|
||||
loop = asyncio.get_event_loop()
|
||||
total, domain_list = await loop.run_in_executor(
|
||||
None, _domains_sync, tld, page, limit, alpha_only, no_sld, keyword
|
||||
)
|
||||
|
||||
if not domain_list:
|
||||
return total, []
|
||||
|
||||
placeholders = ",".join("?" * len(domain_list))
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
async with db.execute(
|
||||
f"SELECT * FROM enriched_domains WHERE domain IN ({placeholders})",
|
||||
domain_list,
|
||||
) as cur:
|
||||
enriched_map = {r["domain"]: dict(r) async for r in cur}
|
||||
|
||||
results = []
|
||||
for d in domain_list:
|
||||
row = enriched_map.get(d, {"domain": d})
|
||||
if live_only and not row.get("is_live"):
|
||||
continue
|
||||
results.append(row)
|
||||
|
||||
return total, results
|
||||
|
||||
|
||||
# ── Stats ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _tld_stats_sync() -> tuple[int, list]:
|
||||
if _index_ready:
|
||||
conn = duckdb.connect(str(DUCKDB_PATH), read_only=True)
|
||||
total = conn.execute("SELECT COUNT(*) FROM domains").fetchone()[0]
|
||||
rows = conn.execute("""
|
||||
SELECT tld, COUNT(*) AS cnt FROM domains
|
||||
WHERE tld != ''
|
||||
GROUP BY tld ORDER BY cnt DESC LIMIT 20
|
||||
""").fetchall()
|
||||
conn.close()
|
||||
else:
|
||||
p = str(PARQUET_PATH)
|
||||
conn = duckdb.connect(":memory:")
|
||||
conn.execute("SET threads=4")
|
||||
total = conn.execute(f"SELECT COUNT(*) FROM read_parquet('{p}')").fetchone()[0]
|
||||
rows = conn.execute(f"""
|
||||
SELECT lower(regexp_extract(domain, '\\.([^.]+)$', 1)) AS tld, COUNT(*) AS cnt
|
||||
FROM read_parquet('{p}')
|
||||
GROUP BY tld ORDER BY cnt DESC LIMIT 20
|
||||
""").fetchall()
|
||||
conn.close()
|
||||
return total, [{"tld": r[0], "count": r[1]} for r in rows]
|
||||
|
||||
|
||||
async def get_stats():
|
||||
parquet = str(PARQUET_PATH)
|
||||
global _tld_cache, _total_cache
|
||||
|
||||
# Total count + TLD breakdown via DuckDB pushdown
|
||||
total = duckdb_query(f"SELECT COUNT(*) FROM read_parquet('{parquet}')")[0][0]
|
||||
|
||||
tld_rows = duckdb_query(f"""
|
||||
SELECT
|
||||
regexp_extract(domain, '\\.([a-zA-Z0-9]+)$', 1) AS tld,
|
||||
COUNT(*) AS cnt
|
||||
FROM read_parquet('{parquet}')
|
||||
GROUP BY tld
|
||||
ORDER BY cnt DESC
|
||||
LIMIT 20
|
||||
""")
|
||||
# Compute TLD breakdown once and cache it
|
||||
if not _tld_cache:
|
||||
loop = asyncio.get_event_loop()
|
||||
_total_cache, _tld_cache = await loop.run_in_executor(None, _tld_stats_sync)
|
||||
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
async with db.execute("SELECT COUNT(*) FROM enriched_domains") as cur:
|
||||
enriched = (await cur.fetchone())[0]
|
||||
threshold = int(os.getenv("SCORE_THRESHOLD", "60"))
|
||||
async with db.execute(
|
||||
"SELECT COUNT(*) FROM enriched_domains WHERE score >= ?", (threshold,)
|
||||
) as cur:
|
||||
async with db.execute("SELECT COUNT(*) FROM enriched_domains WHERE score >= ?", (threshold,)) as cur:
|
||||
hot_leads = (await cur.fetchone())[0]
|
||||
async with db.execute(
|
||||
"SELECT COUNT(*) FROM job_queue WHERE status='pending'"
|
||||
) as cur:
|
||||
queue_pending = (await cur.fetchone())[0]
|
||||
async with db.execute(
|
||||
"SELECT COUNT(*) FROM job_queue WHERE status='running'"
|
||||
) as cur:
|
||||
queue_running = (await cur.fetchone())[0]
|
||||
async with db.execute(
|
||||
"SELECT COUNT(*) FROM job_queue WHERE status='done'"
|
||||
) as cur:
|
||||
queue_done = (await cur.fetchone())[0]
|
||||
async with db.execute(
|
||||
"SELECT COUNT(*) FROM job_queue WHERE status='failed'"
|
||||
) as cur:
|
||||
queue_failed = (await cur.fetchone())[0]
|
||||
async with db.execute("SELECT status, COUNT(*) FROM job_queue GROUP BY status") as cur:
|
||||
q = {r[0]: r[1] async for r in cur}
|
||||
|
||||
return {
|
||||
"total_domains": total,
|
||||
"total_domains": _total_cache,
|
||||
"enriched": enriched,
|
||||
"hot_leads": hot_leads,
|
||||
"tld_breakdown": [{"tld": r[0], "count": r[1]} for r in tld_rows],
|
||||
"tld_breakdown": _tld_cache,
|
||||
"index_status": index_status(),
|
||||
"queue": {
|
||||
"pending": queue_pending,
|
||||
"running": queue_running,
|
||||
"done": queue_done,
|
||||
"failed": queue_failed,
|
||||
"pending": q.get("pending", 0),
|
||||
"running": q.get("running", 0),
|
||||
"done": q.get("done", 0),
|
||||
"failed": q.get("failed", 0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def get_domains(tld=None, page=1, limit=100, live_only=False):
|
||||
parquet = str(PARQUET_PATH)
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
if tld:
|
||||
conditions.append(f"regexp_extract(domain, '\\.([a-zA-Z0-9]+)$', 1) = '{tld}'")
|
||||
if live_only:
|
||||
# Join with enriched_domains to check is_live
|
||||
pass
|
||||
|
||||
where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
offset = (page - 1) * limit
|
||||
|
||||
sql = f"""
|
||||
SELECT domain
|
||||
FROM read_parquet('{parquet}')
|
||||
{where}
|
||||
LIMIT {limit} OFFSET {offset}
|
||||
"""
|
||||
rows = duckdb_query(sql)
|
||||
domains = [r[0] for r in rows]
|
||||
|
||||
# Merge enrichment data from SQLite
|
||||
if domains:
|
||||
placeholders = ",".join("?" * len(domains))
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
async with db.execute(
|
||||
f"SELECT * FROM enriched_domains WHERE domain IN ({placeholders})",
|
||||
domains,
|
||||
) as cur:
|
||||
enriched = {r["domain"]: dict(r) async for r in cur}
|
||||
|
||||
result = []
|
||||
for d in domains:
|
||||
if d in enriched:
|
||||
result.append(enriched[d])
|
||||
else:
|
||||
result.append({"domain": d})
|
||||
return result
|
||||
return []
|
||||
|
||||
# ── Enrichment helpers ───────────────────────────────────────────────────────
|
||||
|
||||
async def get_enriched(min_score=0, cms=None, country=None, page=1, limit=100):
|
||||
offset = (page - 1) * limit
|
||||
conditions = ["score >= ?"]
|
||||
params = [min_score]
|
||||
params: list = [min_score]
|
||||
if cms:
|
||||
conditions.append("cms = ?")
|
||||
params.append(cms)
|
||||
if country:
|
||||
conditions.append("ip_country = ?")
|
||||
params.append(country)
|
||||
|
||||
where = "WHERE " + " AND ".join(conditions)
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
@@ -192,7 +281,11 @@ async def get_enriched(min_score=0, cms=None, country=None, page=1, limit=100):
|
||||
params + [limit, offset],
|
||||
) as cur:
|
||||
rows = [dict(r) async for r in cur]
|
||||
return rows
|
||||
async with db.execute(
|
||||
f"SELECT COUNT(*) FROM enriched_domains {where}", params
|
||||
) as cur:
|
||||
total = (await cur.fetchone())[0]
|
||||
return total, rows
|
||||
|
||||
|
||||
async def queue_domains(domains: list[str]):
|
||||
@@ -206,26 +299,13 @@ async def queue_domains(domains: list[str]):
|
||||
|
||||
async def get_queue_status():
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
async with db.execute(
|
||||
"SELECT status, COUNT(*) FROM job_queue GROUP BY status"
|
||||
) as cur:
|
||||
async with db.execute("SELECT status, COUNT(*) FROM job_queue GROUP BY status") as cur:
|
||||
rows = {r[0]: r[1] async for r in cur}
|
||||
total = sum(rows.values())
|
||||
done = rows.get("done", 0)
|
||||
pending = rows.get("pending", 0)
|
||||
running = rows.get("running", 0)
|
||||
done = rows.get("done", 0)
|
||||
failed = rows.get("failed", 0)
|
||||
|
||||
eta_seconds = None
|
||||
if running > 0 or pending > 0:
|
||||
total = sum(rows.values())
|
||||
rate = int(os.getenv("CONCURRENCY_LIMIT", "50"))
|
||||
eta_seconds = (pending + running) / max(rate / 10, 1)
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"pending": pending,
|
||||
"running": running,
|
||||
"done": done,
|
||||
"failed": failed,
|
||||
"eta_seconds": eta_seconds,
|
||||
}
|
||||
eta_seconds = (pending + running) / max(rate / 10, 1) if (pending + running) > 0 else None
|
||||
return {"total": total, "pending": pending, "running": running, "done": done, "failed": failed, "eta_seconds": eta_seconds}
|
||||
|
||||
72
app/main.py
72
app/main.py
@@ -1,12 +1,10 @@
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import httpx
|
||||
import duckdb
|
||||
import aiosqlite
|
||||
from fastapi import FastAPI, Query
|
||||
from fastapi.responses import StreamingResponse, JSONResponse
|
||||
@@ -18,7 +16,7 @@ load_dotenv()
|
||||
from app.db import (
|
||||
DATA_DIR, PARQUET_PATH, SQLITE_PATH,
|
||||
init_db, get_stats, get_domains, get_enriched,
|
||||
queue_domains, get_queue_status,
|
||||
queue_domains, get_queue_status, build_duckdb_index, index_status,
|
||||
)
|
||||
from app.enricher import start_worker, pause_worker, resume_worker, is_running
|
||||
from app.scorer import run_scoring
|
||||
@@ -37,16 +35,13 @@ async def download_parquet():
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = PARQUET_PATH.with_suffix(".tmp")
|
||||
|
||||
# Resumable download via Range header
|
||||
downloaded = tmp_path.stat().st_size if tmp_path.exists() else 0
|
||||
headers = {"Range": f"bytes={downloaded}-"} if downloaded > 0 else {}
|
||||
|
||||
logger.info("Downloading parquet from %s (offset=%d)...", PARQUET_URL, downloaded)
|
||||
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=None) as client:
|
||||
async with client.stream("GET", PARQUET_URL, headers=headers) as resp:
|
||||
if resp.status_code == 416:
|
||||
# Already fully downloaded
|
||||
tmp_path.rename(PARQUET_PATH)
|
||||
return
|
||||
resp.raise_for_status()
|
||||
@@ -58,41 +53,54 @@ async def download_parquet():
|
||||
f.write(chunk)
|
||||
received += len(chunk)
|
||||
if total:
|
||||
pct = received / total * 100
|
||||
logger.info("Download progress: %.1f%% (%d/%d bytes)", pct, received, total)
|
||||
logger.info("Download: %.1f%% (%d/%d)", received / total * 100, received, total)
|
||||
|
||||
tmp_path.rename(PARQUET_PATH)
|
||||
logger.info("Parquet download complete: %s", PARQUET_PATH)
|
||||
logger.info("Parquet download complete")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
await download_parquet()
|
||||
await init_db()
|
||||
# Build DuckDB index in background — queries still work (slower) while building
|
||||
asyncio.create_task(build_duckdb_index())
|
||||
start_worker()
|
||||
logger.info("DomGod dashboard ready on port 6677")
|
||||
logger.info("DomGod ready on port 6677")
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title="DomGod", lifespan=lifespan)
|
||||
|
||||
|
||||
# ── API routes ──────────────────────────────────────────────────────────────
|
||||
# ── API ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/stats")
|
||||
async def stats():
|
||||
return await get_stats()
|
||||
|
||||
|
||||
@app.get("/api/index/status")
|
||||
async def get_index_status():
|
||||
return index_status()
|
||||
|
||||
|
||||
@app.get("/api/domains")
|
||||
async def domains(
|
||||
tld: str = Query(None),
|
||||
page: int = Query(1, ge=1),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
live_only: bool = Query(False),
|
||||
alpha_only: bool = Query(False),
|
||||
no_sld: bool = Query(False),
|
||||
keyword: str = Query(None),
|
||||
):
|
||||
rows = await get_domains(tld=tld, page=page, limit=limit, live_only=live_only)
|
||||
return {"page": page, "limit": limit, "results": rows}
|
||||
total, rows = await get_domains(
|
||||
tld=tld, page=page, limit=limit,
|
||||
alpha_only=alpha_only, no_sld=no_sld,
|
||||
keyword=keyword, live_only=live_only,
|
||||
)
|
||||
return {"page": page, "limit": limit, "total": total, "results": rows}
|
||||
|
||||
|
||||
@app.post("/api/enrich/batch")
|
||||
@@ -118,7 +126,7 @@ async def enrich_retry():
|
||||
await db.execute("UPDATE job_queue SET status='pending', error=NULL WHERE status='failed'")
|
||||
await db.commit()
|
||||
resume_worker()
|
||||
return {"status": "retrying failed jobs"}
|
||||
return {"status": "retrying"}
|
||||
|
||||
|
||||
@app.post("/api/enrich/pause")
|
||||
@@ -141,8 +149,8 @@ async def enriched(
|
||||
page: int = Query(1, ge=1),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
):
|
||||
rows = await get_enriched(min_score=min_score, cms=cms, country=country, page=page, limit=limit)
|
||||
return {"page": page, "limit": limit, "results": rows}
|
||||
total, rows = await get_enriched(min_score=min_score, cms=cms, country=country, page=page, limit=limit)
|
||||
return {"page": page, "limit": limit, "total": total, "results": rows}
|
||||
|
||||
|
||||
@app.get("/api/export")
|
||||
@@ -157,46 +165,42 @@ async def export_csv(
|
||||
elif tier == "warm":
|
||||
min_score = 50
|
||||
|
||||
max_score = 79 if tier == "warm" else 100
|
||||
|
||||
async def generate():
|
||||
yield "domain,score,cms,ssl_expiry_days,ip_country,is_live,status_code,has_mx,server,page_title,enriched_at\n"
|
||||
page = 1
|
||||
p = 1
|
||||
while True:
|
||||
rows = await get_enriched(min_score=min_score, cms=cms, country=country, page=page, limit=500)
|
||||
_, rows = await get_enriched(min_score=min_score, cms=cms, country=country, page=p, limit=500)
|
||||
if not rows:
|
||||
break
|
||||
for r in rows:
|
||||
# Apply warm tier upper bound
|
||||
if tier == "warm" and r.get("score", 0) >= 80:
|
||||
if r.get("score", 0) > max_score:
|
||||
continue
|
||||
line = ",".join(
|
||||
f'"{str(r.get(col) or "").replace(chr(34), chr(39))}"'
|
||||
for col in [
|
||||
"domain", "score", "cms", "ssl_expiry_days", "ip_country",
|
||||
"is_live", "status_code", "has_mx", "server", "page_title", "enriched_at"
|
||||
]
|
||||
for col in ["domain", "score", "cms", "ssl_expiry_days", "ip_country",
|
||||
"is_live", "status_code", "has_mx", "server", "page_title", "enriched_at"]
|
||||
)
|
||||
yield line + "\n"
|
||||
page += 1
|
||||
p += 1
|
||||
|
||||
filename = f"domgod_leads_score{min_score}{'_' + tier if tier else ''}.csv"
|
||||
fname = f"domgod_{tier or 'export'}_score{min_score}.csv"
|
||||
return StreamingResponse(
|
||||
generate(),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
generate(), media_type="text/csv",
|
||||
headers={"Content-Disposition": f'attachment; filename="{fname}"'},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/score/run")
|
||||
async def score_run():
|
||||
result = await run_scoring()
|
||||
return result
|
||||
return await run_scoring()
|
||||
|
||||
|
||||
# ── Static UI ───────────────────────────────────────────────────────────────
|
||||
# ── Static UI ────────────────────────────────────────────────────────────────
|
||||
static_dir = Path(__file__).parent / "static"
|
||||
app.mount("/", StaticFiles(directory=str(static_dir), html=True), name="static")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("app.main:app", host="0.0.0.0", port=6677, log_level="info")
|
||||
|
||||
@@ -3,592 +3,529 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>DomGod — Domain Intelligence Dashboard</title>
|
||||
<title>DomGod — Domain Intelligence</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js" defer></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0f1117;
|
||||
--surface: #1a1d27;
|
||||
--surface2: #222638;
|
||||
--border: #2e3250;
|
||||
--accent: #6c63ff;
|
||||
--accent2: #00d4aa;
|
||||
--danger: #ff4f6d;
|
||||
--warn: #ffb347;
|
||||
--text: #e8eaf0;
|
||||
--muted: #8891b0;
|
||||
--hot: #ff4f6d;
|
||||
--warm: #ffb347;
|
||||
--cold: #6c7aff;
|
||||
--radius: 10px;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body { background: var(--bg); color: var(--text); font-family: 'Segoe UI', system-ui, sans-serif; font-size: 14px; }
|
||||
a { color: var(--accent2); text-decoration: none; }
|
||||
:root {
|
||||
--bg:#0f1117; --surface:#1a1d27; --surface2:#222638; --border:#2e3250;
|
||||
--accent:#6c63ff; --accent2:#00d4aa; --danger:#ff4f6d; --warn:#ffb347;
|
||||
--text:#e8eaf0; --muted:#8891b0; --hot:#ff4f6d; --warm:#ffb347; --cold:#6c7aff;
|
||||
--r:8px;
|
||||
}
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,sans-serif;font-size:14px}
|
||||
a{color:var(--accent2);text-decoration:none}
|
||||
|
||||
/* Layout */
|
||||
.shell { display: flex; flex-direction: column; min-height: 100vh; }
|
||||
header { background: var(--surface); border-bottom: 1px solid var(--border); padding: 14px 24px; display: flex; align-items: center; gap: 12px; position: sticky; top: 0; z-index: 100; }
|
||||
header h1 { font-size: 20px; font-weight: 700; letter-spacing: -0.5px; }
|
||||
header h1 span { color: var(--accent); }
|
||||
.badge { background: var(--accent); color: #fff; font-size: 11px; padding: 2px 8px; border-radius: 99px; }
|
||||
main { padding: 20px 24px; display: flex; flex-direction: column; gap: 20px; max-width: 1400px; margin: 0 auto; width: 100%; }
|
||||
header{background:var(--surface);border-bottom:1px solid var(--border);padding:12px 20px;display:flex;align-items:center;gap:10px;position:sticky;top:0;z-index:100}
|
||||
header h1{font-size:20px;font-weight:800;letter-spacing:-1px}
|
||||
header h1 span{color:var(--accent)}
|
||||
.badge{background:var(--surface2);border:1px solid var(--border);color:var(--muted);font-size:11px;padding:2px 8px;border-radius:99px}
|
||||
.index-pill{font-size:11px;padding:3px 10px;border-radius:99px;font-weight:600}
|
||||
.index-building{background:#ffb34722;color:var(--warn)}
|
||||
.index-ready{background:#00d4aa22;color:var(--accent2)}
|
||||
|
||||
/* Cards */
|
||||
.card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 18px; }
|
||||
.card-title { font-size: 13px; font-weight: 600; color: var(--muted); text-transform: uppercase; letter-spacing: .5px; margin-bottom: 14px; }
|
||||
main{padding:16px 20px;display:flex;flex-direction:column;gap:16px;max-width:1440px;margin:0 auto;width:100%}
|
||||
|
||||
/* Stats bar */
|
||||
.stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 14px; }
|
||||
.stat-box { background: var(--surface2); border-radius: var(--radius); padding: 14px 16px; border: 1px solid var(--border); }
|
||||
.stat-box .label { font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: .5px; }
|
||||
.stat-box .value { font-size: 26px; font-weight: 700; margin-top: 4px; }
|
||||
.stat-box .sub { font-size: 11px; color: var(--muted); margin-top: 2px; }
|
||||
.v-accent { color: var(--accent2); }
|
||||
.v-hot { color: var(--hot); }
|
||||
.v-warn { color: var(--warn); }
|
||||
.v-muted { color: var(--muted); }
|
||||
.card{background:var(--surface);border:1px solid var(--border);border-radius:var(--r);padding:16px}
|
||||
.card-title{font-size:11px;font-weight:700;color:var(--muted);text-transform:uppercase;letter-spacing:.6px;margin-bottom:12px}
|
||||
|
||||
/* Tabs */
|
||||
.tabs { display: flex; gap: 4px; border-bottom: 1px solid var(--border); margin-bottom: 16px; }
|
||||
.tab { padding: 8px 16px; border-radius: 6px 6px 0 0; cursor: pointer; font-size: 13px; font-weight: 500; color: var(--muted); border: 1px solid transparent; border-bottom: none; }
|
||||
.tab.active { background: var(--surface2); color: var(--text); border-color: var(--border); }
|
||||
.tab:hover:not(.active) { color: var(--text); }
|
||||
/* Stats */
|
||||
.stats-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px}
|
||||
.stat{background:var(--surface2);border:1px solid var(--border);border-radius:var(--r);padding:12px 14px}
|
||||
.stat .lbl{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.4px}
|
||||
.stat .val{font-size:24px;font-weight:800;margin-top:2px}
|
||||
.stat .sub{font-size:11px;color:var(--muted);margin-top:1px}
|
||||
.c-accent{color:var(--accent2)} .c-hot{color:var(--hot)} .c-warn{color:var(--warn)} .c-muted{color:var(--muted)}
|
||||
|
||||
/* Filters */
|
||||
.filter-row { display: flex; flex-wrap: wrap; gap: 10px; align-items: flex-end; margin-bottom: 14px; }
|
||||
.field { display: flex; flex-direction: column; gap: 4px; }
|
||||
.field label { font-size: 11px; color: var(--muted); text-transform: uppercase; }
|
||||
input[type=text], input[type=number], select {
|
||||
background: var(--surface2); border: 1px solid var(--border); color: var(--text);
|
||||
padding: 7px 10px; border-radius: 6px; font-size: 13px; outline: none; min-width: 100px;
|
||||
}
|
||||
input[type=text]:focus, select:focus { border-color: var(--accent); }
|
||||
input[type=range] { accent-color: var(--accent); width: 120px; }
|
||||
.toggle { display: flex; align-items: center; gap: 6px; cursor: pointer; }
|
||||
.toggle input { accent-color: var(--accent); width: 16px; height: 16px; cursor: pointer; }
|
||||
/* Tabs */
|
||||
.tabs{display:flex;gap:2px;padding:0 2px;border-bottom:1px solid var(--border)}
|
||||
.tab{padding:8px 16px;cursor:pointer;font-size:13px;font-weight:500;color:var(--muted);border-radius:6px 6px 0 0;border:1px solid transparent;border-bottom:none;user-select:none}
|
||||
.tab.active{background:var(--surface);color:var(--text);border-color:var(--border)}
|
||||
.tab:hover:not(.active){color:var(--text)}
|
||||
|
||||
/* Buttons */
|
||||
.btn { padding: 7px 14px; border-radius: 6px; font-size: 13px; font-weight: 600; cursor: pointer; border: none; transition: opacity .15s; }
|
||||
.btn:hover { opacity: .85; }
|
||||
.btn-primary { background: var(--accent); color: #fff; }
|
||||
.btn-success { background: var(--accent2); color: #111; }
|
||||
.btn-danger { background: var(--danger); color: #fff; }
|
||||
.btn-warn { background: var(--warn); color: #111; }
|
||||
.btn-ghost { background: var(--surface2); color: var(--text); border: 1px solid var(--border); }
|
||||
.btn:disabled { opacity: .4; cursor: not-allowed; }
|
||||
/* Filters */
|
||||
.filters{display:flex;flex-wrap:wrap;gap:10px;align-items:flex-end;margin-bottom:12px}
|
||||
.field{display:flex;flex-direction:column;gap:3px}
|
||||
.field label{font-size:11px;color:var(--muted);text-transform:uppercase;font-weight:600}
|
||||
input[type=text],input[type=number],select{
|
||||
background:var(--surface2);border:1px solid var(--border);color:var(--text);
|
||||
padding:6px 10px;border-radius:6px;font-size:13px;outline:none
|
||||
}
|
||||
input[type=text]:focus,select:focus{border-color:var(--accent)}
|
||||
input[type=range]{accent-color:var(--accent);width:110px;cursor:pointer}
|
||||
.toggle{display:flex;align-items:center;gap:6px;cursor:pointer;padding:6px 0}
|
||||
.toggle input{accent-color:var(--accent);width:15px;height:15px;cursor:pointer}
|
||||
|
||||
/* Table */
|
||||
.table-wrap { overflow-x: auto; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
||||
th { text-align: left; padding: 8px 10px; font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: .4px; border-bottom: 1px solid var(--border); background: var(--surface2); position: sticky; top: 0; }
|
||||
td { padding: 8px 10px; border-bottom: 1px solid var(--border); }
|
||||
tr:hover td { background: var(--surface2); }
|
||||
.pill { display: inline-block; padding: 2px 8px; border-radius: 99px; font-size: 11px; font-weight: 600; }
|
||||
.pill-green { background: #00d4aa22; color: var(--accent2); }
|
||||
.pill-red { background: #ff4f6d22; color: var(--danger); }
|
||||
.pill-grey { background: #ffffff11; color: var(--muted); }
|
||||
.pill-cms { background: #6c63ff22; color: var(--accent); }
|
||||
/* Buttons */
|
||||
.btn{padding:6px 14px;border-radius:6px;font-size:13px;font-weight:600;cursor:pointer;border:none;transition:opacity .15s;white-space:nowrap}
|
||||
.btn:hover:not(:disabled){opacity:.85} .btn:disabled{opacity:.35;cursor:not-allowed}
|
||||
.btn-primary{background:var(--accent);color:#fff}
|
||||
.btn-success{background:var(--accent2);color:#111}
|
||||
.btn-danger{background:var(--danger);color:#fff}
|
||||
.btn-warn{background:var(--warn);color:#111}
|
||||
.btn-ghost{background:var(--surface2);color:var(--text);border:1px solid var(--border)}
|
||||
.btn-sm{padding:4px 10px;font-size:12px}
|
||||
|
||||
/* Score badge */
|
||||
.score-badge { display: inline-block; padding: 2px 7px; border-radius: 6px; font-weight: 700; font-size: 12px; }
|
||||
/* Table */
|
||||
.table-wrap{overflow-x:auto;border-radius:var(--r);border:1px solid var(--border)}
|
||||
table{width:100%;border-collapse:collapse;font-size:13px}
|
||||
th{text-align:left;padding:8px 10px;font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.4px;background:var(--surface2);border-bottom:1px solid var(--border);white-space:nowrap}
|
||||
td{padding:7px 10px;border-bottom:1px solid var(--border);vertical-align:middle}
|
||||
tr:last-child td{border-bottom:none}
|
||||
tr:hover td{background:var(--surface2)}
|
||||
.pill{display:inline-block;padding:2px 7px;border-radius:99px;font-size:11px;font-weight:600}
|
||||
.p-green{background:#00d4aa22;color:var(--accent2)} .p-red{background:#ff4f6d22;color:var(--danger)}
|
||||
.p-grey{background:#ffffff11;color:var(--muted)} .p-cms{background:#6c63ff22;color:var(--accent)}
|
||||
|
||||
/* Progress bar */
|
||||
.progress-wrap { background: var(--surface2); border-radius: 99px; height: 10px; overflow: hidden; margin: 8px 0; }
|
||||
.progress-bar { height: 100%; background: linear-gradient(90deg, var(--accent), var(--accent2)); border-radius: 99px; transition: width .4s; }
|
||||
/* Score badge */
|
||||
.score{display:inline-block;padding:2px 7px;border-radius:6px;font-weight:800;font-size:12px;min-width:32px;text-align:center}
|
||||
|
||||
/* Pipeline columns */
|
||||
.pipeline { display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 14px; }
|
||||
.pipe-col { background: var(--surface2); border-radius: var(--radius); border: 1px solid var(--border); padding: 14px; }
|
||||
.pipe-col h3 { font-size: 15px; font-weight: 700; margin-bottom: 4px; }
|
||||
.pipe-col .count { font-size: 28px; font-weight: 800; }
|
||||
.pipe-col .samples { margin-top: 10px; display: flex; flex-direction: column; gap: 4px; }
|
||||
.pipe-col .sample { font-size: 12px; color: var(--muted); padding: 4px 8px; background: var(--surface); border-radius: 6px; display: flex; justify-content: space-between; align-items: center; }
|
||||
/* Pagination */
|
||||
.pager{display:flex;align-items:center;gap:8px;margin-top:12px;flex-wrap:wrap}
|
||||
.pager .info{font-size:12px;color:var(--muted)}
|
||||
|
||||
/* Chart */
|
||||
.chart-wrap { max-width: 100%; height: 280px; }
|
||||
/* Progress */
|
||||
.prog-wrap{background:var(--surface2);border-radius:99px;height:10px;overflow:hidden;margin:8px 0}
|
||||
.prog-bar{height:100%;background:linear-gradient(90deg,var(--accent),var(--accent2));border-radius:99px;transition:width .5s}
|
||||
|
||||
/* Pagination */
|
||||
.pagination { display: flex; align-items: center; gap: 8px; margin-top: 12px; }
|
||||
/* Pipeline */
|
||||
.pipeline{display:grid;grid-template-columns:1fr 1fr 1fr;gap:14px}
|
||||
.pipe-col{background:var(--surface2);border-radius:var(--r);border:1px solid var(--border);padding:14px;display:flex;flex-direction:column;gap:8px}
|
||||
.pipe-col h3{font-size:15px;font-weight:700}
|
||||
.pipe-col .count{font-size:30px;font-weight:900;line-height:1}
|
||||
.samples{display:flex;flex-direction:column;gap:4px}
|
||||
.sample{font-size:12px;color:var(--muted);padding:4px 8px;background:var(--surface);border-radius:6px;display:flex;justify-content:space-between;align-items:center;gap:8px}
|
||||
|
||||
/* Enrichment status */
|
||||
.enrich-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); gap: 10px; margin-bottom: 14px; }
|
||||
.enrich-stat { background: var(--surface2); border-radius: 8px; padding: 10px 14px; text-align: center; }
|
||||
.enrich-stat .val { font-size: 22px; font-weight: 700; }
|
||||
.enrich-stat .lbl { font-size: 11px; color: var(--muted); margin-top: 2px; }
|
||||
/* Enrich stats */
|
||||
.eq-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(110px,1fr));gap:10px;margin-bottom:14px}
|
||||
.eq-stat{background:var(--surface2);border-radius:var(--r);padding:10px;text-align:center}
|
||||
.eq-stat .v{font-size:22px;font-weight:800}
|
||||
.eq-stat .l{font-size:11px;color:var(--muted);margin-top:2px}
|
||||
|
||||
@media (max-width: 700px) {
|
||||
.pipeline { grid-template-columns: 1fr; }
|
||||
.stats-grid { grid-template-columns: repeat(2, 1fr); }
|
||||
}
|
||||
/* Toast */
|
||||
.toast{position:fixed;bottom:24px;right:24px;background:var(--surface2);border:1px solid var(--border);border-radius:var(--r);padding:12px 18px;font-size:13px;font-weight:600;z-index:9999;display:flex;align-items:center;gap:10px;box-shadow:0 4px 24px #0008;transition:opacity .3s}
|
||||
.toast.hidden{opacity:0;pointer-events:none}
|
||||
.toast.success{border-color:var(--accent2);color:var(--accent2)}
|
||||
.toast.error{border-color:var(--danger);color:var(--danger)}
|
||||
|
||||
/* Chart */
|
||||
.chart-wrap{height:280px}
|
||||
|
||||
@media(max-width:700px){.pipeline{grid-template-columns:1fr}.stats-grid{grid-template-columns:1fr 1fr}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="shell" x-data="dashboard()" x-init="init()">
|
||||
<body x-data="app()" x-init="init()">
|
||||
|
||||
<header>
|
||||
<!-- Toast -->
|
||||
<div class="toast" :class="[toast.type, {hidden: !toast.show}]" x-text="toast.msg"></div>
|
||||
|
||||
<header>
|
||||
<h1>Dom<span>God</span></h1>
|
||||
<span class="badge" x-text="'v1.0'"></span>
|
||||
<span class="badge" x-text="stats.total_domains ? stats.total_domains.toLocaleString() + ' domains' : 'Loading…'"></span>
|
||||
<span class="index-pill"
|
||||
:class="indexSt.ready ? 'index-ready' : 'index-building'"
|
||||
x-text="indexSt.ready ? '⚡ Index ready' : '⏳ Building index…'">
|
||||
</span>
|
||||
<span style="flex:1"></span>
|
||||
<span style="font-size:12px; color:var(--muted)" x-text="stats.total_domains ? stats.total_domains.toLocaleString() + ' domains' : 'Loading...'"></span>
|
||||
</header>
|
||||
<span style="font-size:12px;color:var(--muted)" x-text="stats.enriched ? stats.enriched.toLocaleString() + ' enriched' : ''"></span>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<main>
|
||||
|
||||
<!-- ① Stats Bar -->
|
||||
<!-- Stats bar -->
|
||||
<div class="card">
|
||||
<div class="card-title">Overview</div>
|
||||
<div class="stats-grid">
|
||||
<div class="stat-box">
|
||||
<div class="label">Total Domains</div>
|
||||
<div class="value v-accent" x-text="stats.total_domains ? stats.total_domains.toLocaleString() : '—'"></div>
|
||||
<div class="sub">in parquet</div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="label">Enriched</div>
|
||||
<div class="value v-accent" x-text="stats.enriched ? stats.enriched.toLocaleString() : '0'"></div>
|
||||
<div class="sub" x-text="stats.total_domains ? ((stats.enriched/stats.total_domains*100).toFixed(3) + '%') : ''"></div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="label">Hot Leads</div>
|
||||
<div class="value v-hot" x-text="stats.hot_leads ? stats.hot_leads.toLocaleString() : '0'"></div>
|
||||
<div class="sub">score ≥ 60</div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="label">Queue Pending</div>
|
||||
<div class="value v-warn" x-text="stats.queue ? stats.queue.pending.toLocaleString() : '0'"></div>
|
||||
<div class="sub" x-text="stats.queue ? stats.queue.running + ' running' : ''"></div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="label">Done / Failed</div>
|
||||
<div class="value v-muted" x-text="stats.queue ? stats.queue.done.toLocaleString() : '0'"></div>
|
||||
<div class="sub" x-text="stats.queue ? stats.queue.failed + ' failed' : ''"></div>
|
||||
</div>
|
||||
<div class="stat"><div class="lbl">Total Domains</div><div class="val c-accent" x-text="stats.total_domains?.toLocaleString() ?? '—'"></div><div class="sub">in dataset</div></div>
|
||||
<div class="stat"><div class="lbl">Enriched</div><div class="val c-accent" x-text="stats.enriched?.toLocaleString() ?? '0'"></div><div class="sub" x-text="stats.total_domains ? ((stats.enriched/stats.total_domains*100).toFixed(3)+'%') : ''"></div></div>
|
||||
<div class="stat"><div class="lbl">Hot Leads</div><div class="val c-hot" x-text="stats.hot_leads?.toLocaleString() ?? '0'"></div><div class="sub">score ≥ 60</div></div>
|
||||
<div class="stat"><div class="lbl">Queue Pending</div><div class="val c-warn" x-text="stats.queue?.pending?.toLocaleString() ?? '0'"></div><div class="sub" x-text="(stats.queue?.running ?? 0) + ' running'"></div></div>
|
||||
<div class="stat"><div class="lbl">Done / Failed</div><div class="val c-muted" x-text="stats.queue?.done?.toLocaleString() ?? '0'"></div><div class="sub" x-text="(stats.queue?.failed ?? 0) + ' failed'"></div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Tabs -->
|
||||
<div class="tabs">
|
||||
<div class="tab" :class="{active: tab==='browse'}" @click="tab='browse'">Browse & Filter</div>
|
||||
<div class="tab" :class="{active: tab==='enrichment'}" @click="tab='enrichment'">Enrichment Queue</div>
|
||||
<div class="tab" :class="{active: tab==='pipeline'}" @click="tab='pipeline'">Lead Pipeline</div>
|
||||
<div class="tab" :class="{active: tab==='chart'}" @click="tab='chart'; renderChart()">TLD Chart</div>
|
||||
<div class="tab" :class="{active:tab==='browse'}" @click="tab='browse'">Browse & Filter</div>
|
||||
<div class="tab" :class="{active:tab==='enrich'}" @click="tab='enrich'; loadQueue()">Enrichment Queue</div>
|
||||
<div class="tab" :class="{active:tab==='pipeline'}" @click="tab='pipeline'; loadPipeline()">Lead Pipeline</div>
|
||||
<div class="tab" :class="{active:tab==='chart'}" @click="tab='chart'; renderChart()">TLD Chart</div>
|
||||
</div>
|
||||
|
||||
<!-- ② Browse & Filter -->
|
||||
<div class="card" x-show="tab==='browse'">
|
||||
<div class="filter-row">
|
||||
<div class="filters">
|
||||
<div class="field">
|
||||
<label>TLD</label>
|
||||
<input type="text" x-model="filter.tld" placeholder="es, com…" @keydown.enter="loadDomains()">
|
||||
<input type="text" x-model="f.tld" placeholder="es, com…" style="width:90px" @keydown.enter="search()">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label>Country</label>
|
||||
<input type="text" x-model="filter.country" placeholder="ES, GB…">
|
||||
<label>Keyword</label>
|
||||
<input type="text" x-model="f.keyword" placeholder="hotel, dental…" style="width:130px" @keydown.enter="search()">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label>Min Score: <strong x-text="filter.min_score"></strong></label>
|
||||
<input type="range" x-model="filter.min_score" min="0" max="100" step="5">
|
||||
<label>Min Score: <strong x-text="f.min_score"></strong></label>
|
||||
<input type="range" x-model="f.min_score" min="0" max="100" step="5">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label>CMS</label>
|
||||
<select x-model="filter.cms">
|
||||
<option value="">Any</option>
|
||||
<select x-model="f.cms" style="width:120px">
|
||||
<option value="">Any CMS</option>
|
||||
<option>wordpress</option><option>joomla</option><option>drupal</option>
|
||||
<option>wix</option><option>squarespace</option><option>shopify</option>
|
||||
<option>prestashop</option><option>magento</option><option>typo3</option><option>opencart</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label>Per page</label>
|
||||
<select x-model="f.limit" style="width:80px">
|
||||
<option value="50">50</option>
|
||||
<option value="100">100</option>
|
||||
<option value="250">250</option>
|
||||
</select>
|
||||
</div>
|
||||
<label class="toggle field"><label>Live only</label><input type="checkbox" x-model="f.live_only"></label>
|
||||
<label class="toggle field">
|
||||
<label>Live only</label>
|
||||
<input type="checkbox" x-model="filter.live_only">
|
||||
<label>Alpha only</label>
|
||||
<input type="checkbox" x-model="f.alpha_only">
|
||||
<span style="font-size:11px;color:var(--muted)">(no hyphens/numbers)</span>
|
||||
</label>
|
||||
<button class="btn btn-primary" @click="loadDomains(1)">Search</button>
|
||||
<label class="toggle field">
|
||||
<label>No SLD</label>
|
||||
<input type="checkbox" x-model="f.no_sld">
|
||||
<span style="font-size:11px;color:var(--muted)">(skip com.es etc)</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div style="display:flex;gap:8px;margin-bottom:12px;flex-wrap:wrap;align-items:center">
|
||||
<button class="btn btn-primary" @click="search()">Search</button>
|
||||
<button class="btn btn-ghost" @click="resetFilters()">Reset</button>
|
||||
<button class="btn btn-success" @click="enqueueSelected()" :disabled="selected.length===0">
|
||||
Enrich selected (<span x-text="selected.length"></span>)
|
||||
+ Enrich (<span x-text="selected.length"></span>) selected
|
||||
</button>
|
||||
<button class="btn btn-ghost btn-sm" @click="selectAll()" x-show="domains.length > 0">Select all on page</button>
|
||||
<button class="btn btn-ghost btn-sm" @click="selected=[]" x-show="selected.length > 0">Clear selection</button>
|
||||
<span class="info" style="font-size:12px;color:var(--muted)" x-show="searchTotal > 0">
|
||||
<strong x-text="searchTotal.toLocaleString()"></strong> matches
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th><input type="checkbox" @change="toggleAll($event)"></th>
|
||||
<th style="width:32px"></th>
|
||||
<th>Domain</th>
|
||||
<th>Score</th>
|
||||
<th>CMS</th>
|
||||
<th>SSL days</th>
|
||||
<th>Country</th>
|
||||
<th>Live</th>
|
||||
<th>Server</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<template x-if="loading">
|
||||
<tr><td colspan="9" style="text-align:center;padding:28px;color:var(--muted)">Searching… (first query without index may take 30-60s)</td></tr>
|
||||
</template>
|
||||
<template x-if="!loading && domains.length === 0">
|
||||
<tr><td colspan="9" style="text-align:center;padding:28px;color:var(--muted)">No results — enter a TLD or keyword and click Search</td></tr>
|
||||
</template>
|
||||
<template x-for="row in domains" :key="row.domain">
|
||||
<tr>
|
||||
<td><input type="checkbox" :value="row.domain" x-model="selected"></td>
|
||||
<td><a :href="'http://'+row.domain" target="_blank" rel="noopener" x-text="row.domain"></a></td>
|
||||
<td>
|
||||
<a :href="'http://'+row.domain" target="_blank" x-text="row.domain"></a>
|
||||
<template x-if="row.score != null">
|
||||
<span class="score" :style="scoreBg(row.score)" x-text="row.score"></span>
|
||||
</template>
|
||||
<template x-if="row.score == null">
|
||||
<span class="pill p-grey">—</span>
|
||||
</template>
|
||||
</td>
|
||||
<td>
|
||||
<span class="score-badge"
|
||||
:style="scoreBg(row.score)"
|
||||
x-text="row.score ?? '—'"></span>
|
||||
</td>
|
||||
<td>
|
||||
<span x-show="row.cms" class="pill pill-cms" x-text="row.cms"></span>
|
||||
<span x-show="!row.cms" class="pill pill-grey">—</span>
|
||||
<span x-show="row.cms" class="pill p-cms" x-text="row.cms"></span>
|
||||
<span x-show="!row.cms" class="pill p-grey">—</span>
|
||||
</td>
|
||||
<td x-text="row.ssl_expiry_days ?? '—'"></td>
|
||||
<td x-text="row.ip_country ?? '—'"></td>
|
||||
<td>
|
||||
<span class="pill" :class="row.is_live ? 'pill-green' : 'pill-grey'" x-text="row.is_live ? 'Yes' : '—'"></span>
|
||||
</td>
|
||||
<td><span class="pill" :class="row.is_live ? 'p-green' : 'p-grey'" x-text="row.is_live ? 'Yes' : '—'"></span></td>
|
||||
<td style="font-size:11px;color:var(--muted);max-width:120px;overflow:hidden;white-space:nowrap;text-overflow:ellipsis" x-text="row.server ?? '—'"></td>
|
||||
<td x-text="row.status_code ?? '—'"></td>
|
||||
</tr>
|
||||
</template>
|
||||
<tr x-show="domains.length===0 && !loading">
|
||||
<td colspan="8" style="text-align:center;color:var(--muted);padding:24px">No results — run a search above</td>
|
||||
</tr>
|
||||
<tr x-show="loading">
|
||||
<td colspan="8" style="text-align:center;color:var(--muted);padding:24px">Loading…</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="pagination">
|
||||
<button class="btn btn-ghost" @click="loadDomains(page-1)" :disabled="page<=1">← Prev</button>
|
||||
<span style="color:var(--muted)">Page <strong x-text="page"></strong></span>
|
||||
<button class="btn btn-ghost" @click="loadDomains(page+1)" :disabled="domains.length < filter.limit">Next →</button>
|
||||
<span style="color:var(--muted);margin-left:8px">Limit:
|
||||
<select x-model="filter.limit" @change="loadDomains(1)" style="width:80px">
|
||||
<option value="50">50</option>
|
||||
<option value="100">100</option>
|
||||
<option value="250">250</option>
|
||||
</select>
|
||||
<!-- Pagination -->
|
||||
<div class="pager">
|
||||
<button class="btn btn-ghost btn-sm" @click="goPage(page-1)" :disabled="page<=1 || loading">← Prev</button>
|
||||
<span class="info">Page <strong x-text="page"></strong>
|
||||
<template x-if="searchTotal > 0">
|
||||
<span x-text="' of ' + Math.ceil(searchTotal / Number(f.limit))"></span>
|
||||
</template>
|
||||
</span>
|
||||
<button class="btn btn-ghost btn-sm" @click="goPage(page+1)" :disabled="loading || domains.length < Number(f.limit)">Next →</button>
|
||||
<span class="info" x-show="searchTotal > 0" x-text="searchTotal.toLocaleString() + ' total results'"></span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ③ Enrichment Queue -->
|
||||
<div class="card" x-show="tab==='enrichment'">
|
||||
<div class="enrich-grid">
|
||||
<div class="enrich-stat">
|
||||
<div class="val v-warn" x-text="queueStatus.pending ?? '—'"></div>
|
||||
<div class="lbl">Pending</div>
|
||||
</div>
|
||||
<div class="enrich-stat">
|
||||
<div class="val v-accent" x-text="queueStatus.running ?? '—'"></div>
|
||||
<div class="lbl">Running</div>
|
||||
</div>
|
||||
<div class="enrich-stat">
|
||||
<div class="val v-accent" x-text="queueStatus.done ?? '—'"></div>
|
||||
<div class="lbl">Done</div>
|
||||
</div>
|
||||
<div class="enrich-stat">
|
||||
<div class="val v-hot" x-text="queueStatus.failed ?? '—'"></div>
|
||||
<div class="lbl">Failed</div>
|
||||
</div>
|
||||
<div class="enrich-stat">
|
||||
<div class="val v-muted" x-text="queueStatus.eta_seconds ? Math.ceil(queueStatus.eta_seconds/60) + 'm' : '—'"></div>
|
||||
<div class="lbl">ETA</div>
|
||||
</div>
|
||||
<div class="card" x-show="tab==='enrich'">
|
||||
<div class="eq-grid">
|
||||
<div class="eq-stat"><div class="v c-warn" x-text="qst.pending ?? '—'"></div><div class="l">Pending</div></div>
|
||||
<div class="eq-stat"><div class="v c-accent" x-text="qst.running ?? '—'"></div><div class="l">Running</div></div>
|
||||
<div class="eq-stat"><div class="v c-accent" x-text="qst.done ?? '—'"></div><div class="l">Done</div></div>
|
||||
<div class="eq-stat"><div class="v c-hot" x-text="qst.failed ?? '—'"></div><div class="l">Failed</div></div>
|
||||
<div class="eq-stat"><div class="v c-muted" x-text="qst.eta_seconds ? Math.ceil(qst.eta_seconds/60)+'m' : '—'"></div><div class="l">ETA</div></div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:6px">
|
||||
<span style="font-size:12px;color:var(--muted)" x-text="progressLabel()"></span>
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:6px;flex-wrap:wrap;gap:8px">
|
||||
<span style="font-size:12px;color:var(--muted)" x-text="qLabel()"></span>
|
||||
<div style="display:flex;gap:8px">
|
||||
<button class="btn btn-success" @click="startEnrich()" x-show="!enrichRunning">▶ Start</button>
|
||||
<button class="btn btn-warn" @click="pauseEnrich()" x-show="enrichRunning">⏸ Pause</button>
|
||||
<button class="btn btn-success" x-show="!qst.worker_running" @click="startEnrich()">▶ Start</button>
|
||||
<button class="btn btn-warn" x-show="qst.worker_running" @click="pauseEnrich()">⏸ Pause</button>
|
||||
<button class="btn btn-ghost" @click="retryFailed()">↺ Retry Failed</button>
|
||||
<button class="btn btn-ghost" @click="runScoring()">★ Score All</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="progress-wrap">
|
||||
<div class="progress-bar" :style="'width:' + progressPct() + '%'"></div>
|
||||
</div>
|
||||
<div style="font-size:11px;color:var(--muted);margin-top:4px" x-text="progressPct().toFixed(1) + '% complete'"></div>
|
||||
<div class="prog-wrap"><div class="prog-bar" :style="'width:'+qPct()+'%'"></div></div>
|
||||
<div style="font-size:11px;color:var(--muted);margin-top:4px" x-text="qPct().toFixed(1)+'% complete'"></div>
|
||||
</div>
|
||||
|
||||
<div style="margin-top:20px">
|
||||
<div class="card-title">Enrich custom domains</div>
|
||||
<div style="display:flex;gap:8px;align-items:flex-start">
|
||||
<textarea
|
||||
x-model="customDomains"
|
||||
placeholder="example.com another.es third.net"
|
||||
style="flex:1;background:var(--surface2);border:1px solid var(--border);color:var(--text);border-radius:6px;padding:8px;min-height:80px;font-size:12px;resize:vertical"
|
||||
></textarea>
|
||||
<button class="btn btn-primary" @click="enqueueCustom()" style="align-self:flex-end">Queue</button>
|
||||
<div class="card-title">Queue custom domains</div>
|
||||
<div style="display:flex;gap:8px;align-items:flex-end">
|
||||
<div style="flex:1">
|
||||
<textarea x-model="customDomains" placeholder="example.com another.es"
|
||||
style="width:100%;background:var(--surface2);border:1px solid var(--border);color:var(--text);border-radius:6px;padding:8px;min-height:80px;font-size:12px;resize:vertical"></textarea>
|
||||
</div>
|
||||
<button class="btn btn-primary" @click="enqueueCustom()">Queue</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ④ Lead Pipeline -->
|
||||
<div class="card" x-show="tab==='pipeline'">
|
||||
<div style="display:flex;justify-content:flex-end;gap:8px;margin-bottom:14px">
|
||||
<button class="btn btn-ghost" @click="loadPipeline()">↻ Refresh</button>
|
||||
<div style="display:flex;justify-content:flex-end;margin-bottom:12px">
|
||||
<button class="btn btn-ghost btn-sm" @click="loadPipeline()">↻ Refresh</button>
|
||||
</div>
|
||||
<div class="pipeline">
|
||||
<!-- Hot -->
|
||||
<div class="pipe-col" style="border-top:3px solid var(--hot)">
|
||||
<h3>🔥 Hot</h3>
|
||||
<div style="color:var(--muted);font-size:12px">Score 80–100</div>
|
||||
<div class="count" style="color:var(--hot)" x-text="pipeline.hot.count.toLocaleString()"></div>
|
||||
<div style="font-size:12px;color:var(--muted)">Score 80–100</div>
|
||||
<div class="count c-hot" x-text="pipeline.hot.count.toLocaleString()"></div>
|
||||
<div class="samples">
|
||||
<template x-for="d in pipeline.hot.samples" :key="d.domain">
|
||||
<div class="sample">
|
||||
<a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a>
|
||||
<span class="score-badge" :style="scoreBg(d.score)" x-text="d.score"></span>
|
||||
</div>
|
||||
<div class="sample"><a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a><span class="score" :style="scoreBg(d.score)" x-text="d.score"></span></div>
|
||||
</template>
|
||||
</div>
|
||||
<button class="btn btn-danger" style="margin-top:12px;width:100%" @click="exportTier('hot')">⬇ Export Hot CSV</button>
|
||||
<button class="btn btn-danger btn-sm" style="margin-top:auto" @click="exportTier('hot')">⬇ Export Hot CSV</button>
|
||||
</div>
|
||||
<!-- Warm -->
|
||||
<div class="pipe-col" style="border-top:3px solid var(--warm)">
|
||||
<h3>♨️ Warm</h3>
|
||||
<div style="color:var(--muted);font-size:12px">Score 50–79</div>
|
||||
<div class="count" style="color:var(--warm)" x-text="pipeline.warm.count.toLocaleString()"></div>
|
||||
<div style="font-size:12px;color:var(--muted)">Score 50–79</div>
|
||||
<div class="count c-warn" x-text="pipeline.warm.count.toLocaleString()"></div>
|
||||
<div class="samples">
|
||||
<template x-for="d in pipeline.warm.samples" :key="d.domain">
|
||||
<div class="sample">
|
||||
<a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a>
|
||||
<span class="score-badge" :style="scoreBg(d.score)" x-text="d.score"></span>
|
||||
</div>
|
||||
<div class="sample"><a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a><span class="score" :style="scoreBg(d.score)" x-text="d.score"></span></div>
|
||||
</template>
|
||||
</div>
|
||||
<button class="btn btn-warn" style="margin-top:12px;width:100%" @click="exportTier('warm')">⬇ Export Warm CSV</button>
|
||||
<button class="btn btn-warn btn-sm" style="margin-top:auto" @click="exportTier('warm')">⬇ Export Warm CSV</button>
|
||||
</div>
|
||||
<!-- Cold -->
|
||||
<div class="pipe-col" style="border-top:3px solid var(--cold)">
|
||||
<h3>🧊 Cold</h3>
|
||||
<div style="color:var(--muted);font-size:12px">Score < 50</div>
|
||||
<div style="font-size:12px;color:var(--muted)">Score < 50</div>
|
||||
<div class="count" style="color:var(--cold)" x-text="pipeline.cold.count.toLocaleString()"></div>
|
||||
<div class="samples">
|
||||
<template x-for="d in pipeline.cold.samples" :key="d.domain">
|
||||
<div class="sample">
|
||||
<a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a>
|
||||
<span class="score-badge" :style="scoreBg(d.score)" x-text="d.score"></span>
|
||||
</div>
|
||||
<div class="sample"><a :href="'http://'+d.domain" target="_blank" x-text="d.domain"></a><span class="score" :style="scoreBg(d.score)" x-text="d.score"></span></div>
|
||||
</template>
|
||||
</div>
|
||||
<button class="btn btn-ghost" style="margin-top:12px;width:100%" @click="exportTier('cold')">⬇ Export Cold CSV</button>
|
||||
<button class="btn btn-ghost btn-sm" style="margin-top:auto" @click="exportTier('cold')">⬇ Export Cold CSV</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ⑤ TLD Chart -->
|
||||
<div class="card" x-show="tab==='chart'">
|
||||
<div class="card-title">Top 20 TLDs</div>
|
||||
<div class="chart-wrap">
|
||||
<canvas id="tldChart"></canvas>
|
||||
</div>
|
||||
<div class="card-title">Top 20 TLDs in dataset</div>
|
||||
<div class="chart-wrap"><canvas id="tldChart"></canvas></div>
|
||||
</div>
|
||||
|
||||
</main>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
function dashboard() {
|
||||
function app() {
|
||||
return {
|
||||
tab: 'browse',
|
||||
stats: {},
|
||||
indexSt: { ready: false, building: false, total: 0 },
|
||||
domains: [],
|
||||
selected: [],
|
||||
loading: false,
|
||||
page: 1,
|
||||
filter: { tld: '', country: '', min_score: 0, cms: '', live_only: false, limit: '100' },
|
||||
queueStatus: {},
|
||||
enrichRunning: false,
|
||||
searchTotal: 0,
|
||||
f: { tld:'', keyword:'', min_score:0, cms:'', live_only:false, alpha_only:false, no_sld:false, limit:'100' },
|
||||
qst: {},
|
||||
customDomains: '',
|
||||
pipeline: {
|
||||
hot: { count: 0, samples: [] },
|
||||
warm: { count: 0, samples: [] },
|
||||
cold: { count: 0, samples: [] },
|
||||
},
|
||||
pipeline: { hot:{count:0,samples:[]}, warm:{count:0,samples:[]}, cold:{count:0,samples:[]} },
|
||||
toast: { show:false, msg:'', type:'success' },
|
||||
_chart: null,
|
||||
_pollInterval: null,
|
||||
_poll: null,
|
||||
_toastTimer: null,
|
||||
|
||||
async init() {
|
||||
await this.loadStats();
|
||||
this.startPolling();
|
||||
},
|
||||
|
||||
startPolling() {
|
||||
this._pollInterval = setInterval(async () => {
|
||||
await this.loadStats();
|
||||
if (this.tab === 'enrichment') await this.loadQueueStatus();
|
||||
if (this.tab === 'pipeline') await this.loadPipeline();
|
||||
this._poll = setInterval(() => {
|
||||
this.loadStats();
|
||||
this.pollIndex();
|
||||
if (this.tab === 'enrich') this.loadQueue();
|
||||
if (this.tab === 'pipeline') this.loadPipeline();
|
||||
}, 3000);
|
||||
},
|
||||
|
||||
async loadStats() {
|
||||
try {
|
||||
const r = await fetch('/api/stats');
|
||||
this.stats = await r.json();
|
||||
} catch(e) {}
|
||||
try { this.stats = await fetch('/api/stats').then(r=>r.json()); } catch(e){}
|
||||
},
|
||||
|
||||
async loadQueueStatus() {
|
||||
try {
|
||||
const r = await fetch('/api/enrich/status');
|
||||
this.queueStatus = await r.json();
|
||||
this.enrichRunning = this.queueStatus.worker_running;
|
||||
} catch(e) {}
|
||||
async pollIndex() {
|
||||
try { this.indexSt = await fetch('/api/index/status').then(r=>r.json()); } catch(e){}
|
||||
},
|
||||
|
||||
async loadDomains(p) {
|
||||
if (p !== undefined) this.page = p;
|
||||
async search() {
|
||||
this.page = 1;
|
||||
await this._fetchDomains();
|
||||
},
|
||||
|
||||
async goPage(p) {
|
||||
if (p < 1) return;
|
||||
this.page = p;
|
||||
await this._fetchDomains();
|
||||
},
|
||||
|
||||
async _fetchDomains() {
|
||||
this.loading = true;
|
||||
const params = new URLSearchParams({
|
||||
page: this.page,
|
||||
limit: this.filter.limit,
|
||||
...(this.filter.tld && { tld: this.filter.tld }),
|
||||
...(this.filter.live_only && { live_only: 'true' }),
|
||||
});
|
||||
const p = new URLSearchParams({ page: this.page, limit: this.f.limit });
|
||||
if (this.f.tld) p.set('tld', this.f.tld.trim());
|
||||
if (this.f.keyword) p.set('keyword', this.f.keyword.trim());
|
||||
if (this.f.live_only) p.set('live_only', 'true');
|
||||
if (this.f.alpha_only) p.set('alpha_only', 'true');
|
||||
if (this.f.no_sld) p.set('no_sld', 'true');
|
||||
try {
|
||||
const r = await fetch('/api/domains?' + params);
|
||||
const data = await r.json();
|
||||
// Filter by country/min_score client-side for enriched rows
|
||||
const data = await fetch('/api/domains?' + p).then(r => r.json());
|
||||
this.searchTotal = data.total ?? 0;
|
||||
// Client-side score/cms filter (only applies to already-enriched rows)
|
||||
this.domains = data.results.filter(row => {
|
||||
if (this.filter.min_score > 0 && (row.score ?? 0) < this.filter.min_score) return false;
|
||||
if (this.filter.country && row.ip_country !== this.filter.country.toUpperCase()) return false;
|
||||
if (this.filter.cms && row.cms !== this.filter.cms) return false;
|
||||
if (this.f.min_score > 0 && row.score != null && row.score < Number(this.f.min_score)) return false;
|
||||
if (this.f.cms && row.cms !== this.f.cms) return false;
|
||||
return true;
|
||||
});
|
||||
} catch(e) { this.domains = []; }
|
||||
} catch(e) {
|
||||
this.domains = [];
|
||||
this.notify('Search failed: ' + e.message, 'error');
|
||||
}
|
||||
this.loading = false;
|
||||
},
|
||||
|
||||
toggleAll(e) {
|
||||
if (e.target.checked) {
|
||||
this.selected = this.domains.map(d => d.domain);
|
||||
} else {
|
||||
this.selected = [];
|
||||
}
|
||||
},
|
||||
selectAll() { this.selected = this.domains.map(d => d.domain); },
|
||||
resetFilters() { this.f = { tld:'', keyword:'', min_score:0, cms:'', live_only:false, alpha_only:false, no_sld:false, limit:'100' }; },
|
||||
|
||||
async enqueueSelected() {
|
||||
if (!this.selected.length) return;
|
||||
await fetch('/api/enrich/batch', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
try {
|
||||
const r = await fetch('/api/enrich/batch', {
|
||||
method:'POST', headers:{'Content-Type':'application/json'},
|
||||
body: JSON.stringify({ domains: this.selected }),
|
||||
});
|
||||
const data = await r.json();
|
||||
if (r.ok) {
|
||||
this.notify(`Queued ${data.queued} domain(s) for enrichment`, 'success');
|
||||
this.selected = [];
|
||||
alert('Queued for enrichment!');
|
||||
} else {
|
||||
this.notify('Error: ' + (data.error || r.statusText), 'error');
|
||||
}
|
||||
} catch(e) { this.notify('Request failed: ' + e.message, 'error'); }
|
||||
},
|
||||
|
||||
async enqueueCustom() {
|
||||
const domains = this.customDomains.split('\n').map(d => d.trim()).filter(Boolean);
|
||||
const domains = this.customDomains.split('\n').map(d=>d.trim()).filter(Boolean);
|
||||
if (!domains.length) return;
|
||||
await fetch('/api/enrich/batch', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
const r = await fetch('/api/enrich/batch', {
|
||||
method:'POST', headers:{'Content-Type':'application/json'},
|
||||
body: JSON.stringify({ domains }),
|
||||
});
|
||||
this.customDomains = '';
|
||||
await this.loadQueueStatus();
|
||||
const data = await r.json();
|
||||
if (r.ok) { this.notify(`Queued ${data.queued} domains`, 'success'); this.customDomains = ''; }
|
||||
else { this.notify('Error: ' + data.error, 'error'); }
|
||||
await this.loadQueue();
|
||||
},
|
||||
|
||||
async startEnrich() {
|
||||
await fetch('/api/enrich/resume', { method: 'POST' });
|
||||
this.enrichRunning = true;
|
||||
await this.loadQueueStatus();
|
||||
async loadQueue() {
|
||||
try { this.qst = await fetch('/api/enrich/status').then(r=>r.json()); } catch(e){}
|
||||
},
|
||||
|
||||
async pauseEnrich() {
|
||||
await fetch('/api/enrich/pause', { method: 'POST' });
|
||||
this.enrichRunning = false;
|
||||
},
|
||||
async startEnrich() { await fetch('/api/enrich/resume',{method:'POST'}); this.notify('Worker started','success'); await this.loadQueue(); },
|
||||
async pauseEnrich() { await fetch('/api/enrich/pause',{method:'POST'}); this.notify('Worker paused','success'); await this.loadQueue(); },
|
||||
async retryFailed() { await fetch('/api/enrich/retry',{method:'POST'}); this.notify('Retrying failed jobs','success'); await this.loadQueue(); },
|
||||
async runScoring() { const r = await fetch('/api/score/run',{method:'POST'}); const d = await r.json(); this.notify(`Scored ${d.scored} domains`,'success'); },
|
||||
|
||||
async retryFailed() {
|
||||
// Mark failed jobs as pending
|
||||
await fetch('/api/enrich/retry', { method: 'POST' });
|
||||
await this.loadQueueStatus();
|
||||
},
|
||||
|
||||
progressPct() {
|
||||
const q = this.queueStatus;
|
||||
if (!q || !q.total) return 0;
|
||||
return (q.done / q.total) * 100;
|
||||
},
|
||||
|
||||
progressLabel() {
|
||||
const q = this.queueStatus;
|
||||
if (!q) return '';
|
||||
return `${q.done ?? 0} done · ${q.pending ?? 0} pending · ${q.running ?? 0} running · ${q.failed ?? 0} failed`;
|
||||
},
|
||||
qPct() { const q=this.qst; if(!q||!q.total) return 0; return (q.done/q.total)*100; },
|
||||
qLabel() { const q=this.qst; return `${q.done??0} done · ${q.pending??0} pending · ${q.running??0} running · ${q.failed??0} failed`; },
|
||||
|
||||
async loadPipeline() {
|
||||
try {
|
||||
const [hot, warm, cold] = await Promise.all([
|
||||
fetch('/api/enriched?min_score=80&limit=5').then(r => r.json()),
|
||||
fetch('/api/enriched?min_score=50&limit=5').then(r => r.json()),
|
||||
fetch('/api/enriched?min_score=0&limit=5').then(r => r.json()),
|
||||
const [hot, warm, cold, hotC, warmC, coldC] = await Promise.all([
|
||||
fetch('/api/enriched?min_score=80&limit=5').then(r=>r.json()),
|
||||
fetch('/api/enriched?min_score=50&limit=6').then(r=>r.json()),
|
||||
fetch('/api/enriched?min_score=0&limit=6').then(r=>r.json()),
|
||||
fetch('/api/enriched?min_score=80&limit=1000').then(r=>r.json()),
|
||||
fetch('/api/enriched?min_score=50&limit=1000').then(r=>r.json()),
|
||||
fetch('/api/enriched?min_score=0&limit=1000').then(r=>r.json()),
|
||||
]);
|
||||
// Fetch counts separately
|
||||
const [hc, wc, cc] = await Promise.all([
|
||||
fetch('/api/enriched?min_score=80&limit=1').then(r => r.json()),
|
||||
fetch('/api/enriched?min_score=50&limit=1').then(r => r.json()),
|
||||
fetch('/api/enriched?min_score=0&limit=1').then(r => r.json()),
|
||||
]);
|
||||
|
||||
const warmFiltered = warm.results.filter(d => d.score < 80);
|
||||
const coldFiltered = cold.results.filter(d => d.score < 50);
|
||||
|
||||
this.pipeline.hot = { count: hot.results.length, samples: hot.results.slice(0,5) };
|
||||
this.pipeline.warm = { count: warmFiltered.length, samples: warmFiltered.slice(0,5) };
|
||||
this.pipeline.cold = { count: coldFiltered.length, samples: coldFiltered.slice(0,5) };
|
||||
} catch(e) {}
|
||||
this.pipeline.hot = { count: hotC.total ?? hot.results.length, samples: hot.results.slice(0,5) };
|
||||
this.pipeline.warm = { count: (warmC.total ?? warm.results.length) - (hotC.total ?? 0), samples: warm.results.filter(d=>d.score<80).slice(0,5) };
|
||||
this.pipeline.cold = { count: (coldC.total ?? cold.results.length) - (warmC.total ?? 0), samples: cold.results.filter(d=>d.score<50).slice(0,5) };
|
||||
},
|
||||
|
||||
exportTier(tier) {
|
||||
window.location = `/api/export?tier=${tier}`;
|
||||
},
|
||||
exportTier(tier) { window.location = `/api/export?tier=${tier}`; },
|
||||
|
||||
scoreBg(score) {
|
||||
if (score == null) return 'background:#333;color:#aaa';
|
||||
if (score >= 80) return 'background:#ff4f6d33;color:#ff4f6d';
|
||||
if (score >= 50) return 'background:#ffb34733;color:#ffb347';
|
||||
scoreBg(s) {
|
||||
if (s == null) return 'background:#333;color:#aaa';
|
||||
if (s >= 80) return 'background:#ff4f6d33;color:#ff4f6d';
|
||||
if (s >= 50) return 'background:#ffb34733;color:#ffb347';
|
||||
return 'background:#6c7aff33;color:#6c7aff';
|
||||
},
|
||||
|
||||
notify(msg, type='success') {
|
||||
clearTimeout(this._toastTimer);
|
||||
this.toast = { show:true, msg, type };
|
||||
this._toastTimer = setTimeout(() => { this.toast.show = false; }, 3500);
|
||||
},
|
||||
|
||||
async renderChart() {
|
||||
await this.$nextTick();
|
||||
const canvas = document.getElementById('tldChart');
|
||||
if (!canvas) return;
|
||||
if (this._chart) { this._chart.destroy(); this._chart = null; }
|
||||
|
||||
const tlds = this.stats.tld_breakdown || [];
|
||||
if (!tlds.length) {
|
||||
await this.loadStats();
|
||||
}
|
||||
const labels = (this.stats.tld_breakdown || []).map(t => '.' + (t.tld || '?'));
|
||||
const data = (this.stats.tld_breakdown || []).map(t => t.count);
|
||||
|
||||
this._chart = new Chart(canvas, {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels,
|
||||
datasets: [{
|
||||
label: 'Domains',
|
||||
data,
|
||||
backgroundColor: 'rgba(108, 99, 255, 0.7)',
|
||||
borderColor: 'rgba(108, 99, 255, 1)',
|
||||
borderWidth: 1,
|
||||
borderRadius: 4,
|
||||
}]
|
||||
type:'bar',
|
||||
data:{
|
||||
labels: tlds.map(t=>'.'+t.tld),
|
||||
datasets:[{ label:'Domains', data:tlds.map(t=>t.count),
|
||||
backgroundColor:'rgba(108,99,255,0.7)', borderColor:'rgba(108,99,255,1)',
|
||||
borderWidth:1, borderRadius:4 }]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: { legend: { display: false } },
|
||||
scales: {
|
||||
x: { ticks: { color: '#8891b0' }, grid: { color: '#2e3250' } },
|
||||
y: { ticks: { color: '#8891b0' }, grid: { color: '#2e3250' } },
|
||||
options:{
|
||||
responsive:true, maintainAspectRatio:false,
|
||||
plugins:{ legend:{display:false} },
|
||||
scales:{
|
||||
x:{ticks:{color:'#8891b0'},grid:{color:'#2e3250'}},
|
||||
y:{ticks:{color:'#8891b0'},grid:{color:'#2e3250'}},
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user