Malin/DomGod
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: bulk validator tab + status/niche/type browse filters

Browse Source 
- New app/validator.py: background HTTP checker for entire dataset
  - 50 concurrent checks, skips already-validated domains
  - Extracts prescreen_status, server, IP, load_time_ms
  - start/stop/status API at /api/validator/start|stop|status

- New dedicated "Validator 🔬" tab with stats grid, TLD filter,
  Start/Stop controls, live progress indicator

- Browse tab: "Live" column replaced with "Status" dot (color-coded
  ● from prescreen_status, falls back to is_live)
- Browse tab: new Status / Niche / Type filter dropdowns

- db.py: added ip TEXT + load_time_ms INTEGER columns + migrations;
  get_enriched() supports prescreen_status/niche/site_type filters

- main.py: /api/enriched extended with prescreen_status/niche/site_type

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Malin
2026-04-18 08:27:24 +02:00
parent 468d76387d
commit 8f387cada2
4 changed files with 450 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
app/static/index.html
View File

@@ -314,6 +314,7 @@ tr:hover td{background:rgba(255,255,255,.025)}
<div class="tabs">
<div class="tab" :class="{active:tab==='browse'}" @click="tab='browse'">Browse & Filter</div>
<div class="tab" :class="{active:tab==='enrich'}" @click="tab='enrich';loadQueue()">Enrichment</div>
<div class="tab" :class="{active:tab==='validator'}" @click="tab='validator';loadValStatus()">Validator 🔬</div>
<div class="tab" :class="{active:tab==='pipeline'}" @click="tab='pipeline';loadPipeline()">Lead Pipeline</div>
<div class="tab" :class="{active:tab==='leads'}" @click="tab='leads';loadLeads(true)">Leads 🤖</div>
<div class="tab" :class="{active:tab==='chart'}" @click="tab='chart';renderChart()">TLD Chart</div>
@@ -343,6 +344,40 @@ tr:hover td{background:rgba(255,255,255,.025)}
<label class="tog"><input type="checkbox" x-model="f.no_sld"><strong>No SLD</strong> <span>(skip com.es)</span></label>
<label class="tog"><input type="checkbox" x-model="f.kit_digital_only"><strong style="color:var(--kd)">🏅 Kit Digital only</strong></label>
<label class="tog"><input type="checkbox" x-model="f.exclude_assessed"><strong>Hide assessed</strong></label>
<div class="field"><label>Status</label>
<select x-model="f.prescreen_status" style="width:105px">
<option value="">Any</option>
<option value="live">● Live</option>
<option value="dead">● Dead</option>
<option value="parked">● Parked</option>
<option value="redirect">↗ Redirect</option>
<option value="none">Not checked</option>
</select>
</div>
<div class="field"><label>Niche</label>
<select x-model="f.niche" style="width:130px">
<option value="">Any</option>
<option>automotive</option><option>beauty_cosmetics</option>
<option>travel_tourism</option><option>hospitality</option>
<option>restaurant_food</option><option>legal</option>
<option>medical_health</option><option>real_estate</option>
<option>technology</option><option>fashion_retail</option>
<option>finance</option><option>education</option>
<option>construction</option><option>sports</option>
<option>entertainment</option><option>agriculture</option>
<option>industrial</option><option>consulting</option><option>other</option>
</select>
</div>
<div class="field"><label>Type</label>
<select x-model="f.site_type" style="width:120px">
<option value="">Any</option>
<option>corporate</option><option>ecommerce</option>
<option>blog</option><option>newspaper</option>
<option>landing_page</option><option>portfolio</option>
<option>directory</option><option>forum</option>
<option>informational</option><option>other</option>
</select>
</div>
</div>
<div style="display:flex;gap:6px;margin-bottom:10px;flex-wrap:wrap;align-items:center">
@@ -375,7 +410,7 @@ tr:hover td{background:rgba(255,255,255,.025)}
<th></th><th>Domain</th><th>Score</th><th>KD</th><th>AI</th>
<th>Niche</th><th>Type</th>
<th>Contact</th><th>CMS</th><th>SSL days</th>
<th>Country</th><th>Live</th>
<th>Country</th><th>Status</th>
</tr>
</thead>
<tbody>
@@ -410,7 +445,7 @@ tr:hover td{background:rgba(255,255,255,.025)}
<!-- Niche -->
<td>
<span x-show="row.niche" class="pill pni" x-text="row.niche"></span>
<span x-show="!row.niche" :class="prescreenStatusIcon(row.prescreen_status)" :title="row.prescreen_status||''" x-text="prescreenStatusIcon(row.prescreen_status)?'●':'—'"></span>
<span x-show="!row.niche" style="color:var(--border)">—</span>
</td>
<!-- Type -->
<td>
@@ -440,7 +475,11 @@ tr:hover td{background:rgba(255,255,255,.025)}
</td>
<td x-text="row.ssl_expiry_days??'—'"></td>
<td x-text="row.ip_country??'—'"></td>
<td><span class="pill" :class="row.is_live?'pg':'pp'" x-text="row.is_live?'Yes':'—'"></span></td>
<td style="text-align:center">
<span x-show="row.prescreen_status" :class="prescreenStatusIcon(row.prescreen_status)" :title="row.prescreen_status">●</span>
<span x-show="!row.prescreen_status && row.is_live" class="ps-live" title="live (from enricher)">●</span>
<span x-show="!row.prescreen_status && !row.is_live" style="color:var(--border)">—</span>
</td>
</tr>
</template>
</tbody>
@@ -509,7 +548,48 @@ tr:hover td{background:rgba(255,255,255,.025)}
</div>
</div>
<!-- ④ Lead Pipeline -->
<!-- ④ Validator -->
<div class="card" x-show="tab==='validator'">
<div class="ct">Bulk Domain Validator</div>
<div style="font-size:12px;color:var(--muted);margin-bottom:14px">
HTTP-checks the entire dataset to determine live/dead/parked/redirect status.
Extracts server type, IP, and load time. Skips already-validated domains.
Results appear as the Status column in Browse &amp; Filter.
</div>
<!-- Stats grid -->
<div class="esg" style="margin-bottom:12px">
<div class="esb"><div class="ev c1" x-text="(valSt.processed??0).toLocaleString()"></div><div class="el">Checked</div></div>
<div class="esb"><div class="ev ps-live" x-text="(valSt.live??0).toLocaleString()"></div><div class="el">Live</div></div>
<div class="esb"><div class="ev ps-dead" x-text="(valSt.dead??0).toLocaleString()"></div><div class="el">Dead</div></div>
<div class="esb"><div class="ev ps-parked" x-text="(valSt.parked??0).toLocaleString()"></div><div class="el">Parked</div></div>
<div class="esb"><div class="ev ps-redirect" x-text="(valSt.redirect??0).toLocaleString()"></div><div class="el">Redirect</div></div>
<div class="esb"><div class="ev c3" x-text="(valSt.rate??0).toFixed(1)"></div><div class="el">dom/sec</div></div>
</div>
<!-- Progress line -->
<div style="font-size:11px;color:var(--muted);margin-bottom:12px"
x-text="valSt.offset ? (valSt.offset??0).toLocaleString()+' rows scanned · '+(valSt.skipped??0).toLocaleString()+' already validated (skipped)' : 'Not started yet'">
</div>
<!-- Controls -->
<div style="display:flex;gap:8px;align-items:flex-end;flex-wrap:wrap">
<div class="field">
<label>TLD filter <span style="font-weight:400;color:var(--muted)">(leave empty for all domains)</span></label>
<input type="text" x-model="valTld" placeholder="es or com or ro" style="width:180px" :disabled="valSt.running">
</div>
<button class="btn bs" :disabled="valSt.running" @click="startValidator()">▶ Start Validator</button>
<button class="btn bd" :disabled="!valSt.running" @click="stopValidator()">⏹ Stop</button>
<span x-show="valSt.running" style="font-size:11px;color:var(--accent2);padding-bottom:6px">⚡ Running…</span>
</div>
<!-- Live rate progress bar (only while running) -->
<div x-show="valSt.running" style="margin-top:14px">
<div class="pw"><div class="pb" style="width:100%;animation:pulse 2s infinite"></div></div>
</div>
</div>
<!-- ⑤ Lead Pipeline -->
<div class="card" x-show="tab==='pipeline'">
<div style="display:flex;justify-content:flex-end;margin-bottom:10px;gap:8px">
<button class="btn bg sm" @click="loadPipeline()">↻ Refresh</button>
@@ -668,8 +748,10 @@ function app() {
aiSt: {pending:0,running:0,done:0,failed:0,total:0},
domains: [], selected: [], aiLang: 'ES',
loading: false, page: 1, searchTotal: 0,
f: {tld:'',keyword:'',min_score:0,cms:'',live_only:false,alpha_only:false,no_sld:false,kit_digital_only:false,exclude_assessed:false,limit:'100'},
f: {tld:'',keyword:'',min_score:0,cms:'',live_only:false,alpha_only:false,no_sld:false,kit_digital_only:false,exclude_assessed:false,limit:'100',prescreen_status:'',niche:'',site_type:''},
qst: {}, customDomains: '',
valSt: {running:false,processed:0,live:0,dead:0,parked:0,redirect:0,skipped:0,offset:0,rate:0},
valTld: '',
leadsQ: {quality:'', country:'', limit:'50'},
leadsData: [], leadsTotal: 0, leadsPage: 1, leadsLoading: false,
prescreening: false,
@@ -691,8 +773,9 @@ function app() {
} else {
this._lastAiDone = this.aiSt.done ?? 0;
}
if(this.tab==='enrich') this.loadQueue();
if(this.tab==='pipeline') this.loadPipeline();
if(this.tab==='enrich') this.loadQueue();
if(this.tab==='validator') this.loadValStatus();
if(this.tab==='pipeline') this.loadPipeline();
if(this.tab==='leads') this.loadLeads();
}, 3000);
},
@@ -727,10 +810,14 @@ function app() {
const data = await fetch('/api/domains?'+p).then(r=>r.json());
this.searchTotal = data.total ?? 0;
let rows = data.results;
if(this.f.min_score>0) rows = rows.filter(r=> r.score==null || r.score>=Number(this.f.min_score));
if(this.f.cms) rows = rows.filter(r=> r.cms===this.f.cms);
if(this.f.kit_digital_only) rows = rows.filter(r=> r.kit_digital);
if(this.f.exclude_assessed) rows = rows.filter(r=> !r.ai_lead_quality);
if(this.f.min_score>0) rows = rows.filter(r=> r.score==null || r.score>=Number(this.f.min_score));
if(this.f.cms) rows = rows.filter(r=> r.cms===this.f.cms);
if(this.f.kit_digital_only) rows = rows.filter(r=> r.kit_digital);
if(this.f.exclude_assessed) rows = rows.filter(r=> !r.ai_lead_quality);
if(this.f.prescreen_status==='none') rows = rows.filter(r=> !r.prescreen_status);
else if(this.f.prescreen_status) rows = rows.filter(r=> r.prescreen_status===this.f.prescreen_status);
if(this.f.niche) rows = rows.filter(r=> r.niche===this.f.niche);
if(this.f.site_type) rows = rows.filter(r=> r.site_type===this.f.site_type);
this.domains = rows;
} catch(e) {
this.domains = [];
@@ -740,7 +827,7 @@ function app() {
},
selectAll() { this.selected = this.domains.map(d=>d.domain); },
resetFilters() { this.f={tld:'',keyword:'',min_score:0,cms:'',live_only:false,alpha_only:false,no_sld:false,kit_digital_only:false,exclude_assessed:false,limit:'100'}; },
resetFilters() { this.f={tld:'',keyword:'',min_score:0,cms:'',live_only:false,alpha_only:false,no_sld:false,kit_digital_only:false,exclude_assessed:false,limit:'100',prescreen_status:'',niche:'',site_type:''}; },
async enqueueSelected() {
if(!this.selected.length) return;
@@ -852,6 +939,22 @@ function app() {
try { this.qst = await fetch('/api/enrich/status').then(r=>r.json()); } catch(e){}
},
async loadValStatus() {
try { this.valSt = await fetch('/api/validator/status').then(r=>r.json()); } catch(e){}
},
async startValidator() {
const p = new URLSearchParams();
if(this.valTld.trim()) p.set('tld', this.valTld.trim());
await fetch('/api/validator/start'+(p.toString()? '?'+p : ''), {method:'POST'});
this.notify('Validator started', 'success');
await this.loadValStatus();
},
async stopValidator() {
await fetch('/api/validator/stop', {method:'POST'});
this.notify('Validator stopped', 'info');
await this.loadValStatus();
},
async restartAiWorker() { await fetch('/api/ai/worker/restart',{method:'POST'}); this.notify('AI worker restarted','info'); await this.loadAiStatus(); },
copyEmail() {
const subj = this.modal.ai.email_subject ? `Subject: ${this.modal.ai.email_subject}\n\n` : '';