From 60c9b495ae715b97064c3bae59f5ca70e853b654 Mon Sep 17 00:00:00 2001 From: Malin Date: Mon, 13 Apr 2026 18:01:34 +0200 Subject: [PATCH] fix: AI worker crash-proof + GDPR/hosting/accessibility analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AI worker fixes (root cause of "nothing reaches Replicate"): - Worker task died silently — no exception handler around while loop - Added try/except around entire loop body with exc_info logging - Added watchdog task that restarts dead workers every 10 seconds - ensure_workers_alive() called on every /api/ai/assess/batch POST - _assess_one() is now a top-level function (not closure) — avoids subtle scoping bugs with async inner functions in while loops - /api/ai/debug endpoint: shows worker alive status, task exception, last 10 queue entries — browse to /api/ai/debug to diagnose - /api/ai/worker/restart endpoint + UI button - "Restart AI worker" button + "Debug AI queue" link in enrichment tab site_analyzer.py — new signals: - IP resolution + ip-api.com for ASN, org, ISP, host country - EU hosting detection (27 EU + EEA + adequacy countries) - GDPR: detects Cookiebot, OneTrust, CookiePro, Osano, Iubenda, Borlabs, CookieYes, Complianz, Usercentrics + text signals - Privacy policy and GDPR text presence - Accessibility: html lang missing, images without alt count, skip nav link, empty links, inputs without labels Gemini prompt additions: - Hosting section: IP, ASN, org/ISP, EU vs non-EU flag - GDPR section: cookie tool, notice, privacy policy - Accessibility section: all quick-scan results - New output fields: hosting_notes, gdpr_compliance, accessibility_issues[] Co-Authored-By: Claude Sonnet 4.6 --- app/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 150 bytes app/__pycache__/db.cpython-311.pyc | Bin 0 -> 28360 bytes app/__pycache__/enricher.cpython-311.pyc | Bin 0 -> 27916 bytes app/__pycache__/replicate_ai.cpython-311.pyc | Bin 0 -> 13062 bytes app/__pycache__/site_analyzer.cpython-311.pyc | Bin 0 -> 16736 bytes app/enricher.py | 93 +++-- app/main.py | 45 ++- app/replicate_ai.py | 137 ++++---- app/site_analyzer.py | 320 ++++++++++++------ app/static/index.html | 19 +- 10 files changed, 409 insertions(+), 205 deletions(-) create mode 100644 app/__pycache__/__init__.cpython-311.pyc create mode 100644 app/__pycache__/db.cpython-311.pyc create mode 100644 app/__pycache__/enricher.cpython-311.pyc create mode 100644 app/__pycache__/replicate_ai.cpython-311.pyc create mode 100644 app/__pycache__/site_analyzer.cpython-311.pyc diff --git a/app/__pycache__/__init__.cpython-311.pyc b/app/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..122516ccefb5c13d66efdee65bc030c9bb588a35 GIT binary patch literal 150 zcmZ3^%ge<81n+*|$pq1lK?DpiLK&agfQ;!3DGb33nv8xc8H$*I{LdiCUz++E`MIh3 zxrsTMdHTr)DVcfc`Y!po?)fSDi3J7v@$s2?nI-Y@dIgogIBatBQ%ZAE?TT1|IzeU^ U^8<+w%#4hT9~fXn5i?K>0Hf(4=l}o! literal 0 HcmV?d00001 diff --git a/app/__pycache__/db.cpython-311.pyc b/app/__pycache__/db.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..84f93ff473ba1cee890a04fa5d832beb602327cb GIT binary patch literal 28360 zcmeHwX>b%*nqX#~SyfqwRJxFaR7e7;Fgg%7h(qWAfh@2PwvmtOLRmn`QVB9k#?+MD z>NYbjSb8ioIH-2xu*Ti(#^LTA_Q%Gue$~35Z|rs}1=kDCdqH~-Mg4bt$z0~NOc*=vE9B(ZU#sgv6xK$9w%h5+D@9IBdx+7jXWjO2)pGq*G39C2hkxY>y}xZ@FfAjAhJPE7|w;STP&|CWKG ztbL;cy<-EOvEIGI1D>Hl&&Z)M&%nz=N5+nLWb6RvOGo2rvwGl?-#lXjFOPW+j}ATG zJ9^A>aNyWF@-Zm*rh;zJ^cfNy+?<~JYDiAWZq* z^iKu3q9Fkx@Yd-df6mAG&j}ecIrPM|K=LEMo$?F5>9;Z%+AIW5L-MnsFn>-F&krRyCj{j}SiL(+b-Brz6&~=|57#WirUD+#-jtsqUbbxf_k)y-I=^jS99w|_^QCd$2LR>I(vNhd} z_<%p0D?=8o4Te`B9GnicwT5QSwtBtU7Kv2p8>SHNOp z)-$o}DaeyQRz0~{#ygm@`odG3Z^A!uDv)A*;Tc$RZe}on4>KiC+0NoiqfTGy42WAg&ny-EflY(sbkco=gv}Z>5oxoX^2;ms~qO1 z;QT!Ms?HzKx$vi^Ro?+>I9CFWIzly5>nWwLU^uIj)5ILm@P}BR(IZ|<%Hj{s2xnk{ z4y4R5SVDn`aLRV%h2f#G0pH=?vHdA~;H|*KY&alH@WIpJl<7#{{(GB(FW5$IYFmW~Kw3(?Ax4 zIw!ht73}Pvncg?Ub^1@A?&OYloIaN_2SZ4&9Oo+`&@p&~GteLY1^(u#d(Ilk>G^e? z=xmpq?MoD6tSt?0lqVgoyN-rCj)srk6dh|M$C?CF;Qr3tutXUwl?254mUMc1WwNT~ z>g46gMfdwNS7s8f)dlbG@44%jDB2DoGa%`zSTHQ<;r09PzrXKry%)R~yzs`QHxkT> z{ClZPRfY#xq(@IKSLsqO<11vXG?uUls{WI;d)Mf1+3R|}hFfhky1mxk&E{Jhbm-nh zqkHp8aDTbRy0^>xAjR zToD!(lTpr>GsDEAVpd6+fH}>-)H$G`&M^%f{2>=iPc^v^ndvEH$V^Ym$ccuvE>X7@ zRjbzK(pA^v$wDP7+yYj0$P#J)nQi4aLyuWj?P41)qt*9M*clZXWBN?3z|uP$oP8gBV&*4M+U|`;Zr~haKg@wkt%$h z4ouGg^E(xU`Fv+<_rATYkL!;N3=i~;0eO3LWUOti*E2YJ==n5%F7P;*2O};0!29>_ z^#ns);4P1T3gbNIkzw0v&cn}~6}*vcduM}FxCVGs=qLD@X-`_3;Axu)1=^7d^1R*M zzQHpc40$^`I{cNeA`sReWdnGn>7zrVp3#BB!@YeO;X;-mKcYgH%b@bYSY&LO&%yW1 zH8pcKz_;;%lOPNLfk2oCX1Fbl>}h?qqwST~J6?Ov+tRwuv)=3V^d9j56;vRaK&VX! z^H5@+a2gigTofI^-(?b0{6a)f5n(}QwcA99kv3-IH#n$jZEroacl;I2GUkNANmRP7n(~Gy5XNTO{4zOv>9mA%p$WbX z{7=Fo+=tFIPu;JoojNdzQbdJ0xZ@fSUHc^0zT33q zIuPqkI$aCvFV4nhlZ~DQHrdz=ps97idCygK)ppsIsNefVujm?-T!RVMAjW6CY<-`- z!UCGBYH2N1=ZQJ-Z)pqVs{H;>4$8Fx2oOtI4kTGutRJWlOBpQne=<=P=XM`Ob7dEhi#njTRA;4=UhWwB%F6O}_Eb3kGa zB$xxq>bk4Z%h5#3Ua`7Qs_wg6J$$EnSganAsz+|0k*Z&av6zJQqV)oMiG`Qyx)^(p zame{t{~voqW{(K3_(e2v13tql)(EE(k zH%{x|F8wzv>IS#zzu9O+pKV6;*`){fw?@N0i+%!P&R)2%zXQ)h9zZG1M!i52GA2t1 zWn%*eM}^L&5O>bPrd zxMOXIhySp8vuNESS+^t%Mem0Y4V5zSf$%IJ%Bw~_e0;bL05FG$n7+b_YErU?9DPj> z0&Cp}Oan%F`e3qCl99|VUOnFkvG7Qa@eKezqxepAlPQ9g=OLL_<~M`?2k;0;vdmN8 zm94y2(R{CL<&xRJnt)3nz-S`jCloD2;jBlp{yzY|Yk$}AuJbC)GZ*dfIN)(A3&Yzu zdd|eHjM-z3nDd05)19}>nL}<)uPo*-D9-SaUWI2wyeP#_s$%QbQBgBz0