debug: add parser logging + enable url2green API

- parser.js: log found files, top-level JSON keys, and extracted
  metric values to stdout (visible via docker logs speedboard)
- runner.js: add --sustainable.useGreenWebHostingAPI flag

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

parser.js

@@ -2,18 +2,20 @@ import { readdir, readFile } from 'fs/promises';
 import { join } from 'path';
 
 /**
- * Recursively find all *.pageSummary.json files under outputFolder.
+ * Recursively find all *.pageSummary.json files.
- * sitespeed.io v39 writes them into:
+ * sitespeed.io v39 writes them to:
- *   pages/<hostname>/<urlpath>/data/<plugin>.pageSummary.json
+ *   <outputFolder>/pages/<hostname>/data/<plugin>.pageSummary.json
  */
 async function findPageSummaries(outputFolder) {
   const summaries = {};
+  const errors = [];
 
   async function walk(dir) {
     let entries;
     try {
       entries = await readdir(dir, { withFileTypes: true });
-    } catch {
+    } catch (err) {
+      errors.push(`readdir failed on ${dir}: ${err.message}`);
       return;
     }
     for (const e of entries) {
@@ -24,11 +26,24 @@ async function findPageSummaries(outputFolder) {
         const plugin = e.name.replace('.pageSummary.json', '');
         if (!summaries[plugin]) summaries[plugin] = [];
         summaries[plugin].push(full);
+        console.log(`[parser] found: ${full}`);
       }
     }
   }
 
   await walk(outputFolder);
+
+  if (errors.length > 0) {
+    console.warn('[parser] walk errors:', errors.join('; '));
+  }
+
+  const keys = Object.keys(summaries);
+  if (keys.length === 0) {
+    console.warn(`[parser] no pageSummary JSON files found under ${outputFolder}`);
+  } else {
+    console.log(`[parser] plugins found: ${keys.join(', ')}`);
+  }
+
   return summaries;
 }
 
@@ -37,7 +52,6 @@ async function readJson(filePath) {
   return JSON.parse(raw);
 }
 
-// Safe deep-get: safe(obj, 'a', 'b', 'c') === obj?.a?.b?.c
 function safe(obj, ...path) {
   let cur = obj;
   for (const key of path) {
@@ -47,7 +61,6 @@ function safe(obj, ...path) {
   return cur ?? null;
 }
 
-// Get the median from a stats object { median, min, max, ... }
 function med(obj) {
   return safe(obj, 'median');
 }
@@ -56,23 +69,14 @@ export async function parseResults(outputFolder, _url) {
   const summaries = await findPageSummaries(outputFolder);
   const metrics = {};
 
-  // ─── browsertime.pageSummary ───────────────────────────────────────────────
+  // ─── browsertime.pageSummary ──────────────────────────────────────────────
-  // All aggregated values live under `statistics.*`
-  // Key layout (from browsertimeAggregator.js):
-  //   statistics.timings.largestContentfulPaint  — LCP renderTime
-  //   statistics.timings.fullyLoaded
-  //   statistics.pageTimings.*                   — pageLoadTime, backEndTime, etc.
-  //   statistics.paintTiming.*                   — 'first-contentful-paint', etc.
-  //   statistics.pageinfo.cumulativeLayoutShift  — CLS
-  //   statistics.visualMetrics.*                 — SpeedIndex, FirstVisualChange, etc.
-  //   statistics.cpu.longTasks.*                 — totalBlockingTime, tasks, etc.
   const btFiles = summaries['browsertime'] || [];
   if (btFiles.length > 0) {
     const bt = await readJson(btFiles[0]);
     const s = safe(bt, 'statistics');
+    console.log('[parser] browsertime statistics keys:', s ? Object.keys(s).join(', ') : 'NONE');
 
     if (s) {
-      // Core Web Vitals
       metrics.lcp  = med(safe(s, 'timings', 'largestContentfulPaint'));
       metrics.fcp  = med(safe(s, 'paintTiming', 'first-contentful-paint'));
       metrics.cls  = med(safe(s, 'pageinfo', 'cumulativeLayoutShift'));
@@ -80,7 +84,6 @@ export async function parseResults(outputFolder, _url) {
       metrics.ttfb = med(safe(s, 'pageTimings', 'backEndTime'));
       metrics.max_potential_fid = med(safe(s, 'cpu', 'longTasks', 'maxPotentialFid'));
 
-      // Navigation timings (pageTimings keys come from browsertime's pageTimings object)
       metrics.page_load_time     = med(safe(s, 'pageTimings', 'pageLoadTime'));
       metrics.fully_loaded       = med(safe(s, 'timings', 'fullyLoaded'));
       metrics.dom_content_loaded = med(safe(s, 'pageTimings', 'domContentLoadedTime'));
@@ -89,25 +92,27 @@ export async function parseResults(outputFolder, _url) {
       metrics.back_end_time      = med(safe(s, 'pageTimings', 'backEndTime'));
       metrics.time_to_first_byte = metrics.ttfb;
 
-      // Visual metrics (from sitespeed-scroll-server / ffmpeg video analysis)
       metrics.speed_index            = med(safe(s, 'visualMetrics', 'SpeedIndex'));
       metrics.first_visual_change    = med(safe(s, 'visualMetrics', 'FirstVisualChange'));
       metrics.last_visual_change     = med(safe(s, 'visualMetrics', 'LastVisualChange'));
       metrics.visual_complete_85     = med(safe(s, 'visualMetrics', 'VisualComplete85'));
       metrics.perceptual_speed_index = med(safe(s, 'visualMetrics', 'PerceptualSpeedIndex'));
 
-      // CPU / Long Tasks
       metrics.long_tasks_count    = med(safe(s, 'cpu', 'longTasks', 'tasks'));
       metrics.long_tasks_duration = med(safe(s, 'cpu', 'longTasks', 'totalDuration'));
+
+      console.log('[parser] lcp=%s fcp=%s tbt=%s cls=%s ttfb=%s si=%s',
+        metrics.lcp, metrics.fcp, metrics.tbt, metrics.cls, metrics.ttfb, metrics.speed_index);
     }
+  } else {
+    console.warn('[parser] no browsertime.pageSummary.json found');
   }
 
-  // ─── axe.pageSummary ──────────────────────────────────────────────────────
+  // ─── axe.pageSummary ─────────────────────────────────────────────────────
-  // Produced by AxeAggregator.summarizeStats():
-  //   { violations: { critical: {median,…}, serious, moderate, minor }, … }
   const axeFiles = summaries['axe'] || [];
   if (axeFiles.length > 0) {
     const axe = await readJson(axeFiles[0]);
+    console.log('[parser] axe top-level keys:', Object.keys(axe).join(', '));
     metrics.axe_critical = med(safe(axe, 'violations', 'critical')) ?? 0;
     metrics.axe_serious  = med(safe(axe, 'violations', 'serious'))  ?? 0;
     metrics.axe_moderate = med(safe(axe, 'violations', 'moderate')) ?? 0;
@@ -115,64 +120,63 @@ export async function parseResults(outputFolder, _url) {
   }
 
   // ─── coach.pageSummary ────────────────────────────────────────────────────
-  // Coach sends the median-run's full coach result: { advice: { score, performance, … } }
   const coachFiles = summaries['coach'] || [];
   if (coachFiles.length > 0) {
     const coach = await readJson(coachFiles[0]);
+    console.log('[parser] coach top-level keys:', Object.keys(coach).join(', '));
     metrics.score_overall       = safe(coach, 'advice', 'score');
     metrics.score_performance   = safe(coach, 'advice', 'performance', 'score');
     metrics.score_accessibility = safe(coach, 'advice', 'accessibility', 'score');
     metrics.score_bestpractice  = safe(coach, 'advice', 'bestpractice', 'score');
     metrics.score_privacy       = safe(coach, 'advice', 'privacy', 'score');
+    console.log('[parser] coach scores: overall=%s perf=%s',
+      metrics.score_overall, metrics.score_performance);
   }
 
   // ─── pagexray.pageSummary ─────────────────────────────────────────────────
-  // Sent as pageSummary[0] with `.statistics` added by the aggregator.
-  // Use statistics.*.median when available; fall back to direct property.
   const xrayFiles = summaries['pagexray'] || [];
   if (xrayFiles.length > 0) {
     const xray = await readJson(xrayFiles[0]);
     const xs = safe(xray, 'statistics');
+    console.log('[parser] pagexray top-level keys:', Object.keys(xray).join(', '));
+    if (xs) console.log('[parser] pagexray.statistics keys:', Object.keys(xs).join(', '));
 
     function xv(statPath, directPath) {
-      // Try statistics.X.median first, then direct property
+      const fromStats = xs ? med(safe(xs, ...statPath)) : null;
-      const fromStats = med(safe(xs, ...statPath));
       if (fromStats !== null) return fromStats;
       return safe(xray, ...directPath);
     }
 
     metrics.transfer_total = xv(['transferSize'], ['transferSize']);
     metrics.requests_total = xv(['requests'],     ['requests']);
-
     metrics.transfer_html  = xv(['contentTypes', 'html',       'transferSize'], ['contentTypes', 'html',       'transferSize']);
     metrics.transfer_js    = xv(['contentTypes', 'javascript', 'transferSize'], ['contentTypes', 'javascript', 'transferSize']);
     metrics.transfer_css   = xv(['contentTypes', 'css',        'transferSize'], ['contentTypes', 'css',        'transferSize']);
     metrics.transfer_image = xv(['contentTypes', 'image',      'transferSize'], ['contentTypes', 'image',      'transferSize']);
     metrics.transfer_font  = xv(['contentTypes', 'font',       'transferSize'], ['contentTypes', 'font',       'transferSize']);
-
     metrics.requests_js    = xv(['contentTypes', 'javascript', 'requests'], ['contentTypes', 'javascript', 'requests']);
     metrics.requests_css   = xv(['contentTypes', 'css',        'requests'], ['contentTypes', 'css',        'requests']);
     metrics.requests_image = xv(['contentTypes', 'image',      'requests'], ['contentTypes', 'image',      'requests']);
     metrics.requests_font  = xv(['contentTypes', 'font',       'requests'], ['contentTypes', 'font',       'requests']);
-
     metrics.third_party_transfer = xv(['thirdParty', 'transferSize'], ['thirdParty', 'transferSize']);
     metrics.third_party_requests = xv(['thirdParty', 'requests'],     ['thirdParty', 'requests']);
+
+    console.log('[parser] transfer_total=%s requests_total=%s', metrics.transfer_total, metrics.requests_total);
   }
 
   // ─── sustainable.pageSummary ──────────────────────────────────────────────
-  // Aggregator stores per-URL stats:
-  //   { co2PerPageView: {median,…}, co2FirstParty: {median,…}, co2ThirdParty: {median,…} }
   const sustainFiles = summaries['sustainable'] || [];
   if (sustainFiles.length > 0) {
     const sust = await readJson(sustainFiles[0]);
+    console.log('[parser] sustainable top-level keys:', Object.keys(sust).join(', '));
     metrics.co2_per_page_view = med(safe(sust, 'co2PerPageView'));
-    metrics.co2_total         = med(safe(sust, 'co2PerPageView')); // totalCO2 not in per-URL summary
+    metrics.co2_total         = med(safe(sust, 'co2PerPageView'));
     metrics.co2_first_party   = med(safe(sust, 'co2FirstParty'));
     metrics.co2_third_party   = med(safe(sust, 'co2ThirdParty'));
   }
 
-  // Strip nulls so we don't overwrite good DB values with NULL
+  const extracted = Object.entries(metrics).filter(([, v]) => v !== null && v !== undefined);
-  return Object.fromEntries(
+  console.log(`[parser] extracted ${extracted.length} metrics`);
-    Object.entries(metrics).filter(([, v]) => v !== null && v !== undefined)
+
-  );
+  return Object.fromEntries(extracted);
 }

runner.js

@@ -19,6 +19,7 @@ export function runTest(job, onLine) {
       '--outputFolder', outputFolder,
       '--json',
       '--sustainable.enable',
+      '--sustainable.useGreenWebHostingAPI',
       '--axe.enable',
       '--coach',
     ];