Malin/speedboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug: add parser logging + enable url2green API

Browse Source 
- parser.js: log found files, top-level JSON keys, and extracted
  metric values to stdout (visible via docker logs speedboard)
- runner.js: add --sustainable.useGreenWebHostingAPI flag

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Malin
2026-04-07 10:44:14 +02:00
parent 500f69efcc
commit dc8fed337f
2 changed files with 57 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
parser.js
View File

@@ -2,18 +2,20 @@ import { readdir, readFile } from 'fs/promises';
import { join } from 'path'; import { join } from 'path';
/** /**
* Recursively find all *.pageSummary.json files under outputFolder. * Recursively find all *.pageSummary.json files.
* sitespeed.io v39 writes them into: * sitespeed.io v39 writes them to:
* pages/<hostname>/<urlpath>/data/<plugin>.pageSummary.json * <outputFolder>/pages/<hostname>/data/<plugin>.pageSummary.json
*/ */
async function findPageSummaries(outputFolder) { async function findPageSummaries(outputFolder) {
const summaries = {}; const summaries = {};
const errors = [];
async function walk(dir) { async function walk(dir) {
let entries; let entries;
try { try {
entries = await readdir(dir, { withFileTypes: true }); entries = await readdir(dir, { withFileTypes: true });
} catch { } catch (err) {
errors.push(`readdir failed on ${dir}: ${err.message}`);
return; return;
} }
for (const e of entries) { for (const e of entries) {
@@ -24,11 +26,24 @@ async function findPageSummaries(outputFolder) {
const plugin = e.name.replace('.pageSummary.json', ''); const plugin = e.name.replace('.pageSummary.json', '');
if (!summaries[plugin]) summaries[plugin] = []; if (!summaries[plugin]) summaries[plugin] = [];
summaries[plugin].push(full); summaries[plugin].push(full);
console.log(`[parser] found: ${full}`);
} }
} }
} }
await walk(outputFolder); await walk(outputFolder);
if (errors.length > 0) {
console.warn('[parser] walk errors:', errors.join('; '));
}
const keys = Object.keys(summaries);
if (keys.length === 0) {
console.warn(`[parser] no pageSummary JSON files found under ${outputFolder}`);
} else {
console.log(`[parser] plugins found: ${keys.join(', ')}`);
}
return summaries; return summaries;
} }
@@ -37,7 +52,6 @@ async function readJson(filePath) {
return JSON.parse(raw); return JSON.parse(raw);
} }
// Safe deep-get: safe(obj, 'a', 'b', 'c') === obj?.a?.b?.c
function safe(obj, ...path) { function safe(obj, ...path) {
let cur = obj; let cur = obj;
for (const key of path) { for (const key of path) {
@@ -47,7 +61,6 @@ function safe(obj, ...path) {
return cur ?? null; return cur ?? null;
} }
// Get the median from a stats object { median, min, max, ... }
function med(obj) { function med(obj) {
return safe(obj, 'median'); return safe(obj, 'median');
} }
@@ -56,23 +69,14 @@ export async function parseResults(outputFolder, _url) {
const summaries = await findPageSummaries(outputFolder); const summaries = await findPageSummaries(outputFolder);
const metrics = {}; const metrics = {};
// ─── browsertime.pageSummary ─────────────────────────────────────────────── // ─── browsertime.pageSummary ──────────────────────────────────────────────
// All aggregated values live under `statistics.*`
// Key layout (from browsertimeAggregator.js):
// statistics.timings.largestContentfulPaint — LCP renderTime
// statistics.timings.fullyLoaded
// statistics.pageTimings.* — pageLoadTime, backEndTime, etc.
// statistics.paintTiming.* — 'first-contentful-paint', etc.
// statistics.pageinfo.cumulativeLayoutShift — CLS
// statistics.visualMetrics.* — SpeedIndex, FirstVisualChange, etc.
// statistics.cpu.longTasks.* — totalBlockingTime, tasks, etc.
const btFiles = summaries['browsertime'] || []; const btFiles = summaries['browsertime'] || [];
if (btFiles.length > 0) { if (btFiles.length > 0) {
const bt = await readJson(btFiles[0]); const bt = await readJson(btFiles[0]);
const s = safe(bt, 'statistics'); const s = safe(bt, 'statistics');
console.log('[parser] browsertime statistics keys:', s ? Object.keys(s).join(', ') : 'NONE');
if (s) { if (s) {
// Core Web Vitals
metrics.lcp = med(safe(s, 'timings', 'largestContentfulPaint')); metrics.lcp = med(safe(s, 'timings', 'largestContentfulPaint'));
metrics.fcp = med(safe(s, 'paintTiming', 'first-contentful-paint')); metrics.fcp = med(safe(s, 'paintTiming', 'first-contentful-paint'));
metrics.cls = med(safe(s, 'pageinfo', 'cumulativeLayoutShift')); metrics.cls = med(safe(s, 'pageinfo', 'cumulativeLayoutShift'));
@@ -80,34 +84,35 @@ export async function parseResults(outputFolder, _url) {
metrics.ttfb = med(safe(s, 'pageTimings', 'backEndTime')); metrics.ttfb = med(safe(s, 'pageTimings', 'backEndTime'));
metrics.max_potential_fid = med(safe(s, 'cpu', 'longTasks', 'maxPotentialFid')); metrics.max_potential_fid = med(safe(s, 'cpu', 'longTasks', 'maxPotentialFid'));
// Navigation timings (pageTimings keys come from browsertime's pageTimings object) metrics.page_load_time = med(safe(s, 'pageTimings', 'pageLoadTime'));
metrics.page_load_time = med(safe(s, 'pageTimings', 'pageLoadTime')); metrics.fully_loaded = med(safe(s, 'timings', 'fullyLoaded'));
metrics.fully_loaded = med(safe(s, 'timings', 'fullyLoaded'));
metrics.dom_content_loaded = med(safe(s, 'pageTimings', 'domContentLoadedTime')); metrics.dom_content_loaded = med(safe(s, 'pageTimings', 'domContentLoadedTime'));
metrics.dom_interactive = med(safe(s, 'pageTimings', 'domInteractiveTime')); metrics.dom_interactive = med(safe(s, 'pageTimings', 'domInteractiveTime'));
metrics.front_end_time = med(safe(s, 'pageTimings', 'frontEndTime')); metrics.front_end_time = med(safe(s, 'pageTimings', 'frontEndTime'));
metrics.back_end_time = med(safe(s, 'pageTimings', 'backEndTime')); metrics.back_end_time = med(safe(s, 'pageTimings', 'backEndTime'));
metrics.time_to_first_byte = metrics.ttfb; metrics.time_to_first_byte = metrics.ttfb;
// Visual metrics (from sitespeed-scroll-server / ffmpeg video analysis) metrics.speed_index = med(safe(s, 'visualMetrics', 'SpeedIndex'));
metrics.speed_index = med(safe(s, 'visualMetrics', 'SpeedIndex')); metrics.first_visual_change = med(safe(s, 'visualMetrics', 'FirstVisualChange'));
metrics.first_visual_change = med(safe(s, 'visualMetrics', 'FirstVisualChange')); metrics.last_visual_change = med(safe(s, 'visualMetrics', 'LastVisualChange'));
metrics.last_visual_change = med(safe(s, 'visualMetrics', 'LastVisualChange')); metrics.visual_complete_85 = med(safe(s, 'visualMetrics', 'VisualComplete85'));
metrics.visual_complete_85 = med(safe(s, 'visualMetrics', 'VisualComplete85'));
metrics.perceptual_speed_index = med(safe(s, 'visualMetrics', 'PerceptualSpeedIndex')); metrics.perceptual_speed_index = med(safe(s, 'visualMetrics', 'PerceptualSpeedIndex'));
// CPU / Long Tasks
metrics.long_tasks_count = med(safe(s, 'cpu', 'longTasks', 'tasks')); metrics.long_tasks_count = med(safe(s, 'cpu', 'longTasks', 'tasks'));
metrics.long_tasks_duration = med(safe(s, 'cpu', 'longTasks', 'totalDuration')); metrics.long_tasks_duration = med(safe(s, 'cpu', 'longTasks', 'totalDuration'));
console.log('[parser] lcp=%s fcp=%s tbt=%s cls=%s ttfb=%s si=%s',
metrics.lcp, metrics.fcp, metrics.tbt, metrics.cls, metrics.ttfb, metrics.speed_index);
} }
} else {
console.warn('[parser] no browsertime.pageSummary.json found');
} }
// ─── axe.pageSummary ────────────────────────────────────────────────────── // ─── axe.pageSummary ─────────────────────────────────────────────────────
// Produced by AxeAggregator.summarizeStats():
// { violations: { critical: {median,…}, serious, moderate, minor }, … }
const axeFiles = summaries['axe'] || []; const axeFiles = summaries['axe'] || [];
if (axeFiles.length > 0) { if (axeFiles.length > 0) {
const axe = await readJson(axeFiles[0]); const axe = await readJson(axeFiles[0]);
console.log('[parser] axe top-level keys:', Object.keys(axe).join(', '));
metrics.axe_critical = med(safe(axe, 'violations', 'critical')) ?? 0; metrics.axe_critical = med(safe(axe, 'violations', 'critical')) ?? 0;
metrics.axe_serious = med(safe(axe, 'violations', 'serious')) ?? 0; metrics.axe_serious = med(safe(axe, 'violations', 'serious')) ?? 0;
metrics.axe_moderate = med(safe(axe, 'violations', 'moderate')) ?? 0; metrics.axe_moderate = med(safe(axe, 'violations', 'moderate')) ?? 0;
@@ -115,64 +120,63 @@ export async function parseResults(outputFolder, _url) {
} }
// ─── coach.pageSummary ──────────────────────────────────────────────────── // ─── coach.pageSummary ────────────────────────────────────────────────────
// Coach sends the median-run's full coach result: { advice: { score, performance, … } }
const coachFiles = summaries['coach'] || []; const coachFiles = summaries['coach'] || [];
if (coachFiles.length > 0) { if (coachFiles.length > 0) {
const coach = await readJson(coachFiles[0]); const coach = await readJson(coachFiles[0]);
metrics.score_overall = safe(coach, 'advice', 'score'); console.log('[parser] coach top-level keys:', Object.keys(coach).join(', '));
metrics.score_performance = safe(coach, 'advice', 'performance', 'score'); metrics.score_overall = safe(coach, 'advice', 'score');
metrics.score_performance = safe(coach, 'advice', 'performance', 'score');
metrics.score_accessibility = safe(coach, 'advice', 'accessibility', 'score'); metrics.score_accessibility = safe(coach, 'advice', 'accessibility', 'score');
metrics.score_bestpractice = safe(coach, 'advice', 'bestpractice', 'score'); metrics.score_bestpractice = safe(coach, 'advice', 'bestpractice', 'score');
metrics.score_privacy = safe(coach, 'advice', 'privacy', 'score'); metrics.score_privacy = safe(coach, 'advice', 'privacy', 'score');
console.log('[parser] coach scores: overall=%s perf=%s',
metrics.score_overall, metrics.score_performance);
} }
// ─── pagexray.pageSummary ───────────────────────────────────────────────── // ─── pagexray.pageSummary ─────────────────────────────────────────────────
// Sent as pageSummary[0] with `.statistics` added by the aggregator.
// Use statistics.*.median when available; fall back to direct property.
const xrayFiles = summaries['pagexray'] || []; const xrayFiles = summaries['pagexray'] || [];
if (xrayFiles.length > 0) { if (xrayFiles.length > 0) {
const xray = await readJson(xrayFiles[0]); const xray = await readJson(xrayFiles[0]);
const xs = safe(xray, 'statistics'); const xs = safe(xray, 'statistics');
console.log('[parser] pagexray top-level keys:', Object.keys(xray).join(', '));
if (xs) console.log('[parser] pagexray.statistics keys:', Object.keys(xs).join(', '));
function xv(statPath, directPath) { function xv(statPath, directPath) {
// Try statistics.X.median first, then direct property const fromStats = xs ? med(safe(xs, ...statPath)) : null;
const fromStats = med(safe(xs, ...statPath));
if (fromStats !== null) return fromStats; if (fromStats !== null) return fromStats;
return safe(xray, ...directPath); return safe(xray, ...directPath);
} }
metrics.transfer_total = xv(['transferSize'], ['transferSize']); metrics.transfer_total = xv(['transferSize'], ['transferSize']);
metrics.requests_total = xv(['requests'], ['requests']); metrics.requests_total = xv(['requests'], ['requests']);
metrics.transfer_html = xv(['contentTypes', 'html', 'transferSize'], ['contentTypes', 'html', 'transferSize']); metrics.transfer_html = xv(['contentTypes', 'html', 'transferSize'], ['contentTypes', 'html', 'transferSize']);
metrics.transfer_js = xv(['contentTypes', 'javascript', 'transferSize'], ['contentTypes', 'javascript', 'transferSize']); metrics.transfer_js = xv(['contentTypes', 'javascript', 'transferSize'], ['contentTypes', 'javascript', 'transferSize']);
metrics.transfer_css = xv(['contentTypes', 'css', 'transferSize'], ['contentTypes', 'css', 'transferSize']); metrics.transfer_css = xv(['contentTypes', 'css', 'transferSize'], ['contentTypes', 'css', 'transferSize']);
metrics.transfer_image = xv(['contentTypes', 'image', 'transferSize'], ['contentTypes', 'image', 'transferSize']); metrics.transfer_image = xv(['contentTypes', 'image', 'transferSize'], ['contentTypes', 'image', 'transferSize']);
metrics.transfer_font = xv(['contentTypes', 'font', 'transferSize'], ['contentTypes', 'font', 'transferSize']); metrics.transfer_font = xv(['contentTypes', 'font', 'transferSize'], ['contentTypes', 'font', 'transferSize']);
metrics.requests_js = xv(['contentTypes', 'javascript', 'requests'], ['contentTypes', 'javascript', 'requests']); metrics.requests_js = xv(['contentTypes', 'javascript', 'requests'], ['contentTypes', 'javascript', 'requests']);
metrics.requests_css = xv(['contentTypes', 'css', 'requests'], ['contentTypes', 'css', 'requests']); metrics.requests_css = xv(['contentTypes', 'css', 'requests'], ['contentTypes', 'css', 'requests']);
metrics.requests_image = xv(['contentTypes', 'image', 'requests'], ['contentTypes', 'image', 'requests']); metrics.requests_image = xv(['contentTypes', 'image', 'requests'], ['contentTypes', 'image', 'requests']);
metrics.requests_font = xv(['contentTypes', 'font', 'requests'], ['contentTypes', 'font', 'requests']); metrics.requests_font = xv(['contentTypes', 'font', 'requests'], ['contentTypes', 'font', 'requests']);
metrics.third_party_transfer = xv(['thirdParty', 'transferSize'], ['thirdParty', 'transferSize']);
metrics.third_party_requests = xv(['thirdParty', 'requests'], ['thirdParty', 'requests']);
metrics.third_party_transfer = xv(['thirdParty', 'transferSize'], ['thirdParty', 'transferSize']); console.log('[parser] transfer_total=%s requests_total=%s', metrics.transfer_total, metrics.requests_total);
metrics.third_party_requests = xv(['thirdParty', 'requests'], ['thirdParty', 'requests']);
} }
// ─── sustainable.pageSummary ────────────────────────────────────────────── // ─── sustainable.pageSummary ──────────────────────────────────────────────
// Aggregator stores per-URL stats:
// { co2PerPageView: {median,…}, co2FirstParty: {median,…}, co2ThirdParty: {median,…} }
const sustainFiles = summaries['sustainable'] || []; const sustainFiles = summaries['sustainable'] || [];
if (sustainFiles.length > 0) { if (sustainFiles.length > 0) {
const sust = await readJson(sustainFiles[0]); const sust = await readJson(sustainFiles[0]);
console.log('[parser] sustainable top-level keys:', Object.keys(sust).join(', '));
metrics.co2_per_page_view = med(safe(sust, 'co2PerPageView')); metrics.co2_per_page_view = med(safe(sust, 'co2PerPageView'));
metrics.co2_total = med(safe(sust, 'co2PerPageView')); // totalCO2 not in per-URL summary metrics.co2_total = med(safe(sust, 'co2PerPageView'));
metrics.co2_first_party = med(safe(sust, 'co2FirstParty')); metrics.co2_first_party = med(safe(sust, 'co2FirstParty'));
metrics.co2_third_party = med(safe(sust, 'co2ThirdParty')); metrics.co2_third_party = med(safe(sust, 'co2ThirdParty'));
} }
// Strip nulls so we don't overwrite good DB values with NULL const extracted = Object.entries(metrics).filter(([, v]) => v !== null && v !== undefined);
return Object.fromEntries( console.log(`[parser] extracted ${extracted.length} metrics`);
Object.entries(metrics).filter(([, v]) => v !== null && v !== undefined)
); return Object.fromEntries(extracted);
} }

1
runner.js
View File

@@ -19,6 +19,7 @@ export function runTest(job, onLine) {
'--outputFolder', outputFolder, '--outputFolder', outputFolder,
'--json', '--json',
'--sustainable.enable', '--sustainable.enable',
'--sustainable.useGreenWebHostingAPI',
'--axe.enable', '--axe.enable',
'--coach', '--coach',
]; ];