diff --git a/parser.js b/parser.js index 63dbff0..a0bfcfc 100644 --- a/parser.js +++ b/parser.js @@ -2,18 +2,20 @@ import { readdir, readFile } from 'fs/promises'; import { join } from 'path'; /** - * Recursively find all *.pageSummary.json files under outputFolder. - * sitespeed.io v39 writes them into: - * pages///data/.pageSummary.json + * Recursively find all *.pageSummary.json files. + * sitespeed.io v39 writes them to: + * /pages//data/.pageSummary.json */ async function findPageSummaries(outputFolder) { const summaries = {}; + const errors = []; async function walk(dir) { let entries; try { entries = await readdir(dir, { withFileTypes: true }); - } catch { + } catch (err) { + errors.push(`readdir failed on ${dir}: ${err.message}`); return; } for (const e of entries) { @@ -24,11 +26,24 @@ async function findPageSummaries(outputFolder) { const plugin = e.name.replace('.pageSummary.json', ''); if (!summaries[plugin]) summaries[plugin] = []; summaries[plugin].push(full); + console.log(`[parser] found: ${full}`); } } } await walk(outputFolder); + + if (errors.length > 0) { + console.warn('[parser] walk errors:', errors.join('; ')); + } + + const keys = Object.keys(summaries); + if (keys.length === 0) { + console.warn(`[parser] no pageSummary JSON files found under ${outputFolder}`); + } else { + console.log(`[parser] plugins found: ${keys.join(', ')}`); + } + return summaries; } @@ -37,7 +52,6 @@ async function readJson(filePath) { return JSON.parse(raw); } -// Safe deep-get: safe(obj, 'a', 'b', 'c') === obj?.a?.b?.c function safe(obj, ...path) { let cur = obj; for (const key of path) { @@ -47,7 +61,6 @@ function safe(obj, ...path) { return cur ?? null; } -// Get the median from a stats object { median, min, max, ... } function med(obj) { return safe(obj, 'median'); } @@ -56,23 +69,14 @@ export async function parseResults(outputFolder, _url) { const summaries = await findPageSummaries(outputFolder); const metrics = {}; - // ─── browsertime.pageSummary ─────────────────────────────────────────────── - // All aggregated values live under `statistics.*` - // Key layout (from browsertimeAggregator.js): - // statistics.timings.largestContentfulPaint — LCP renderTime - // statistics.timings.fullyLoaded - // statistics.pageTimings.* — pageLoadTime, backEndTime, etc. - // statistics.paintTiming.* — 'first-contentful-paint', etc. - // statistics.pageinfo.cumulativeLayoutShift — CLS - // statistics.visualMetrics.* — SpeedIndex, FirstVisualChange, etc. - // statistics.cpu.longTasks.* — totalBlockingTime, tasks, etc. + // ─── browsertime.pageSummary ────────────────────────────────────────────── const btFiles = summaries['browsertime'] || []; if (btFiles.length > 0) { const bt = await readJson(btFiles[0]); const s = safe(bt, 'statistics'); + console.log('[parser] browsertime statistics keys:', s ? Object.keys(s).join(', ') : 'NONE'); if (s) { - // Core Web Vitals metrics.lcp = med(safe(s, 'timings', 'largestContentfulPaint')); metrics.fcp = med(safe(s, 'paintTiming', 'first-contentful-paint')); metrics.cls = med(safe(s, 'pageinfo', 'cumulativeLayoutShift')); @@ -80,34 +84,35 @@ export async function parseResults(outputFolder, _url) { metrics.ttfb = med(safe(s, 'pageTimings', 'backEndTime')); metrics.max_potential_fid = med(safe(s, 'cpu', 'longTasks', 'maxPotentialFid')); - // Navigation timings (pageTimings keys come from browsertime's pageTimings object) - metrics.page_load_time = med(safe(s, 'pageTimings', 'pageLoadTime')); - metrics.fully_loaded = med(safe(s, 'timings', 'fullyLoaded')); + metrics.page_load_time = med(safe(s, 'pageTimings', 'pageLoadTime')); + metrics.fully_loaded = med(safe(s, 'timings', 'fullyLoaded')); metrics.dom_content_loaded = med(safe(s, 'pageTimings', 'domContentLoadedTime')); - metrics.dom_interactive = med(safe(s, 'pageTimings', 'domInteractiveTime')); - metrics.front_end_time = med(safe(s, 'pageTimings', 'frontEndTime')); - metrics.back_end_time = med(safe(s, 'pageTimings', 'backEndTime')); + metrics.dom_interactive = med(safe(s, 'pageTimings', 'domInteractiveTime')); + metrics.front_end_time = med(safe(s, 'pageTimings', 'frontEndTime')); + metrics.back_end_time = med(safe(s, 'pageTimings', 'backEndTime')); metrics.time_to_first_byte = metrics.ttfb; - // Visual metrics (from sitespeed-scroll-server / ffmpeg video analysis) - metrics.speed_index = med(safe(s, 'visualMetrics', 'SpeedIndex')); - metrics.first_visual_change = med(safe(s, 'visualMetrics', 'FirstVisualChange')); - metrics.last_visual_change = med(safe(s, 'visualMetrics', 'LastVisualChange')); - metrics.visual_complete_85 = med(safe(s, 'visualMetrics', 'VisualComplete85')); + metrics.speed_index = med(safe(s, 'visualMetrics', 'SpeedIndex')); + metrics.first_visual_change = med(safe(s, 'visualMetrics', 'FirstVisualChange')); + metrics.last_visual_change = med(safe(s, 'visualMetrics', 'LastVisualChange')); + metrics.visual_complete_85 = med(safe(s, 'visualMetrics', 'VisualComplete85')); metrics.perceptual_speed_index = med(safe(s, 'visualMetrics', 'PerceptualSpeedIndex')); - // CPU / Long Tasks metrics.long_tasks_count = med(safe(s, 'cpu', 'longTasks', 'tasks')); metrics.long_tasks_duration = med(safe(s, 'cpu', 'longTasks', 'totalDuration')); + + console.log('[parser] lcp=%s fcp=%s tbt=%s cls=%s ttfb=%s si=%s', + metrics.lcp, metrics.fcp, metrics.tbt, metrics.cls, metrics.ttfb, metrics.speed_index); } + } else { + console.warn('[parser] no browsertime.pageSummary.json found'); } - // ─── axe.pageSummary ────────────────────────────────────────────────────── - // Produced by AxeAggregator.summarizeStats(): - // { violations: { critical: {median,…}, serious, moderate, minor }, … } + // ─── axe.pageSummary ───────────────────────────────────────────────────── const axeFiles = summaries['axe'] || []; if (axeFiles.length > 0) { const axe = await readJson(axeFiles[0]); + console.log('[parser] axe top-level keys:', Object.keys(axe).join(', ')); metrics.axe_critical = med(safe(axe, 'violations', 'critical')) ?? 0; metrics.axe_serious = med(safe(axe, 'violations', 'serious')) ?? 0; metrics.axe_moderate = med(safe(axe, 'violations', 'moderate')) ?? 0; @@ -115,64 +120,63 @@ export async function parseResults(outputFolder, _url) { } // ─── coach.pageSummary ──────────────────────────────────────────────────── - // Coach sends the median-run's full coach result: { advice: { score, performance, … } } const coachFiles = summaries['coach'] || []; if (coachFiles.length > 0) { const coach = await readJson(coachFiles[0]); - metrics.score_overall = safe(coach, 'advice', 'score'); - metrics.score_performance = safe(coach, 'advice', 'performance', 'score'); + console.log('[parser] coach top-level keys:', Object.keys(coach).join(', ')); + metrics.score_overall = safe(coach, 'advice', 'score'); + metrics.score_performance = safe(coach, 'advice', 'performance', 'score'); metrics.score_accessibility = safe(coach, 'advice', 'accessibility', 'score'); - metrics.score_bestpractice = safe(coach, 'advice', 'bestpractice', 'score'); - metrics.score_privacy = safe(coach, 'advice', 'privacy', 'score'); + metrics.score_bestpractice = safe(coach, 'advice', 'bestpractice', 'score'); + metrics.score_privacy = safe(coach, 'advice', 'privacy', 'score'); + console.log('[parser] coach scores: overall=%s perf=%s', + metrics.score_overall, metrics.score_performance); } // ─── pagexray.pageSummary ───────────────────────────────────────────────── - // Sent as pageSummary[0] with `.statistics` added by the aggregator. - // Use statistics.*.median when available; fall back to direct property. const xrayFiles = summaries['pagexray'] || []; if (xrayFiles.length > 0) { const xray = await readJson(xrayFiles[0]); const xs = safe(xray, 'statistics'); + console.log('[parser] pagexray top-level keys:', Object.keys(xray).join(', ')); + if (xs) console.log('[parser] pagexray.statistics keys:', Object.keys(xs).join(', ')); function xv(statPath, directPath) { - // Try statistics.X.median first, then direct property - const fromStats = med(safe(xs, ...statPath)); + const fromStats = xs ? med(safe(xs, ...statPath)) : null; if (fromStats !== null) return fromStats; return safe(xray, ...directPath); } metrics.transfer_total = xv(['transferSize'], ['transferSize']); metrics.requests_total = xv(['requests'], ['requests']); - metrics.transfer_html = xv(['contentTypes', 'html', 'transferSize'], ['contentTypes', 'html', 'transferSize']); metrics.transfer_js = xv(['contentTypes', 'javascript', 'transferSize'], ['contentTypes', 'javascript', 'transferSize']); metrics.transfer_css = xv(['contentTypes', 'css', 'transferSize'], ['contentTypes', 'css', 'transferSize']); metrics.transfer_image = xv(['contentTypes', 'image', 'transferSize'], ['contentTypes', 'image', 'transferSize']); metrics.transfer_font = xv(['contentTypes', 'font', 'transferSize'], ['contentTypes', 'font', 'transferSize']); - metrics.requests_js = xv(['contentTypes', 'javascript', 'requests'], ['contentTypes', 'javascript', 'requests']); metrics.requests_css = xv(['contentTypes', 'css', 'requests'], ['contentTypes', 'css', 'requests']); metrics.requests_image = xv(['contentTypes', 'image', 'requests'], ['contentTypes', 'image', 'requests']); metrics.requests_font = xv(['contentTypes', 'font', 'requests'], ['contentTypes', 'font', 'requests']); + metrics.third_party_transfer = xv(['thirdParty', 'transferSize'], ['thirdParty', 'transferSize']); + metrics.third_party_requests = xv(['thirdParty', 'requests'], ['thirdParty', 'requests']); - metrics.third_party_transfer = xv(['thirdParty', 'transferSize'], ['thirdParty', 'transferSize']); - metrics.third_party_requests = xv(['thirdParty', 'requests'], ['thirdParty', 'requests']); + console.log('[parser] transfer_total=%s requests_total=%s', metrics.transfer_total, metrics.requests_total); } // ─── sustainable.pageSummary ────────────────────────────────────────────── - // Aggregator stores per-URL stats: - // { co2PerPageView: {median,…}, co2FirstParty: {median,…}, co2ThirdParty: {median,…} } const sustainFiles = summaries['sustainable'] || []; if (sustainFiles.length > 0) { const sust = await readJson(sustainFiles[0]); + console.log('[parser] sustainable top-level keys:', Object.keys(sust).join(', ')); metrics.co2_per_page_view = med(safe(sust, 'co2PerPageView')); - metrics.co2_total = med(safe(sust, 'co2PerPageView')); // totalCO2 not in per-URL summary + metrics.co2_total = med(safe(sust, 'co2PerPageView')); metrics.co2_first_party = med(safe(sust, 'co2FirstParty')); metrics.co2_third_party = med(safe(sust, 'co2ThirdParty')); } - // Strip nulls so we don't overwrite good DB values with NULL - return Object.fromEntries( - Object.entries(metrics).filter(([, v]) => v !== null && v !== undefined) - ); + const extracted = Object.entries(metrics).filter(([, v]) => v !== null && v !== undefined); + console.log(`[parser] extracted ${extracted.length} metrics`); + + return Object.fromEntries(extracted); } diff --git a/runner.js b/runner.js index c45e759..af1ad8d 100644 --- a/runner.js +++ b/runner.js @@ -19,6 +19,7 @@ export function runTest(job, onLine) { '--outputFolder', outputFolder, '--json', '--sustainable.enable', + '--sustainable.useGreenWebHostingAPI', '--axe.enable', '--coach', ];