speedboard/parser.js

import { readdir, readFile } from 'fs/promises';
import { join } from 'path';

/**
 * Recursively find all *.pageSummary.json files under outputFolder.
 * sitespeed.io v39 writes them into:
 *   pages/<hostname>/<urlpath>/data/<plugin>.pageSummary.json
 */
async function findPageSummaries(outputFolder) {
  const summaries = {};

  async function walk(dir) {
    let entries;
    try {
      entries = await readdir(dir, { withFileTypes: true });
    } catch {
      return;
    }
    for (const e of entries) {
      const full = join(dir, e.name);
      if (e.isDirectory()) {
        await walk(full);
      } else if (e.name.endsWith('.pageSummary.json')) {
        const plugin = e.name.replace('.pageSummary.json', '');
        if (!summaries[plugin]) summaries[plugin] = [];
        summaries[plugin].push(full);
      }
    }
  }

  await walk(outputFolder);
  return summaries;
}

async function readJson(filePath) {
  const raw = await readFile(filePath, 'utf8');
  return JSON.parse(raw);
}

// Safe deep-get: safe(obj, 'a', 'b', 'c') === obj?.a?.b?.c
function safe(obj, ...path) {
  let cur = obj;
  for (const key of path) {
    if (cur == null || typeof cur !== 'object') return null;
    cur = cur[key];
  }
  return cur ?? null;
}

// Get the median from a stats object { median, min, max, ... }
function med(obj) {
  return safe(obj, 'median');
}

export async function parseResults(outputFolder, _url) {
  const summaries = await findPageSummaries(outputFolder);
  const metrics = {};

  // ─── browsertime.pageSummary ───────────────────────────────────────────────
  // All aggregated values live under `statistics.*`
  // Key layout (from browsertimeAggregator.js):
  //   statistics.timings.largestContentfulPaint  — LCP renderTime
  //   statistics.timings.fullyLoaded
  //   statistics.pageTimings.*                   — pageLoadTime, backEndTime, etc.
  //   statistics.paintTiming.*                   — 'first-contentful-paint', etc.
  //   statistics.pageinfo.cumulativeLayoutShift  — CLS
  //   statistics.visualMetrics.*                 — SpeedIndex, FirstVisualChange, etc.
  //   statistics.cpu.longTasks.*                 — totalBlockingTime, tasks, etc.
  const btFiles = summaries['browsertime'] || [];
  if (btFiles.length > 0) {
    const bt = await readJson(btFiles[0]);
    const s = safe(bt, 'statistics');

    if (s) {
      // Core Web Vitals
      metrics.lcp  = med(safe(s, 'timings', 'largestContentfulPaint'));
      metrics.fcp  = med(safe(s, 'paintTiming', 'first-contentful-paint'));
      metrics.cls  = med(safe(s, 'pageinfo', 'cumulativeLayoutShift'));
      metrics.tbt  = med(safe(s, 'cpu', 'longTasks', 'totalBlockingTime'));
      metrics.ttfb = med(safe(s, 'pageTimings', 'backEndTime'));
      metrics.max_potential_fid = med(safe(s, 'cpu', 'longTasks', 'maxPotentialFid'));

      // Navigation timings (pageTimings keys come from browsertime's pageTimings object)
      metrics.page_load_time    = med(safe(s, 'pageTimings', 'pageLoadTime'));
      metrics.fully_loaded      = med(safe(s, 'timings', 'fullyLoaded'));
      metrics.dom_content_loaded = med(safe(s, 'pageTimings', 'domContentLoadedTime'));
      metrics.dom_interactive   = med(safe(s, 'pageTimings', 'domInteractiveTime'));
      metrics.front_end_time    = med(safe(s, 'pageTimings', 'frontEndTime'));
      metrics.back_end_time     = med(safe(s, 'pageTimings', 'backEndTime'));
      metrics.time_to_first_byte = metrics.ttfb;

      // Visual metrics (from sitespeed-scroll-server / ffmpeg video analysis)
      metrics.speed_index           = med(safe(s, 'visualMetrics', 'SpeedIndex'));
      metrics.first_visual_change   = med(safe(s, 'visualMetrics', 'FirstVisualChange'));
      metrics.last_visual_change    = med(safe(s, 'visualMetrics', 'LastVisualChange'));
      metrics.visual_complete_85    = med(safe(s, 'visualMetrics', 'VisualComplete85'));
      metrics.perceptual_speed_index = med(safe(s, 'visualMetrics', 'PerceptualSpeedIndex'));

      // CPU / Long Tasks
      metrics.long_tasks_count    = med(safe(s, 'cpu', 'longTasks', 'tasks'));
      metrics.long_tasks_duration = med(safe(s, 'cpu', 'longTasks', 'totalDuration'));
    }
  }

  // ─── axe.pageSummary ──────────────────────────────────────────────────────
  // Produced by AxeAggregator.summarizeStats():
  //   { violations: { critical: {median,…}, serious, moderate, minor }, … }
  const axeFiles = summaries['axe'] || [];
  if (axeFiles.length > 0) {
    const axe = await readJson(axeFiles[0]);
    metrics.axe_critical = med(safe(axe, 'violations', 'critical')) ?? 0;
    metrics.axe_serious  = med(safe(axe, 'violations', 'serious'))  ?? 0;
    metrics.axe_moderate = med(safe(axe, 'violations', 'moderate')) ?? 0;
    metrics.axe_minor    = med(safe(axe, 'violations', 'minor'))    ?? 0;
  }

  // ─── coach.pageSummary ────────────────────────────────────────────────────
  // Coach sends the median-run's full coach result: { advice: { score, performance, … } }
  const coachFiles = summaries['coach'] || [];
  if (coachFiles.length > 0) {
    const coach = await readJson(coachFiles[0]);
    metrics.score_overall      = safe(coach, 'advice', 'score');
    metrics.score_performance  = safe(coach, 'advice', 'performance', 'score');
    metrics.score_accessibility = safe(coach, 'advice', 'accessibility', 'score');
    metrics.score_bestpractice = safe(coach, 'advice', 'bestpractice', 'score');
    metrics.score_privacy      = safe(coach, 'advice', 'privacy', 'score');
  }

  // ─── pagexray.pageSummary ─────────────────────────────────────────────────
  // Sent as pageSummary[0] with `.statistics` added by the aggregator.
  // Use statistics.*.median when available; fall back to direct property.
  const xrayFiles = summaries['pagexray'] || [];
  if (xrayFiles.length > 0) {
    const xray = await readJson(xrayFiles[0]);
    const xs = safe(xray, 'statistics');

    function xv(statPath, directPath) {
      // Try statistics.X.median first, then direct property
      const fromStats = med(safe(xs, ...statPath));
      if (fromStats !== null) return fromStats;
      return safe(xray, ...directPath);
    }

    metrics.transfer_total = xv(['transferSize'], ['transferSize']);
    metrics.requests_total = xv(['requests'],     ['requests']);

    metrics.transfer_html  = xv(['contentTypes', 'html',       'transferSize'], ['contentTypes', 'html',       'transferSize']);
    metrics.transfer_js    = xv(['contentTypes', 'javascript', 'transferSize'], ['contentTypes', 'javascript', 'transferSize']);
    metrics.transfer_css   = xv(['contentTypes', 'css',        'transferSize'], ['contentTypes', 'css',        'transferSize']);
    metrics.transfer_image = xv(['contentTypes', 'image',      'transferSize'], ['contentTypes', 'image',      'transferSize']);
    metrics.transfer_font  = xv(['contentTypes', 'font',       'transferSize'], ['contentTypes', 'font',       'transferSize']);

    metrics.requests_js    = xv(['contentTypes', 'javascript', 'requests'], ['contentTypes', 'javascript', 'requests']);
    metrics.requests_css   = xv(['contentTypes', 'css',        'requests'], ['contentTypes', 'css',        'requests']);
    metrics.requests_image = xv(['contentTypes', 'image',      'requests'], ['contentTypes', 'image',      'requests']);
    metrics.requests_font  = xv(['contentTypes', 'font',       'requests'], ['contentTypes', 'font',       'requests']);

    metrics.third_party_transfer  = xv(['thirdParty', 'transferSize'], ['thirdParty', 'transferSize']);
    metrics.third_party_requests  = xv(['thirdParty', 'requests'],     ['thirdParty', 'requests']);
  }

  // ─── sustainable.pageSummary ──────────────────────────────────────────────
  // Aggregator stores per-URL stats:
  //   { co2PerPageView: {median,…}, co2FirstParty: {median,…}, co2ThirdParty: {median,…} }
  const sustainFiles = summaries['sustainable'] || [];
  if (sustainFiles.length > 0) {
    const sust = await readJson(sustainFiles[0]);
    metrics.co2_per_page_view = med(safe(sust, 'co2PerPageView'));
    metrics.co2_total         = med(safe(sust, 'co2PerPageView')); // totalCO2 not in per-URL summary
    metrics.co2_first_party   = med(safe(sust, 'co2FirstParty'));
    metrics.co2_third_party   = med(safe(sust, 'co2ThirdParty'));
  }

  // Strip nulls so we don't overwrite good DB values with NULL
  return Object.fromEntries(
    Object.entries(metrics).filter(([, v]) => v !== null && v !== undefined)
  );
}