From b86138ae7b68d979938ccb235731038e26783a5f Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Wed, 19 Aug 2026 22:30:38 +0000 Subject: [PATCH] test(skills): align TasteForge final contract --- skills/tasteforge-video/SKILL.md | 40 +++- tests/ci/tasteforge-video-skill.test.js | 187 +++++++++++++++++- .../tasteforge-video/final-contract.json | 151 ++++++++++++++ 3 files changed, 368 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/tasteforge-video/final-contract.json diff --git a/skills/tasteforge-video/SKILL.md b/skills/tasteforge-video/SKILL.md index 42c9d2572..49d93be8e 100644 --- a/skills/tasteforge-video/SKILL.md +++ b/skills/tasteforge-video/SKILL.md @@ -138,20 +138,42 @@ replacing full-frame 3D work. The returned receipt is the bundle boundary. It binds every emitted evidence artifact by relative path, byte size, SHA-256, genre, modality, `provider_execution:false`, and exact reference/time provenance. The receipt -itself requires `provider_calls:0`, `provider_execution:false`, and -`dry_run:true`. Every modality manifest and every nested request must contain -all four exact fail-closed fields: `provider_calls:0`, -`provider_execution:false`, `dry_run:true`, and `submit:false`; each request -also requires `provider_call_mode:"disabled"`. A missing field is a rejection, -not a default, and `dry_run:false` must be rejected before output is written. -Whole-file evidence uses an explicit whole-file time basis and never invents -timestamps. +requires `provider_calls:0` as an exact integer (the JSON boolean `false` is +invalid), `provider_execution:false`, and `dry_run:true`. Every genre spec also +requires explicit `dry_run:true`. The Resolve effect recipe requires that same +exact integer `provider_calls:0`, `provider_execution:false`, and `dry_run:true`. +Every modality manifest and every nested request must contain all four exact +fail-closed fields: integer `provider_calls:0`, `provider_execution:false`, +`dry_run:true`, and `submit:false`; each request also requires +`provider_call_mode:"disabled"`. A missing field is a rejection, not a default, +and `dry_run:false` must be rejected before output is written. + +Treat booleans as invalid numbers everywhere in timeline, evidence, probe, and +source-duration data. Every such numeric value must be a finite real: reject +`true`, `false`, NaN, infinities, negative event starts, non-positive durations, +out-of-range evidence times, and events ending beyond the declared finite +positive timeline. Whole-file evidence uses an explicit whole-file time basis +and never invents timestamps. + +Receipt references are the duration authority. Key each validated reference +duration by its cited SHA-256; duplicate occurrences of one digest must agree +on duration or the bundle is invalid. Every effect evidence `source_duration` +and every subject-anchor `source_duration` must equal that digest's validated +receipt duration, not merely contain its cited time. Probe duration and all +probe measurements must describe the same stable bytes used for byte count and +SHA-256. If the source mutates while probing or rehashes differently while it +is still available, fail closed rather than emitting or accepting a receipt. Always run bundle validation after creation. A missing image, video, or 3D-asset manifest must fail closed. Genericized or duplicate genres, periodic schedules, unanchored CV effects, missing placement constraints, provider-execution flags, unbound output files, byte-size drift, or SHA-256 tampering must fail closed. -Do not repair a failed receipt by deleting evidence or weakening validation. +Reject output roots, intermediates, or artifacts that are symlinks, and reject +special files (including FIFOs and devices); outputs must remain regular files +under a real directory tree. If local `ffmpeg` or `ffprobe` is unavailable, the +CLI must return its bounded nonzero local-media-processing error without a +Python traceback. Do not repair a failed receipt by deleting evidence or +weakening validation. ## Example Session diff --git a/tests/ci/tasteforge-video-skill.test.js b/tests/ci/tasteforge-video-skill.test.js index b758f545e..23c310173 100644 --- a/tests/ci/tasteforge-video-skill.test.js +++ b/tests/ci/tasteforge-video-skill.test.js @@ -21,12 +21,126 @@ function readJson(relativePath) { } function assertExactDryRunBoundary(payload, label) { - assert.strictEqual(payload.provider_calls, 0, `${label} must require provider_calls:0`); + assert.ok( + Number.isInteger(payload.provider_calls) && payload.provider_calls === 0, + `${label} must require exact integer provider_calls:0` + ); assert.strictEqual(payload.provider_execution, false, `${label} must require provider_execution:false`); assert.strictEqual(payload.dry_run, true, `${label} must require dry_run:true`); assert.strictEqual(payload.submit, false, `${label} must require submit:false`); } +function assertFiniteReal(value, label, { positive = false, nonnegative = false } = {}) { + assert.strictEqual(typeof value, "number", `${label} must be a real number, not a boolean`); + assert.ok(Number.isFinite(value), `${label} must be finite`); + if (positive) assert.ok(value > 0, `${label} must be positive`); + if (nonnegative) assert.ok(value >= 0, `${label} must be nonnegative`); +} + +function assertFiniteEvidenceTree(value, label) { + if (typeof value === "number" || typeof value === "boolean") { + assertFiniteReal(value, label); + } else if (Array.isArray(value)) { + value.forEach((nested) => assertFiniteEvidenceTree(nested, label)); + } else if (value && typeof value === "object") { + Object.values(value).forEach((nested) => assertFiniteEvidenceTree(nested, label)); + } +} + +function validateFinalContractFixture(fixture) { + for (const spec of fixture.genre_specs) { + assert.strictEqual(spec.dry_run, true, `genre ${spec.number} must require dry_run:true`); + } + + assertExactDryRunBoundary({ ...fixture.receipt, submit: false }, "receipt"); + const durationByDigest = new Map(); + for (const reference of fixture.receipt.references) { + assertFiniteReal(reference.source_duration, "receipt source_duration", { positive: true }); + const prior = durationByDigest.get(reference.sha256); + assert.ok( + prior === undefined || prior === reference.source_duration, + "duplicate digest has conflicting source durations" + ); + durationByDigest.set(reference.sha256, reference.source_duration); + assertFiniteEvidenceTree(reference.probe, "probe evidence"); + assert.strictEqual(reference.probe.duration, reference.source_duration); + for (const time of [ + ...(reference.probe.sample_times || []), + ...(reference.probe.scene_changes || []), + ...(reference.probe.style_samples || []).map((sample) => sample.time), + ]) { + assertFiniteReal(time, "probe evidence time", { nonnegative: true }); + assert.ok(time <= reference.source_duration, "probe evidence time exceeds source duration"); + } + } + + const recipe = fixture.effect_recipe; + assertExactDryRunBoundary({ ...recipe, submit: false }, "effect recipe"); + assertFiniteReal(recipe.timeline_duration, "timeline duration", { positive: true }); + for (const event of recipe.events) { + assertFiniteReal(event.time, "effect event time", { nonnegative: true }); + assertFiniteReal(event.duration, "effect event duration", { positive: true }); + assert.ok( + event.time + event.duration <= recipe.timeline_duration, + "effect event exceeds timeline duration" + ); + const evidence = event.evidence; + const evidenceDuration = durationByDigest.get(evidence.reference_sha256); + assert.ok(evidenceDuration !== undefined, "effect evidence cites unknown SHA-256"); + assert.strictEqual( + evidence.source_duration, + evidenceDuration, + "effect evidence duration must equal receipt reference duration" + ); + assertFiniteReal(evidence.time, "effect evidence time", { nonnegative: true }); + assert.ok(evidence.time <= evidenceDuration, "effect evidence time exceeds source duration"); + if (event.requires_subject_anchor) { + const anchor = event.subject_anchor; + const anchorDuration = durationByDigest.get(anchor.source_ref_sha256); + assert.ok(anchorDuration !== undefined, "subject anchor cites unknown SHA-256"); + assert.strictEqual( + anchor.source_duration, + anchorDuration, + "subject-anchor duration must equal receipt reference duration" + ); + assertFiniteReal(anchor.evidence_time, "subject-anchor evidence time", { nonnegative: true }); + assert.ok(anchor.evidence_time <= anchorDuration, "anchor evidence time exceeds source duration"); + assert.strictEqual(anchor.lost_policy, "disable_effect_until_track_recovers"); + } + } + + for (const modality of ["image", "video", "3d_asset"]) { + const manifest = fixture.manifests[modality]; + assert.ok(manifest, `missing ${modality} manifest`); + assertExactDryRunBoundary(manifest, `${modality} manifest`); + assert.ok(manifest.requests.length > 0, `${modality} requests must not be empty`); + for (const request of manifest.requests) { + assertExactDryRunBoundary(request, `${modality} request`); + assert.strictEqual(request.provider_call_mode, "disabled"); + } + } + + const binding = fixture.source_binding; + assert.strictEqual(binding.probed_sha256, binding.before_probe_sha256); + assert.strictEqual(binding.receipt_sha256, binding.before_probe_sha256); + assert.strictEqual( + binding.after_probe_sha256, + binding.before_probe_sha256, + "source mutation during probe must fail closed" + ); + + assert.ok(fixture.missing_media_tool_error.exit_code > 0, "missing media tool must exit nonzero"); + assert.ok(fixture.missing_media_tool_error.stderr.length < 256, "missing-tool error must stay bounded"); + assert.doesNotMatch(fixture.missing_media_tool_error.stderr, /Traceback/i); + + for (const entry of fixture.output_entries) { + assert.ok( + entry.type === "regular_file" || entry.type === "directory", + `output ${entry.path} must reject symlinks and special files` + ); + } +} + function validateRejectedContractFixture(fixture) { if (fixture.kind === "manifest") { assertExactDryRunBoundary(fixture.payload, "manifest"); @@ -137,14 +251,85 @@ test("defines the fail-closed file-driven multimodal contract", () => { /genre.*modality/is, /exact reference\/time provenance/i, /provider_calls:\s*0/i, + /exact integer/i, + /genre spec.*dry_run:\s*true/is, + /effect recipe.*provider_calls:\s*0/is, /dry_run:\s*true/i, /submit:\s*false/i, + /finite real/i, + /booleans as invalid numbers/i, + /duplicate occurrences.*digest.*agree.*duration/is, + /effect evidence `source_duration`.*validated\s+receipt duration/is, + /subject-anchor `source_duration`.*validated\s+receipt duration/is, + /same stable bytes/i, + /mutates while probing/i, + /ffmpeg.*ffprobe.*bounded nonzero.*without a\s+Python traceback/is, + /symlinks.*special files/is, /disable_effect_until_track_recovers/i, ]) assert.match(skill, phrase); assert.match(skill, /missing.*manifest.*fail closed/is); assert.match(skill, /tamper.*fail closed/is); }); +test("executable fixture enforces the independently passed final contract", () => { + const baseline = readJson("tests/fixtures/tasteforge-video/final-contract.json"); + const clone = () => JSON.parse(JSON.stringify(baseline)); + assert.doesNotThrow(() => validateFinalContractFixture(clone())); + + const mutations = [ + ["genre dry_run false", (value) => { value.genre_specs[0].dry_run = false; }], + ["receipt boolean provider_calls", (value) => { value.receipt.provider_calls = false; }], + ["effect boolean provider_calls", (value) => { value.effect_recipe.provider_calls = false; }], + ["manifest boolean provider_calls", (value) => { value.manifests.video.provider_calls = false; }], + ["request boolean provider_calls", (value) => { + value.manifests.video.requests[0].provider_calls = false; + }], + ["boolean timeline number", (value) => { value.effect_recipe.events[0].time = false; }], + ["non-finite event duration", (value) => { value.effect_recipe.events[0].duration = NaN; }], + ["non-finite evidence duration", (value) => { + value.effect_recipe.events[0].evidence.source_duration = Infinity; + }], + ["boolean probe measurement", (value) => { + value.receipt.references[0].probe.style_samples[0].luma = false; + }], + ["effect duration not bound to digest", (value) => { + value.effect_recipe.events[0].evidence.source_duration = 5; + }], + ["anchor duration not bound to digest", (value) => { + value.effect_recipe.events[0].subject_anchor.source_duration = 5; + }], + ["duplicate digest conflicting duration", (value) => { + const duplicate = JSON.parse(JSON.stringify(value.receipt.references[0])); + duplicate.source_duration = 7; + duplicate.probe.duration = 7; + value.receipt.references.push(duplicate); + }], + ["source mutation during probe", (value) => { + value.source_binding.after_probe_sha256 = "b".repeat(64); + }], + ["missing-tool success exit", (value) => { value.missing_media_tool_error.exit_code = 0; }], + ["missing-tool traceback", (value) => { + value.missing_media_tool_error.stderr = "Traceback (most recent call last): secret\n"; + }], + ["symlink output", (value) => { + value.output_entries.push({ path: "resolve", type: "symlink" }); + }], + ["special-file output", (value) => { + value.output_entries.push({ path: "resolve/pipe", type: "fifo" }); + }], + ]; + + for (const [label, mutate] of mutations) { + const fixture = clone(); + mutate(fixture); + assert.throws( + () => validateFinalContractFixture(fixture), + undefined, + `${label} was not rejected` + ); + } +}); + test("executable fixtures reject dry_run:false and continue_without_anchor", () => { for (const fixtureName of [ "reject-dry-run-false.json", diff --git a/tests/fixtures/tasteforge-video/final-contract.json b/tests/fixtures/tasteforge-video/final-contract.json new file mode 100644 index 000000000..a06d14a05 --- /dev/null +++ b/tests/fixtures/tasteforge-video/final-contract.json @@ -0,0 +1,151 @@ +{ + "genre_specs": [ + { + "number": 1, + "style_fingerprint": "flash-ethereal", + "dry_run": true + }, + { + "number": 2, + "style_fingerprint": "fluid-sketch", + "dry_run": true + } + ], + "receipt": { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "references": [ + { + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_duration": 6, + "probe": { + "duration": 6, + "sample_times": [0.75, 2.25], + "scene_changes": [0.75], + "style_samples": [ + { + "time": 0.75, + "luma": 0.2, + "saturation": 0.4 + } + ] + } + } + ] + }, + "effect_recipe": { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "timeline_duration": 6, + "events": [ + { + "effect": "cv_subject_glitch", + "time": 0.5, + "duration": 0.25, + "evidence": { + "reference_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "time": 0.75, + "source_duration": 6 + }, + "requires_subject_anchor": true, + "subject_anchor": { + "source_ref_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "evidence_time": 0.75, + "source_duration": 6, + "lost_policy": "disable_effect_until_track_recovers" + } + }, + { + "effect": "flash_bloom", + "time": 2, + "duration": 0.5, + "evidence": { + "reference_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "time": 2.25, + "source_duration": 6 + }, + "requires_subject_anchor": false + }, + { + "effect": "ink_bleed", + "time": 4.25, + "duration": 0.5, + "evidence": { + "reference_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "time": 0.75, + "source_duration": 6 + }, + "requires_subject_anchor": false + } + ] + }, + "manifests": { + "image": { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "submit": false, + "requests": [ + { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "submit": false, + "provider_call_mode": "disabled" + } + ] + }, + "video": { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "submit": false, + "requests": [ + { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "submit": false, + "provider_call_mode": "disabled" + } + ] + }, + "3d_asset": { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "submit": false, + "requests": [ + { + "provider_calls": 0, + "provider_execution": false, + "dry_run": true, + "submit": false, + "provider_call_mode": "disabled" + } + ] + } + }, + "source_binding": { + "before_probe_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "probed_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "receipt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "after_probe_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "missing_media_tool_error": { + "exit_code": 2, + "stderr": "ERROR local media processing unavailable\n" + }, + "output_entries": [ + { + "path": "genres/01.json", + "type": "regular_file" + }, + { + "path": "manifests", + "type": "directory" + } + ] +}