From dfbbcc115288eae7ae89a60ac25c98bb8e6570fd Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:34:58 -0400 Subject: [PATCH 1/3] fix(orchestrate): parse exponent-form measurement values Signed-off-by: Rudy Celekli --- .../__tests__/measurements-exponents.test.ts | 28 +++++++++++++++++++ .../orchestrate/scripts/measurements.ts | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts diff --git a/orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts b/orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts new file mode 100644 index 000000000..8178b0480 --- /dev/null +++ b/orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts @@ -0,0 +1,28 @@ +import { expect, test } from "bun:test"; +import { compareMeasurement } from "../measurements.ts"; + +function check(measured: string, claimed: string) { + return compareMeasurement({ spec: { name: "bytes", command: "printf metric" }, measured, claim: { name: "bytes", before: "0", op: "→", after: claimed } }); +} + +test("native large-number output uses numeric tolerance", () => { + const result = check(String(1.05e21), String(1e21)); + expect(result.outcome).toBe("match"); + expect(result.measuredNumeric).toBe(1.05e21); +}); + +test("native small-number output reports its full numeric value", () => { + const result = check(String(1e-7), String(1e-7)); + expect(result.outcome).toBe("match"); + expect(result.measuredNumeric).toBe(1e-7); +}); + +test("equivalent decimal and exponent output compare equally with units", () => { + expect(check("1e3 MB", "1000 MB").outcome).toBe("match"); +}); + +test("out-of-tolerance, unit mismatch, and nonnumeric strings remain rejected", () => { + expect(check("2e21", "1e21").outcome).toBe("value-mismatch"); + expect(check("1e3 KB", "1000 MB").outcome).toBe("value-mismatch"); + expect(check("answer", "other").outcome).toBe("value-mismatch"); +}); diff --git a/orchestrate/skills/orchestrate/scripts/measurements.ts b/orchestrate/skills/orchestrate/scripts/measurements.ts index 2cc008e9c..e6b687ee1 100644 --- a/orchestrate/skills/orchestrate/scripts/measurements.ts +++ b/orchestrate/skills/orchestrate/scripts/measurements.ts @@ -204,7 +204,7 @@ interface NumericWithUnit { * fall through to string comparison instead of pretending it's numeric. */ function parseNumericWithUnit(value: string): NumericWithUnit | null { - const match = /^([-+]?\d+(?:\.\d+)?)\s*([a-zA-Z%]+)?$/.exec(value.trim()); + const match = /^([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*([a-zA-Z%]+)?$/.exec(value.trim()); if (!match) return null; const numeric = Number.parseFloat(match[1]); if (!Number.isFinite(numeric)) return null; @@ -214,7 +214,7 @@ function parseNumericWithUnit(value: string): NumericWithUnit | null { function parseFiniteNumber(value: string): number | null { // Used for the `measuredNumeric` display field: lenient leading-numeric // strip, separate from the end-anchored unit-aware comparison. - const match = /^[-+]?\d+(?:\.\d+)?/.exec(value.trim()); + const match = /^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/.exec(value.trim()); if (!match) return null; const parsed = Number.parseFloat(match[0]); return Number.isFinite(parsed) ? parsed : null; From 682df157cbd6a13534306299615c0b0a038f1d78 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 17:54:18 -0400 Subject: [PATCH 2/3] test(orchestrate): cover relative exponent comparisons Signed-off-by: Rudy Celekli --- .../scripts/__tests__/measurements-exponents.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts b/orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts index 8178b0480..d998384a1 100644 --- a/orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts +++ b/orchestrate/skills/orchestrate/scripts/__tests__/measurements-exponents.test.ts @@ -26,3 +26,8 @@ test("out-of-tolerance, unit mismatch, and nonnumeric strings remain rejected", expect(check("1e3 KB", "1000 MB").outcome).toBe("value-mismatch"); expect(check("answer", "other").outcome).toBe("value-mismatch"); }); + +test("subunit exponents respect relative drift", () => { + expect(check("1e-6", "1e-7").outcome).toBe("value-mismatch"); + expect(check("1.05e-7", "1e-7").outcome).toBe("match"); +}); From 8198121f4b381c627ee14fa7e5202e1f068d94d9 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Fri, 16 Oct 2026 14:47:19 -0400 Subject: [PATCH 3/3] fix(orchestrate): compare fractional measurements relatively --- .../__tests__/measurements-compare.test.ts | 17 +++++++++++++++++ .../skills/orchestrate/scripts/measurements.ts | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/orchestrate/skills/orchestrate/scripts/__tests__/measurements-compare.test.ts b/orchestrate/skills/orchestrate/scripts/__tests__/measurements-compare.test.ts index 64f437490..cee60d29f 100644 --- a/orchestrate/skills/orchestrate/scripts/__tests__/measurements-compare.test.ts +++ b/orchestrate/skills/orchestrate/scripts/__tests__/measurements-compare.test.ts @@ -44,6 +44,23 @@ describe("compareMeasurement / applyMeasurementParser / buildMeasurementEnv", () expect((result.driftFraction ?? 0) > 1).toBeTruthy(); }); + test("Fractional measurements use relative tolerance at either unit scale", () => { + for (const [after, measured, outcome] of [ + ["0.01 s", "0.1 s", "value-mismatch"], + ["10 ms", "100 ms", "value-mismatch"], + ["-0.01 s", "-0.1 s", "value-mismatch"], + ["0.01 s", "0.0105 s", "match"], + ["10 ms", "10.5 ms", "match"], + ["0 s", "0.09 s", "match"], + ["0 s", "0.2 s", "value-mismatch"], + ] as const) { + const result = compareMeasurement({ spec: baseSpec, measured, claim: claim(after) }); + expect(result.outcome).toBe(outcome); + } + const result = compareMeasurement({ spec: baseSpec, measured: "0.1 s", claim: claim("0.01 s") }); + expect(result.driftFraction).toBeCloseTo(9); + }); + test("Custom tolerance widens matching window", () => { const result = compareMeasurement({ spec: { ...baseSpec, toleranceFraction: 0.5 }, diff --git a/orchestrate/skills/orchestrate/scripts/measurements.ts b/orchestrate/skills/orchestrate/scripts/measurements.ts index e6b687ee1..582e89691 100644 --- a/orchestrate/skills/orchestrate/scripts/measurements.ts +++ b/orchestrate/skills/orchestrate/scripts/measurements.ts @@ -163,7 +163,10 @@ export function compareMeasurement( detail, }; } - const denom = Math.max(Math.abs(claimParsed.numeric), 1); + // Keep relative tolerance independent of the unit scale. Flooring at 1 + // accepts large errors for fractional measurements (e.g. seconds). + // A zero claim retains the existing absolute-tolerance fallback. + const denom = Math.abs(claimParsed.numeric) || 1; const drift = Math.abs(measuredParsed.numeric - claimParsed.numeric) / denom; const outcome: MeasurementCheck["outcome"] =