Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,23 @@ describe("compareMeasurement / applyMeasurementParser / buildMeasurementEnv", ()
expect((result.driftFraction ?? 0) > 1).toBeTruthy();
});

test("Fractional measurements use relative tolerance at either unit scale", () => {
for (const [after, measured, outcome] of [
["0.01 s", "0.1 s", "value-mismatch"],
["10 ms", "100 ms", "value-mismatch"],
["-0.01 s", "-0.1 s", "value-mismatch"],
["0.01 s", "0.0105 s", "match"],
["10 ms", "10.5 ms", "match"],
["0 s", "0.09 s", "match"],
["0 s", "0.2 s", "value-mismatch"],
] as const) {
const result = compareMeasurement({ spec: baseSpec, measured, claim: claim(after) });
expect(result.outcome).toBe(outcome);
}
const result = compareMeasurement({ spec: baseSpec, measured: "0.1 s", claim: claim("0.01 s") });
expect(result.driftFraction).toBeCloseTo(9);
});

test("Custom tolerance widens matching window", () => {
const result = compareMeasurement({
spec: { ...baseSpec, toleranceFraction: 0.5 },
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { expect, test } from "bun:test";
import { compareMeasurement } from "../measurements.ts";

function check(measured: string, claimed: string) {
return compareMeasurement({ spec: { name: "bytes", command: "printf metric" }, measured, claim: { name: "bytes", before: "0", op: "→", after: claimed } });
}

test("native large-number output uses numeric tolerance", () => {
const result = check(String(1.05e21), String(1e21));
expect(result.outcome).toBe("match");
expect(result.measuredNumeric).toBe(1.05e21);
});

test("native small-number output reports its full numeric value", () => {
const result = check(String(1e-7), String(1e-7));
expect(result.outcome).toBe("match");
expect(result.measuredNumeric).toBe(1e-7);
});

test("equivalent decimal and exponent output compare equally with units", () => {
expect(check("1e3 MB", "1000 MB").outcome).toBe("match");
});

test("out-of-tolerance, unit mismatch, and nonnumeric strings remain rejected", () => {
expect(check("2e21", "1e21").outcome).toBe("value-mismatch");
expect(check("1e3 KB", "1000 MB").outcome).toBe("value-mismatch");
expect(check("answer", "other").outcome).toBe("value-mismatch");
});

test("subunit exponents respect relative drift", () => {
expect(check("1e-6", "1e-7").outcome).toBe("value-mismatch");
expect(check("1.05e-7", "1e-7").outcome).toBe("match");
});
9 changes: 6 additions & 3 deletions orchestrate/skills/orchestrate/scripts/measurements.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,10 @@ export function compareMeasurement(
detail,
};
}
const denom = Math.max(Math.abs(claimParsed.numeric), 1);
// Keep relative tolerance independent of the unit scale. Flooring at 1
// accepts large errors for fractional measurements (e.g. seconds).
// A zero claim retains the existing absolute-tolerance fallback.
const denom = Math.abs(claimParsed.numeric) || 1;
const drift =
Math.abs(measuredParsed.numeric - claimParsed.numeric) / denom;
const outcome: MeasurementCheck["outcome"] =
Expand Down Expand Up @@ -204,7 +207,7 @@ interface NumericWithUnit {
* fall through to string comparison instead of pretending it's numeric.
*/
function parseNumericWithUnit(value: string): NumericWithUnit | null {
const match = /^([-+]?\d+(?:\.\d+)?)\s*([a-zA-Z%]+)?$/.exec(value.trim());
const match = /^([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*([a-zA-Z%]+)?$/.exec(value.trim());
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
if (!match) return null;
const numeric = Number.parseFloat(match[1]);
if (!Number.isFinite(numeric)) return null;
Expand All @@ -214,7 +217,7 @@ function parseNumericWithUnit(value: string): NumericWithUnit | null {
function parseFiniteNumber(value: string): number | null {
// Used for the `measuredNumeric` display field: lenient leading-numeric
// strip, separate from the end-anchored unit-aware comparison.
const match = /^[-+]?\d+(?:\.\d+)?/.exec(value.trim());
const match = /^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/.exec(value.trim());
if (!match) return null;
const parsed = Number.parseFloat(match[0]);
return Number.isFinite(parsed) ? parsed : null;
Expand Down