diff --git a/Cargo.lock b/Cargo.lock index 2c52490b..89bd824d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" dependencies = [ "asap-types", "serde", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-metricsql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" dependencies = [ "asap-types", "metricsql_parser", @@ -395,7 +395,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -404,7 +404,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -426,12 +426,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" dependencies = [ "serde", "serde_json", @@ -1712,7 +1712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2454,7 +2454,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2802,7 +2802,7 @@ dependencies = [ [[package]] name = "metricsql_common" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" dependencies = [ "chrono", ] @@ -2810,7 +2810,7 @@ dependencies = [ [[package]] name = "metricsql_parser" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=9f526a9#9f526a93130781e3d16744a8cc28e86f6b827f0c" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=a9651cc#a9651cc3705e7c9e50db8ccff68421fd74a91590" dependencies = [ "ahash", "chrono", @@ -3905,7 +3905,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4439,7 +4439,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5261,7 +5261,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 159af986..0d7ad49f 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -91,8 +91,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -100,9 +100,9 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } -asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } -asap-frontend-sql = { git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } +asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } +asap-frontend-sql = { git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 1944f944..503d44e4 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 5df1d0c9..555b6c1e 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,8 +38,8 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } -asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } +asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } # Shared external (workspace) serde.workspace = true @@ -131,7 +131,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "9f526a9" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "a9651cc" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" diff --git a/tools/erp-figure1/README.md b/tools/erp-figure1/README.md new file mode 100644 index 00000000..700c2754 --- /dev/null +++ b/tools/erp-figure1/README.md @@ -0,0 +1,54 @@ +# ERP Figure 1 runner + +`run.py` is the experiment boundary for the five Figure 1 arms: + +1. `autosketch_per_query` +2. `planner_analytical` +3. `planner_erp` +4. `asap_no_sharing` +5. `exact` + +The manifest pins a dataset SHA-256 and one candidate space, memory budget, +accuracy requirement, window workload, multi-family `ErpShapeObservation`, and +the complete fit/divergence/confidence selection policy from ASAPPlanner #380. +Each arm runs in a separate process. +The runner measures wall time, user/system CPU, and peak RSS with GNU `time`; +the arm reports its selected plan, selected candidate records, retained state +bytes, measured error, and any additional metrics. The runner checks that every +selected candidate belongs to the common space and that non-exact arms meet the +total memory and accuracy bounds. A successful arm must echo the exact contract from +`ASAP_FIGURE1_CONTRACT_JSON`; the dataset path is supplied in +`ASAP_FIGURE1_DATASET`. Contract drift aborts the experiment rather than +producing a comparison. + +The observation must contain positive cardinality/event counts and one or more +unique fitted families. Every fit records numeric parameters, non-negative +goodness-of-fit, and confidence in `[0,1]`. The shared selection policy includes +event/cardinality/parameter gates plus maximum goodness-of-fit, minimum +confidence, and minimum confidence margin. This keeps poor or ambiguous fits +visible to every arm instead of letting the ERP arm silently use an older +single-family shape contract. + +```sh +python3 tools/erp-figure1/run.py \ + --manifest /path/to/immutable-figure1-manifest.json \ + --output /path/to/new-results.json +python3 -m unittest tools/erp-figure1/test_run.py -v +``` + +Each command must print one JSON object: + +```json +{ + "contract": {"the exact object supplied in ASAP_FIGURE1_CONTRACT_JSON": true}, + "selected_plan": {"family": "cms", "width": 1024, "depth": 5}, + "selected_candidates": [{"family": "cms", "width": 1024, "depth": 5}], + "metrics": {"state_bytes": 40960, "max_error": 0.007}, + "provenance": {"revision": "..."} +} +``` + +The runner does not contain sketch, Planner, or AutoSketch implementations. The +arm commands must invoke those production/reference implementations. Failed +arms remain in the output and make the command fail. Do not publish a Figure 1 +from smoke adapters or from arms that merely relabel a fixed CLI configuration. diff --git a/tools/erp-figure1/run.py b/tools/erp-figure1/run.py new file mode 100644 index 00000000..f022129d --- /dev/null +++ b/tools/erp-figure1/run.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Run five isolated ERP evaluation arms under one immutable contract.""" +import argparse, hashlib, json, os, pathlib, subprocess, tempfile, time + +ARM_NAMES = ["autosketch_per_query", "planner_analytical", "planner_erp", "asap_no_sharing", "exact"] +REQUIRED_METRICS = {"state_bytes", "max_error"} + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def sha256_file(path): + digest = hashlib.sha256() + with open(path, "rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate_manifest(manifest, root): + if manifest.get("schema_version") != 1: + raise ValueError("unsupported manifest schema_version") + dataset = manifest["dataset"] + path = (root / dataset["path"]).resolve() if not pathlib.Path(dataset["path"]).is_absolute() else pathlib.Path(dataset["path"]) + actual = sha256_file(path) + if actual != dataset["sha256"]: + raise ValueError(f"dataset checksum mismatch: expected {dataset['sha256']}, got {actual}") + constraints = manifest["constraints"] + for key in ( + "candidate_space", "memory_budget_bytes", "accuracy", "window", + "erp_observation", "erp_selection_policy", + ): + if key not in constraints: + raise ValueError(f"missing common constraint {key}") + if constraints["memory_budget_bytes"] <= 0 or not constraints["candidate_space"]: + raise ValueError("candidate space and memory budget must be non-empty/positive") + validate_erp_contract(constraints["erp_observation"], constraints["erp_selection_policy"]) + arms = manifest["arms"] + if [arm.get("name") for arm in arms] != ARM_NAMES: + raise ValueError(f"arms must be ordered exactly as {ARM_NAMES}") + if any(not isinstance(arm.get("command"), list) or not arm["command"] for arm in arms): + raise ValueError("every arm requires a non-empty argv command") + return path, actual, constraints + + +def validate_erp_contract(observation, policy): + if observation.get("cardinality", 0) <= 0 or observation.get("observed_events", 0) <= 0: + raise ValueError("ERP observation cardinality/events must be positive") + fits = observation.get("fits") + if not isinstance(fits, list) or not fits: + raise ValueError("ERP observation requires at least one fitted family") + families = set() + for fit in fits: + family = fit.get("family") + if not isinstance(family, str) or not family or family in families: + raise ValueError("ERP fit families must be unique and non-empty") + families.add(family) + parameters = fit.get("parameters") + goodness, confidence = fit.get("goodness_of_fit"), fit.get("confidence") + if not isinstance(parameters, dict) or any(not isinstance(v, (int, float)) for v in parameters.values()): + raise ValueError("ERP fit parameters must be numeric") + if not isinstance(goodness, (int, float)) or goodness < 0: + raise ValueError("ERP goodness_of_fit must be non-negative") + if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1: + raise ValueError("ERP confidence must be within [0,1]") + fingerprint = observation.get("empirical_fingerprint") + if fingerprint is not None and (not isinstance(fingerprint, str) or not fingerprint): + raise ValueError("ERP empirical_fingerprint must be absent or non-empty") + required = { + "minimum_benchmark_events", "max_log2_cardinality_distance", + "max_parameter_distance", "max_goodness_of_fit", "minimum_confidence", + "minimum_confidence_margin", + } + if required - set(policy): + raise ValueError(f"ERP selection policy missing {sorted(required - set(policy))}") + if policy["minimum_benchmark_events"] <= 0: + raise ValueError("ERP minimum benchmark events must be positive") + for key in ("max_log2_cardinality_distance", "max_parameter_distance", "max_goodness_of_fit"): + if not isinstance(policy[key], (int, float)) or policy[key] < 0: + raise ValueError(f"ERP {key} must be non-negative") + for key in ("minimum_confidence", "minimum_confidence_margin"): + if not isinstance(policy[key], (int, float)) or not 0 <= policy[key] <= 1: + raise ValueError(f"ERP {key} must be within [0,1]") + + +def validate_result(name, result, expected_contract, candidate_space): + if result.get("contract") != expected_contract: + raise ValueError(f"{name} did not execute the identical evaluation contract") + missing = REQUIRED_METRICS - set(result.get("metrics", {})) + if missing or "selected_plan" not in result or "selected_candidates" not in result: + raise ValueError(f"{name} result missing selection/metrics: {sorted(missing)}") + selected = result["selected_candidates"] + if not isinstance(selected, list) or (name != "exact" and not selected): + raise ValueError(f"{name} must report selected candidate records") + legal = {canonical(candidate) for candidate in candidate_space} + if any(canonical(candidate) not in legal for candidate in selected): + raise ValueError(f"{name} selected a candidate outside the common space") + metrics = result["metrics"] + state_bytes, error = metrics["state_bytes"], metrics["max_error"] + if not isinstance(state_bytes, int) or state_bytes < 0 or not isinstance(error, (int, float)): + raise ValueError(f"{name} reported invalid state/error metrics") + if name != "exact" and state_bytes > expected_contract["memory_budget_bytes"]: + raise ValueError(f"{name} exceeded the common memory budget") + accuracy = expected_contract["accuracy"] + if accuracy.get("metric") != "max_error" or "upper_bound" not in accuracy: + raise ValueError("accuracy contract must define max_error upper_bound") + if error > accuracy["upper_bound"]: + raise ValueError(f"{name} violated the common accuracy constraint") + + +def run_arm(arm, expected_contract, candidate_space, cwd, dataset_path=None): + with tempfile.NamedTemporaryFile(prefix="asap-figure1-time-", delete=False) as timing: + timing_path = timing.name + argv = ["/usr/bin/time", "-f", '{"user_seconds":%U,"system_seconds":%S,"peak_rss_kb":%M}', "-o", timing_path, "--"] + arm["command"] + started = time.monotonic_ns() + env = os.environ.copy() + env["ASAP_FIGURE1_CONTRACT_JSON"] = canonical(expected_contract) + if dataset_path is not None: + env["ASAP_FIGURE1_DATASET"] = str(dataset_path) + proc = subprocess.run(argv, cwd=cwd, capture_output=True, text=True, env=env) + wall_ns = time.monotonic_ns() - started + try: + timing_lines = pathlib.Path(timing_path).read_text().splitlines() + resources = json.loads(timing_lines[-1]) + finally: + pathlib.Path(timing_path).unlink(missing_ok=True) + record = {"name": arm["name"], "command": arm["command"], "exit_code": proc.returncode, + "measured_resources": {**resources, "wall_seconds": wall_ns / 1e9}} + if proc.returncode: + record.update({"status": "failed", "stderr": proc.stderr}) + return record + result = json.loads(proc.stdout) + validate_result(arm["name"], result, expected_contract, candidate_space) + record.update({"status": "completed", "selected_plan": result["selected_plan"], + "metrics": result["metrics"], "provenance": result.get("provenance", {})}) + return record + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", required=True, type=pathlib.Path) + parser.add_argument("--output", required=True, type=pathlib.Path) + args = parser.parse_args() + if args.output.exists(): + parser.error("output already exists") + manifest_path = args.manifest.resolve(); root = manifest_path.parent + manifest = json.loads(manifest_path.read_text()) + dataset_path, dataset_sha, constraints = validate_manifest(manifest, root) + candidate_sha = hashlib.sha256(canonical(constraints["candidate_space"]).encode()).hexdigest() + contract = {"dataset_sha256": dataset_sha, "candidate_space_sha256": candidate_sha, + "memory_budget_bytes": constraints["memory_budget_bytes"], + "accuracy": constraints["accuracy"], "window": constraints["window"], + "erp_observation": constraints["erp_observation"], + "erp_selection_policy": constraints["erp_selection_policy"]} + report = {"schema_version": 1, "manifest_sha256": hashlib.sha256(canonical(manifest).encode()).hexdigest(), + "dataset": {"path": str(dataset_path), "sha256": dataset_sha}, "contract": contract, + "host": {"uname": list(os.uname())}, "arms": []} + for arm in manifest["arms"]: + report["arms"].append( + run_arm(arm, contract, constraints["candidate_space"], root, dataset_path) + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "x") as sink: + json.dump(report, sink, indent=2); sink.write("\n") + if any(row["status"] != "completed" for row in report["arms"]): + raise SystemExit(1) + +if __name__ == "__main__": main() diff --git a/tools/erp-figure1/test_run.py b/tools/erp-figure1/test_run.py new file mode 100644 index 00000000..72ca1731 --- /dev/null +++ b/tools/erp-figure1/test_run.py @@ -0,0 +1,48 @@ +import hashlib, importlib.util, json, pathlib, tempfile, unittest +P=pathlib.Path(__file__).with_name('run.py'); S=importlib.util.spec_from_file_location('runner',P); R=importlib.util.module_from_spec(S); S.loader.exec_module(R) +class ContractTests(unittest.TestCase): + def manifest(self, root): + data=root/'data.tsv'; data.write_text('0\t1\n'); sha=hashlib.sha256(data.read_bytes()).hexdigest() + contract={'candidate_space':[{'family':'cms','width':64,'depth':2}],'memory_budget_bytes':1024,'accuracy':{'metric':'max_error','upper_bound':0.01},'window':{'pane_seconds':60,'window_panes':[1,5]}, + 'erp_observation':{'cardinality':20,'observed_events':1000,'fits':[{'family':'zipf','parameters':{'exponent':1.1},'goodness_of_fit':0.03,'confidence':0.9}],'empirical_fingerprint':'fixture-v1'}, + 'erp_selection_policy':{'minimum_benchmark_events':100,'max_log2_cardinality_distance':1.0,'max_parameter_distance':0.2,'max_goodness_of_fit':0.1,'minimum_confidence':0.8,'minimum_confidence_margin':0.1}} + arms=[{'name':n,'command':['true']} for n in R.ARM_NAMES] + return {'schema_version':1,'dataset':{'path':'data.tsv','sha256':sha},'constraints':contract,'arms':arms} + def test_accepts_one_shared_contract(self): + with tempfile.TemporaryDirectory() as d: R.validate_manifest(self.manifest(pathlib.Path(d)),pathlib.Path(d)) + def test_rejects_dataset_drift(self): + with tempfile.TemporaryDirectory() as d: + root=pathlib.Path(d); m=self.manifest(root); (root/'data.tsv').write_text('changed') + with self.assertRaisesRegex(ValueError,'checksum mismatch'): R.validate_manifest(m,root) + def test_rejects_missing_or_reordered_arm(self): + with tempfile.TemporaryDirectory() as d: + root=pathlib.Path(d); m=self.manifest(root); m['arms'].reverse() + with self.assertRaisesRegex(ValueError,'ordered exactly'): R.validate_manifest(m,root) + def test_rejects_ambiguous_or_invalid_fit_contract(self): + with tempfile.TemporaryDirectory() as d: + root=pathlib.Path(d); m=self.manifest(root) + m['constraints']['erp_observation']['fits'].append(dict(m['constraints']['erp_observation']['fits'][0])) + with self.assertRaisesRegex(ValueError,'unique'): R.validate_manifest(m,root) + m=self.manifest(root); m['constraints']['erp_observation']['fits'][0]['confidence']=1.1 + with self.assertRaisesRegex(ValueError,'confidence'): R.validate_manifest(m,root) + def test_rejects_arm_contract_drift(self): + with tempfile.TemporaryDirectory() as d: + root=pathlib.Path(d); script=root/'arm.py'; script.write_text("import json; print(json.dumps({'contract':{},'selected_plan':{},'metrics':{'state_bytes':0,'max_error':0}}))") + with self.assertRaisesRegex(ValueError,'identical evaluation contract'): R.run_arm({'name':'exact','command':['python3',str(script)]},{'x':1},[],root) + def test_failed_arm_keeps_resources_and_stderr(self): + with tempfile.TemporaryDirectory() as d: + row=R.run_arm({'name':'planner_erp','command':['python3','-c','import sys; print("bad", file=sys.stderr); sys.exit(3)']}, + {'memory_budget_bytes':1,'accuracy':{'metric':'max_error','upper_bound':0}},[],pathlib.Path(d)) + self.assertEqual((row['status'],row['exit_code']),('failed',3)) + self.assertIn('bad',row['stderr']); self.assertIn('peak_rss_kb',row['measured_resources']) + def test_rejects_out_of_space_over_budget_or_inaccurate_selection(self): + contract={'memory_budget_bytes':100,'accuracy':{'metric':'max_error','upper_bound':0.01}} + base={'contract':contract,'selected_plan':{},'selected_candidates':[{'id':'a'}], + 'metrics':{'state_bytes':10,'max_error':0.0}} + R.validate_result('planner_erp',base,contract,[{'id':'a'}]) + for field,value,pattern in [('candidate',None,'outside'),('state_bytes',101,'memory'),('max_error',0.02,'accuracy')]: + row=json.loads(json.dumps(base)) + if field=='candidate': row['selected_candidates']=[{'id':'b'}] + else: row['metrics'][field]=value + with self.assertRaisesRegex(ValueError,pattern): R.validate_result('planner_erp',row,contract,[{'id':'a'}]) +if __name__=='__main__': unittest.main()