diff --git a/README.md b/README.md index 94c6c9b..9db84e7 100644 --- a/README.md +++ b/README.md @@ -645,6 +645,20 @@ With that set, `pymatching_only` writes: The directory is created lazily on the first write, so it is safe to point at a path that does not yet exist. +**Multiple GPUs.** `local_run.sh` uses every visible GPU, and every rank +reads the same files. Each rank decodes its own contiguous block of shots, in +rank order, and the reported counts and LER are summed over all ranks: + +* A positive `test.num_samples` limits the total number of shots per file. + Each rank decodes an equal share of the selected prefix, with one extra + shot on each of the first ranks when the total is uneven. +* The selected prefix must have at least one shot per rank. Use fewer GPUs + for smaller samples. + +The `PREDECODER_DECODE_OUTPUT_DIR` arrays are gathered on rank 0 in rank +order before they are written, so each array has one row per decoded shot, in +file order, and covers the same shots as the reported counts. + #### Smoke script ```bash @@ -1195,7 +1209,7 @@ Key scalars (as shown in TensorBoard): - **`Metrics/LER`**: Logical Error Rate on the evaluation target (computed during training-time evaluation). Lower is better. - Averaging: computed over `cfg.test.num_samples` Monte Carlo shots **per basis** (X and Z). - Default: `cfg.test.num_samples = 262144` (hardcoded for the current public release). - - Distributed: each rank uses `cfg.test.num_samples // world_size` shots per basis (any remainder is dropped). + - Distributed on generated samples: each rank uses `cfg.test.num_samples // world_size` shots per basis (any remainder is dropped). - **`Metrics/LER_Reduction_Factor`**: Ratio of post-predecoder LER to baseline LER (a “relative improvement” factor). `>1` means improvement. If both are 0, we log `1.0`. - Averaging: derived from the same LER evaluation run (same shot count as `Metrics/LER`). - **`Metrics/PyMatching_Speedup`**: Average PyMatching speedup from the pre-decoder: `latency_baseline / latency_after`. `>1` means faster decoding of PyMatching after pre-decoding. diff --git a/code/data/datapipe_stim.py b/code/data/datapipe_stim.py index f8a13a4..5bbf64b 100644 --- a/code/data/datapipe_stim.py +++ b/code/data/datapipe_stim.py @@ -214,6 +214,31 @@ def __getitem__(self, idx): ) +def _stim_file_shard( + *, file_shots: int, requested_total: int, rank: int, world_size: int, samples_path +) -> tuple: + """Return the ``[start, stop)`` rows of a Stim sample file that ``rank`` decodes. + + Apply the global shot limit before splitting the selected prefix of the file. + The first ranks take one extra row when the total is uneven. + """ + world_size = max(int(world_size), 1) + rank = int(rank) + if not 0 <= rank < world_size: + raise ValueError(f"rank={rank} is out of range for world_size={world_size}") + total = min(requested_total, file_shots) if requested_total > 0 else file_shots + if world_size > 1 and total < world_size: + raise ValueError( + f"Stim sample file {samples_path} has {total} selected shots, " + f"too few to give each of {world_size} ranks at least one shot" + ) + if rank == 0 and world_size > 1 and total < file_shots: + print(f"[Stim samples] Decoding {total} of {file_shots} shots in {samples_path}") + count, remainder = divmod(total, world_size) + start = rank * count + min(rank, remainder) + return start, start + count + int(rank < remainder) + + class QCDataPipePreDecoder_Memory_from_stim_file(Dataset): """ Datapipe for offline inference from Stim detector-sample files. @@ -240,6 +265,10 @@ class QCDataPipePreDecoder_Memory_from_stim_file(Dataset): ``sha256()`` is compared against ``metadata['noise_model_sha256']``. strict_noise: ``True`` (default) raises on noise-fingerprint drift; ``False`` downgrades the failure to a :class:`UserWarning`. + rank, world_size: Distributed shard of the file to expose. Every rank + reads the same file and keeps its own contiguous block, in rank + order. A positive ``num_samples`` limits the total across all ranks. + The first ranks receive one extra row when the split is uneven. """ def __init__( @@ -254,6 +283,8 @@ def __init__( code_rotation='XV', noise_model=None, strict_noise: bool = True, + rank: int = 0, + world_size: int = 1, ): self.distance = int(distance) self.n_rounds = max(int(n_rounds), 1) @@ -310,8 +341,14 @@ def __init__( noise_model_label=noise_label, strict_noise=bool(strict_noise), ) - if self.requested_num_samples > 0: - dets_and_obs = dets_and_obs[:self.requested_num_samples] + start, stop = _stim_file_shard( + file_shots=int(dets_and_obs.shape[0]), + requested_total=self.requested_num_samples, + rank=rank, + world_size=world_size, + samples_path=samples_path, + ) + dets_and_obs = dets_and_obs[start:stop] self.samples_path = samples_path self.metadata_path = metadata_path diff --git a/code/data/factory.py b/code/data/factory.py index 343f995..36e2802 100644 --- a/code/data/factory.py +++ b/code/data/factory.py @@ -41,10 +41,10 @@ def create_datapipe(cfg): raise ValueError("Invalid datapipe code") @staticmethod - def create_datapipe_inference(cfg): + def create_datapipe_inference(cfg, *, file_shot_limit=None): """Create datapipe for inference using Stim.""" if cfg.code == "surface": - return DatapipeFactory._create_surface_datapipe_inference(cfg) + return DatapipeFactory._create_surface_datapipe_inference(cfg, file_shot_limit) elif cfg.code == "color": return DatapipeFactory._create_color_datapipe_inference(cfg) else: @@ -74,7 +74,7 @@ def _create_color_datapipe(cfg): raise ValueError(f"Datapipe not implemented: {cfg.datapipe}") @staticmethod - def _create_surface_datapipe_inference(cfg): + def _create_surface_datapipe_inference(cfg, file_shot_limit=None): """ Datapipe for inference using Stim. """ @@ -93,8 +93,10 @@ def _create_surface_datapipe_inference(cfg): try: import torch.distributed as dist rank = dist.get_rank() if dist.is_initialized() else 0 + world_size = dist.get_world_size() if dist.is_initialized() else 1 except: rank = 0 + world_size = 1 if rank == 0: print( @@ -117,10 +119,16 @@ def _create_surface_datapipe_inference(cfg): # shapes because they only depend on (distance, n_rounds). measure_basis = "X" + num_samples = cfg.test.num_samples dataset_kwargs: dict = {} if stim_samples_dir: + if file_shot_limit is not None: + num_samples = file_shot_limit dataset_cls = QCDataPipePreDecoder_Memory_from_stim_file dataset_kwargs["stim_samples_dir"] = stim_samples_dir + # Every rank reads the same file; shard it so ranks decode disjoint shots. + dataset_kwargs["rank"] = rank + dataset_kwargs["world_size"] = world_size # PREDECODER_STIM_STRICT_NOISE=0 downgrades the noise-fingerprint # mismatch from an error to a UserWarning. Default is strict. strict_env = os.environ.get("PREDECODER_STIM_STRICT_NOISE", "").strip().lower() @@ -132,7 +140,7 @@ def _create_surface_datapipe_inference(cfg): test_dataset = dataset_cls( distance=cfg.distance, n_rounds=cfg.n_rounds, - num_samples=cfg.test.num_samples, + num_samples=num_samples, error_mode=error_mode_value, p_error=cfg.test.p_error, measure_basis=measure_basis, diff --git a/code/evaluation/failure_analysis.py b/code/evaluation/failure_analysis.py index 0c16e4e..ee63cf7 100644 --- a/code/evaluation/failure_analysis.py +++ b/code/evaluation/failure_analysis.py @@ -932,7 +932,9 @@ def decoder_ablation_study(model, device, dist, cfg): cfg_copy = deepcopy(cfg) cfg_copy.test.num_samples = samples_per_gpu cfg_copy.test.meas_basis_test = basis - test_dataset = DatapipeFactory.create_datapipe_inference(cfg_copy) + test_dataset = DatapipeFactory.create_datapipe_inference( + cfg_copy, file_shot_limit=total_samples + ) finally: torch.set_rng_state(torch_state) if cuda_state is not None: diff --git a/code/evaluation/logical_error_rate.py b/code/evaluation/logical_error_rate.py index b39fbcf..149864c 100644 --- a/code/evaluation/logical_error_rate.py +++ b/code/evaluation/logical_error_rate.py @@ -150,6 +150,25 @@ def _save_decode_array(output_dir: Optional[Path], basis: str, name: str, arr) - np.save(output_dir / f"{basis}_{name}.npy", np.asarray(arr)) +def _gather_decode_rows(dist, arr): + """Collect a per-rank decode-output array on rank 0, in rank order. + + Every rank decodes its own block of shots and the counts are all-reduced, so + the saved arrays must cover all ranks' shots too. ``arr`` may be ``None`` on a + rank that decoded nothing. This is a collective: call it on every rank. Returns + the concatenated rows on rank 0 (``None`` if no rank had any) and ``None`` on + the other ranks. + """ + if dist.world_size <= 1: + return arr + parts = [None] * dist.world_size if dist.rank == 0 else None + torch.distributed.gather_object(arr, parts, dst=0) + if dist.rank != 0: + return None + parts = [part for part in parts if part is not None] + return np.concatenate(parts, axis=0) if parts else None + + def _parse_quant_format(rank: int = 0) -> str: """Read and validate the QUANT_FORMAT environment variable. @@ -1009,7 +1028,9 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic # Create datapipe with reduced sample count per GPU cfg_copy = deepcopy(cfg) cfg_copy.test.num_samples = samples_per_gpu - test_dataset = DatapipeFactory.create_datapipe_inference(cfg_copy) + test_dataset = DatapipeFactory.create_datapipe_inference( + cfg_copy, file_shot_limit=total_samples + ) finally: # Restore random states torch.set_rng_state(torch_state) @@ -1058,11 +1079,24 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic baseline_pred = np.asarray(baseline_pred, dtype=np.uint8).reshape(-1, circuit.num_observables) num_pymatch_errors = int((baseline_pred != stim_obs).sum()) - if dist.rank == 0: - _save_decode_array(decode_output_dir, basis, "observables", stim_obs) - _save_decode_array(decode_output_dir, basis, "pymatching_predictions", baseline_pred) + if decode_output_dir is not None: + all_obs = _gather_decode_rows(dist, stim_obs) + all_baseline_pred = _gather_decode_rows(dist, baseline_pred) + if dist.rank == 0: + _save_decode_array(decode_output_dir, basis, "observables", all_obs) + _save_decode_array( + decode_output_dir, basis, "pymatching_predictions", all_baseline_pred + ) if decode_mode == "pymatching_only": + num_shots = int(stim_obs.shape[0]) + if dist.world_size > 1: + t_pymatch = torch.tensor(num_pymatch_errors, device=device, dtype=torch.long) + t_n = torch.tensor(num_shots, device=device, dtype=torch.long) + torch.distributed.all_reduce(t_pymatch, op=torch.distributed.ReduceOp.SUM) + torch.distributed.all_reduce(t_n, op=torch.distributed.ReduceOp.SUM) + num_pymatch_errors = int(t_pymatch.item()) + num_shots = int(t_n.item()) if dist.rank == 0: LAST_DEM_TIMING.update( { @@ -1076,8 +1110,7 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic f"[DEM Timing] build={dem_build_s:.2f}s decode={dem_decode_s:.2f}s " f"(basis={basis}, decode_mode=pymatching_only)" ) - return num_pymatch_errors, int(stim_obs.shape[0] - ), num_pymatch_errors, float("nan"), float("nan") + return num_pymatch_errors, num_shots, num_pymatch_errors, float("nan"), float("nan") # --- DataLoader: NO DistributedSampler - each GPU processes ALL of its own samples --- test_loader_kwargs = dict(cfg.test.dataloader) @@ -1381,8 +1414,9 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic ) / stim_dets.size if stim_dets.size > 0 else 0.0 floor_time_per_round = None detector_shape = None - final_prediction_rows = [] if dist.rank == 0 and decode_output_dir is not None else None - residual_rows = [] if dist.rank == 0 and decode_output_dir is not None else None + # Collected on every rank; gathered on rank 0 in rank order before saving. + final_prediction_rows = [] if decode_output_dir is not None else None + residual_rows = [] if decode_output_dir is not None else None t_start = time.perf_counter() t_model_time = 0.0 @@ -1576,21 +1610,30 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic total_samples = int(t_n.item()) num_pymatch_errors = int(t_pymatch.item()) - if dist.rank == 0: - if final_prediction_rows: - _save_decode_array( - decode_output_dir, - basis, - "ising_decoding_pymatching_predictions", - np.concatenate(final_prediction_rows, axis=0), - ) - if residual_rows: - _save_decode_array( - decode_output_dir, - basis, - "predecoder_residual_detectors", - np.concatenate(residual_rows, axis=0), - ) + if decode_output_dir is not None: + all_final_predictions = _gather_decode_rows( + dist, + np.concatenate(final_prediction_rows, axis=0) if final_prediction_rows else None, + ) + all_residuals = _gather_decode_rows( + dist, + np.concatenate(residual_rows, axis=0) if residual_rows else None, + ) + if dist.rank == 0: + if all_final_predictions is not None: + _save_decode_array( + decode_output_dir, + basis, + "ising_decoding_pymatching_predictions", + all_final_predictions, + ) + if all_residuals is not None: + _save_decode_array( + decode_output_dir, + basis, + "predecoder_residual_detectors", + all_residuals, + ) # Latency: single-shot (batch_size=1, matcher.decode) on a small subset on rank 0 only, # timed after the main loop for a clean CPU state. @@ -1700,7 +1743,9 @@ def compute_syndrome_density_reduction(model, device, dist, cfg) -> dict: from copy import deepcopy cfg_copy = deepcopy(cfg) cfg_copy.test.num_samples = samples_per_gpu - test_dataset = DatapipeFactory.create_datapipe_inference(cfg_copy) + test_dataset = DatapipeFactory.create_datapipe_inference( + cfg_copy, file_shot_limit=total_samples + ) if rank == 0: print(f"[Syndrome Density] Each GPU processes {len(test_dataset)} samples") diff --git a/code/tests/test_offline_stim_decoding.py b/code/tests/test_offline_stim_decoding.py index 06878d8..0a44aca 100644 --- a/code/tests/test_offline_stim_decoding.py +++ b/code/tests/test_offline_stim_decoding.py @@ -14,12 +14,14 @@ # limitations under the License. """Tests for offline decoding from Stim detector-sample files.""" +import json import os import sys import tempfile import unittest from pathlib import Path from types import SimpleNamespace +from unittest import mock import numpy as np import pymatching @@ -34,6 +36,7 @@ QCDataPipePreDecoder_Memory_from_stim_file, QCDataPipePreDecoder_Memory_inference, ) +from data.factory import DatapipeFactory from data.predecoder_transform import dets_to_predecoder_inputs from evaluation.logical_error_rate import count_logical_errors_with_errorbar from qec.noise_model import NoiseModel @@ -769,36 +772,37 @@ def test_helper_handles_t_equals_one(self): self.assertTrue(torch.all(train_x[:, 2] == 0)) -class TestOfflineStimLER(unittest.TestCase): - - def _base_cfg(self, num_samples: int): - return SimpleNamespace( - code="surface", - datapipe="memory", - distance=3, +def _base_cfg(num_samples: int, p_error: float = 0.003, meas_basis_test: str = "both"): + return SimpleNamespace( + code="surface", + datapipe="memory", + distance=3, + n_rounds=3, + data=SimpleNamespace( + error_mode="circuit_level_surface_custom", + code_rotation="XV", + ), + test=SimpleNamespace( + num_samples=num_samples, n_rounds=3, - data=SimpleNamespace( - error_mode="circuit_level_surface_custom", - code_rotation="XV", - ), - test=SimpleNamespace( - num_samples=num_samples, - n_rounds=3, - p_error=0.003, - meas_basis_test="both", - noise_model="none", - latency_num_samples=0, - th_data=0.0, - th_syn=0.0, - sampling_mode="threshold", - temperature=1.0, - dataloader={ - "batch_size": 16, - "num_workers": 0, - "persistent_workers": False, - }, - ), - ) + p_error=p_error, + meas_basis_test=meas_basis_test, + noise_model="none", + latency_num_samples=0, + th_data=0.0, + th_syn=0.0, + sampling_mode="threshold", + temperature=1.0, + dataloader={ + "batch_size": 16, + "num_workers": 0, + "persistent_workers": False, + }, + ), + ) + + +class TestOfflineStimLER(unittest.TestCase): def _run_file_ler(self, root: Path, *, decode_mode: str, model: nn.Module, cfg): old_env = { @@ -831,7 +835,7 @@ def test_pymatching_only_ler_matches_golden_x_and_z(self): root, decode_mode="pymatching_only", model=_UnusedModel(), - cfg=self._base_cfg(96), + cfg=_base_cfg(96), ) for basis in ("X", "Z"): @@ -855,7 +859,7 @@ def test_ising_decoding_pymatching_smoke_matches_baseline_for_zero_correction_mo root, decode_mode="ising_decoding_pymatching", model=_ZeroCorrectionModel(), - cfg=self._base_cfg(32), + cfg=_base_cfg(32), ) for basis in ("X", "Z"): @@ -874,8 +878,7 @@ def test_pymatching_only_single_basis_x_branch(self): root = Path(tmp) artifact = _write_artifact(root, basis="X", shots=64) - cfg = self._base_cfg(64) - cfg.test.meas_basis_test = "X" + cfg = _base_cfg(64, meas_basis_test="X") result = self._run_file_ler( root, @@ -890,5 +893,181 @@ def test_pymatching_only_single_basis_x_branch(self): self.assertEqual(observed, artifact.ler) +def _distributed_ler_worker(rank, world_size, init_file, root, shots, result_path): + torch.distributed.init_process_group( + backend="gloo", init_method=f"file://{init_file}", rank=rank, world_size=world_size + ) + try: + os.environ["PREDECODER_STIM_SAMPLES_DIR"] = root + os.environ["PREDECODER_TORCH_COMPILE"] = "0" + os.environ["PREDECODER_INFERENCE_NUM_WORKERS"] = "0" + dist = SimpleNamespace(rank=rank, world_size=world_size, device=torch.device("cpu")) + results = {} + for decode_mode, model in ( + ("pymatching_only", _UnusedModel()), + ("ising_decoding_pymatching", _ZeroCorrectionModel()), + ): + os.environ["PREDECODER_DECODE_MODE"] = decode_mode + os.environ["PREDECODER_DECODE_OUTPUT_DIR"] = str(Path(root) / "out" / decode_mode) + results[decode_mode] = count_logical_errors_with_errorbar( + model, dist.device, dist, _base_cfg(shots, p_error=0.02) + ) + if rank == 0: + Path(result_path).write_text(json.dumps(results)) + finally: + torch.distributed.destroy_process_group() + + +class TestOfflineStimDistributed(unittest.TestCase): + """Under torchrun every rank opens the same ``.dets`` file. The LER loop + splits the requested shots across ranks and all-reduces the + counts, so each rank must decode its own block of the file.""" + + def _shard(self, root: Path, num_samples: int, rank: int, world_size: int): + with mock.patch.dict(os.environ, {"PREDECODER_STIM_SAMPLES_DIR": str(root)}), \ + mock.patch("torch.distributed.is_initialized", return_value=True), \ + mock.patch("torch.distributed.get_rank", return_value=rank), \ + mock.patch("torch.distributed.get_world_size", return_value=world_size): + cfg = _base_cfg(num_samples, p_error=0.02, meas_basis_test="X") + return DatapipeFactory.create_datapipe_inference(cfg).dets_and_obs + + def test_ranks_get_disjoint_blocks_of_the_file(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=64, p_error=0.02).dets_and_obs + ) + shards = [self._shard(root, num_samples=64, rank=r, world_size=2) for r in range(2)] + self.assertTrue(torch.equal(shards[0], full[:32])) + self.assertTrue(torch.equal(shards[1], full[32:])) + + def test_file_shorter_than_requested_is_split_across_ranks(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=10, p_error=0.02).dets_and_obs + ) + shards = [self._shard(root, num_samples=1000, rank=r, world_size=2) for r in range(2)] + self.assertTrue(torch.equal(torch.cat(shards), full)) + + def test_remainder_shots_go_to_the_first_ranks(self): + """10 shots on 4 ranks: 3, 3, 2, 2 in file order, nothing dropped.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=10, p_error=0.02).dets_and_obs + ) + for num_samples in (10, 1000): + with self.subTest(num_samples=num_samples): + shards = [ + self._shard(root, num_samples=num_samples, rank=r, world_size=4) + for r in range(4) + ] + self.assertEqual([len(shard) for shard in shards], [3, 3, 2, 2]) + self.assertTrue(torch.equal(torch.cat(shards), full)) + + def test_global_cap_is_applied_before_sharding(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=11, p_error=0.02).dets_and_obs + ) + for cap, sizes in ((10, [5, 5]), (9, [5, 4]), (0, [6, 5])): + with self.subTest(cap=cap): + shards = [ + self._shard(root, num_samples=cap, rank=r, world_size=2) for r in range(2) + ] + self.assertEqual([len(shard) for shard in shards], sizes) + self.assertTrue(torch.equal(torch.cat(shards), full[:sum(sizes)])) + with self.assertRaisesRegex(ValueError, "too few"): + self._shard(root, num_samples=1, rank=0, world_size=2) + + def test_rank_out_of_range_raises(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_artifact(root, basis="X", shots=16, p_error=0.02) + for rank in (2, -1): + with self.subTest(rank=rank): + with self.assertRaisesRegex(ValueError, "out of range for world_size=2"): + self._shard(root, num_samples=8, rank=rank, world_size=2) + + def test_fewer_shots_than_ranks_raises_on_every_rank(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_artifact(root, basis="X", shots=3, p_error=0.02) + for rank in range(4): + with self.subTest(rank=rank): + with self.assertRaisesRegex(ValueError, "too few to give each of 4 ranks"): + self._shard(root, num_samples=1000, rank=rank, world_size=4) + + def test_two_rank_gloo_ler_matches_whole_file(self): + """Two gloo ranks must report the same counts and write the same decode-output + arrays as one process on the same files. An odd shot count checks that the + remainder shot is decoded.""" + shots = 257 + decode_modes = ( + ("pymatching_only", _UnusedModel()), + ("ising_decoding_pymatching", _ZeroCorrectionModel()), + ) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifacts = { + basis: _write_artifact(root, basis=basis, shots=shots, p_error=0.02) + for basis in ("X", "Z") + } + single = {} + for decode_mode, model in decode_modes: + env = { + "PREDECODER_STIM_SAMPLES_DIR": str(root), + "PREDECODER_TORCH_COMPILE": "0", + "PREDECODER_INFERENCE_NUM_WORKERS": "0", + "PREDECODER_DECODE_MODE": decode_mode, + "PREDECODER_DECODE_OUTPUT_DIR": str(root / "single" / decode_mode), + } + with mock.patch.dict(os.environ, env): + single[decode_mode] = count_logical_errors_with_errorbar( + model, torch.device("cpu"), _DummyDist(), _base_cfg(shots, p_error=0.02) + ) + + result_path = root / "results.json" + torch.multiprocessing.spawn( + _distributed_ler_worker, + args=(2, str(root / "rendezvous"), str(root), shots, str(result_path)), + nprocs=2, + join=True, + ) + results = json.loads(result_path.read_text()) + self.assertEqual(set(results), {mode for mode, _ in decode_modes}) + array_names = { + "pymatching_only": ("observables", "pymatching_predictions"), + "ising_decoding_pymatching": + ( + "observables", + "pymatching_predictions", + "ising_decoding_pymatching_predictions", + "predecoder_residual_detectors", + ), + } + for decode_mode, result in results.items(): + for basis in ("X", "Z"): + with self.subTest(decode_mode=decode_mode, basis=basis): + expected = single[decode_mode][basis] + self.assertEqual(result[basis]["num shots"], shots) + for key in ("num shots", "pymatch flips", "logical errors"): + self.assertEqual(result[basis][key], expected[key]) + for name in array_names[decode_mode]: + arr = np.load(root / "out" / decode_mode / f"{basis}_{name}.npy") + self.assertEqual(arr.shape[0], shots) + np.testing.assert_array_equal( + arr, + np.load(root / "single" / decode_mode / f"{basis}_{name}.npy"), + ) + num_obs = artifacts[basis].predictions.shape[1] + np.testing.assert_array_equal( + np.load(root / "out" / decode_mode / f"{basis}_observables.npy"), + artifacts[basis].dets_and_obs[:, -num_obs:], + ) + + if __name__ == "__main__": unittest.main()