From 429eb89bfd91a0df26af8bc19306ab8acb1875f9 Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Sun, 15 Nov 2026 22:53:12 +0100 Subject: [PATCH 1/3] Shard offline Stim sample files across ranks and reduce pymatching_only counts With PREDECODER_STIM_SAMPLES_DIR set and more than one rank, every rank built its dataset from the leading rows of the .dets file, so all ranks decoded the same block and the all-reduce counted it world_size times. pymatching_only also returned before the all-reduce, so rank 0 reported only its own shard. Give each rank a contiguous, disjoint block of the file (rank and world_size passed through DatapipeFactory), and all-reduce the pymatching_only error and shot counts like the pre-decoder path does. Signed-off-by: Dezhen Lu --- code/data/datapipe_stim.py | 21 ++++- code/data/factory.py | 5 + code/evaluation/logical_error_rate.py | 11 ++- code/tests/test_offline_stim_decoding.py | 113 +++++++++++++++++++++++ 4 files changed, 147 insertions(+), 3 deletions(-) diff --git a/code/data/datapipe_stim.py b/code/data/datapipe_stim.py index f8a13a4..40f05fe 100644 --- a/code/data/datapipe_stim.py +++ b/code/data/datapipe_stim.py @@ -240,6 +240,10 @@ class QCDataPipePreDecoder_Memory_from_stim_file(Dataset): ``sha256()`` is compared against ``metadata['noise_model_sha256']``. strict_noise: ``True`` (default) raises on noise-fingerprint drift; ``False`` downgrades the failure to a :class:`UserWarning`. + rank, world_size: Distributed shard of the file to expose. Every rank + reads the same file, so each rank keeps its own contiguous block of + ``min(num_samples, file_shots // world_size)`` shots and no shot is + decoded twice. The defaults (``0``, ``1``) keep the whole file. """ def __init__( @@ -254,6 +258,8 @@ def __init__( code_rotation='XV', noise_model=None, strict_noise: bool = True, + rank: int = 0, + world_size: int = 1, ): self.distance = int(distance) self.n_rounds = max(int(n_rounds), 1) @@ -310,8 +316,21 @@ def __init__( noise_model_label=noise_label, strict_noise=bool(strict_noise), ) + # Callers pass the per-rank shot count (num_samples // world_size), and + # every rank opens the same file: give each rank a disjoint block. + world_size = max(int(world_size), 1) + rank = int(rank) + if not 0 <= rank < world_size: + raise ValueError(f"rank={rank} is out of range for world_size={world_size}") + shots_per_rank = dets_and_obs.shape[0] // world_size if self.requested_num_samples > 0: - dets_and_obs = dets_and_obs[:self.requested_num_samples] + shots_per_rank = min(shots_per_rank, self.requested_num_samples) + if world_size > 1 and shots_per_rank == 0: + raise ValueError( + f"Stim sample file {samples_path} has {dets_and_obs.shape[0]} shots, " + f"too few to give each of {world_size} ranks at least one shot" + ) + dets_and_obs = dets_and_obs[rank * shots_per_rank:(rank + 1) * shots_per_rank] self.samples_path = samples_path self.metadata_path = metadata_path diff --git a/code/data/factory.py b/code/data/factory.py index 343f995..1f8bbd9 100644 --- a/code/data/factory.py +++ b/code/data/factory.py @@ -93,8 +93,10 @@ def _create_surface_datapipe_inference(cfg): try: import torch.distributed as dist rank = dist.get_rank() if dist.is_initialized() else 0 + world_size = dist.get_world_size() if dist.is_initialized() else 1 except: rank = 0 + world_size = 1 if rank == 0: print( @@ -121,6 +123,9 @@ def _create_surface_datapipe_inference(cfg): if stim_samples_dir: dataset_cls = QCDataPipePreDecoder_Memory_from_stim_file dataset_kwargs["stim_samples_dir"] = stim_samples_dir + # Every rank reads the same file; shard it so ranks decode disjoint shots. + dataset_kwargs["rank"] = rank + dataset_kwargs["world_size"] = world_size # PREDECODER_STIM_STRICT_NOISE=0 downgrades the noise-fingerprint # mismatch from an error to a UserWarning. Default is strict. strict_env = os.environ.get("PREDECODER_STIM_STRICT_NOISE", "").strip().lower() diff --git a/code/evaluation/logical_error_rate.py b/code/evaluation/logical_error_rate.py index b39fbcf..ec2d523 100644 --- a/code/evaluation/logical_error_rate.py +++ b/code/evaluation/logical_error_rate.py @@ -1063,6 +1063,14 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic _save_decode_array(decode_output_dir, basis, "pymatching_predictions", baseline_pred) if decode_mode == "pymatching_only": + num_shots = int(stim_obs.shape[0]) + if dist.world_size > 1: + t_pymatch = torch.tensor(num_pymatch_errors, device=device, dtype=torch.long) + t_n = torch.tensor(num_shots, device=device, dtype=torch.long) + torch.distributed.all_reduce(t_pymatch, op=torch.distributed.ReduceOp.SUM) + torch.distributed.all_reduce(t_n, op=torch.distributed.ReduceOp.SUM) + num_pymatch_errors = int(t_pymatch.item()) + num_shots = int(t_n.item()) if dist.rank == 0: LAST_DEM_TIMING.update( { @@ -1076,8 +1084,7 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic f"[DEM Timing] build={dem_build_s:.2f}s decode={dem_decode_s:.2f}s " f"(basis={basis}, decode_mode=pymatching_only)" ) - return num_pymatch_errors, int(stim_obs.shape[0] - ), num_pymatch_errors, float("nan"), float("nan") + return num_pymatch_errors, num_shots, num_pymatch_errors, float("nan"), float("nan") # --- DataLoader: NO DistributedSampler - each GPU processes ALL of its own samples --- test_loader_kwargs = dict(cfg.test.dataloader) diff --git a/code/tests/test_offline_stim_decoding.py b/code/tests/test_offline_stim_decoding.py index 06878d8..7047faa 100644 --- a/code/tests/test_offline_stim_decoding.py +++ b/code/tests/test_offline_stim_decoding.py @@ -14,12 +14,14 @@ # limitations under the License. """Tests for offline decoding from Stim detector-sample files.""" +import json import os import sys import tempfile import unittest from pathlib import Path from types import SimpleNamespace +from unittest import mock import numpy as np import pymatching @@ -34,6 +36,7 @@ QCDataPipePreDecoder_Memory_from_stim_file, QCDataPipePreDecoder_Memory_inference, ) +from data.factory import DatapipeFactory from data.predecoder_transform import dets_to_predecoder_inputs from evaluation.logical_error_rate import count_logical_errors_with_errorbar from qec.noise_model import NoiseModel @@ -890,5 +893,115 @@ def test_pymatching_only_single_basis_x_branch(self): self.assertEqual(observed, artifact.ler) +def _distributed_ler_cfg(num_samples: int, p_error: float, basis: str): + return SimpleNamespace( + code="surface", + datapipe="memory", + distance=3, + n_rounds=3, + data=SimpleNamespace( + error_mode="circuit_level_surface_custom", + code_rotation="XV", + ), + test=SimpleNamespace( + num_samples=num_samples, + n_rounds=3, + p_error=p_error, + meas_basis_test=basis, + noise_model="none", + latency_num_samples=0, + th_data=0.0, + th_syn=0.0, + sampling_mode="threshold", + temperature=1.0, + dataloader={ + "batch_size": 16, + "num_workers": 0, + "persistent_workers": False, + }, + ), + ) + + +def _distributed_ler_worker(rank, world_size, init_file, root, shots, result_path): + torch.distributed.init_process_group( + backend="gloo", init_method=f"file://{init_file}", rank=rank, world_size=world_size + ) + try: + os.environ["PREDECODER_STIM_SAMPLES_DIR"] = root + os.environ["PREDECODER_TORCH_COMPILE"] = "0" + os.environ["PREDECODER_INFERENCE_NUM_WORKERS"] = "0" + dist = SimpleNamespace(rank=rank, world_size=world_size, device=torch.device("cpu")) + results = {} + for decode_mode, model in ( + ("pymatching_only", _UnusedModel()), + ("ising_decoding_pymatching", _ZeroCorrectionModel()), + ): + os.environ["PREDECODER_DECODE_MODE"] = decode_mode + results[decode_mode] = count_logical_errors_with_errorbar( + model, dist.device, dist, _distributed_ler_cfg(shots, 0.02, "both") + ) + if rank == 0: + Path(result_path).write_text(json.dumps(results)) + finally: + torch.distributed.destroy_process_group() + + +class TestOfflineStimDistributed(unittest.TestCase): + """Under torchrun every rank opens the same ``.dets`` file. The LER loop + asks each rank for ``num_samples // world_size`` shots and all-reduces the + counts, so each rank must decode its own block of the file.""" + + def _shard(self, root: Path, num_samples: int, rank: int, world_size: int): + with mock.patch.dict(os.environ, {"PREDECODER_STIM_SAMPLES_DIR": str(root)}), \ + mock.patch("torch.distributed.is_initialized", return_value=True), \ + mock.patch("torch.distributed.get_rank", return_value=rank), \ + mock.patch("torch.distributed.get_world_size", return_value=world_size): + cfg = _distributed_ler_cfg(num_samples, 0.02, "X") + return DatapipeFactory.create_datapipe_inference(cfg).dets_and_obs + + def test_ranks_get_disjoint_blocks_of_the_file(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=64, p_error=0.02).dets_and_obs + ) + shards = [self._shard(root, num_samples=32, rank=r, world_size=2) for r in range(2)] + self.assertTrue(torch.equal(shards[0], full[:32])) + self.assertTrue(torch.equal(shards[1], full[32:])) + + def test_file_shorter_than_requested_is_split_across_ranks(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=10, p_error=0.02).dets_and_obs + ) + shards = [self._shard(root, num_samples=1000, rank=r, world_size=2) for r in range(2)] + self.assertTrue(torch.equal(torch.cat(shards), full)) + + def test_two_rank_gloo_ler_matches_whole_file(self): + shots = 256 + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + expected = { + basis: _write_artifact(root, basis=basis, shots=shots, p_error=0.02).errors + for basis in ("X", "Z") + } + result_path = root / "results.json" + torch.multiprocessing.spawn( + _distributed_ler_worker, + args=(2, str(root / "rendezvous"), str(root), shots, str(result_path)), + nprocs=2, + join=True, + ) + results = json.loads(result_path.read_text()) + for decode_mode, result in results.items(): + for basis in ("X", "Z"): + with self.subTest(decode_mode=decode_mode, basis=basis): + self.assertEqual(result[basis]["num shots"], shots) + self.assertEqual(result[basis]["pymatch flips"], expected[basis]) + self.assertEqual(result[basis]["logical errors"], expected[basis]) + + if __name__ == "__main__": unittest.main() From 3b7fe4a05be9affacb7eef1f0260ae64c96e306b Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Tue, 17 Nov 2026 08:33:15 +0100 Subject: [PATCH 2/3] Gather decode outputs across ranks and spread remainder shots With an output directory set, rank 0 wrote only its own block of shots, so the saved arrays did not cover the shots behind the reported counts. Gather the observables, PyMatching predictions, pre-decoder predictions and residual detectors on rank 0 in rank order before saving them. When num_samples does not cap the shard, split the whole Stim file and give the first file_shots % world_size ranks one extra shot, so a file yields the same shot count on N ranks as on one. When it does cap the shard, rank 0 prints how many of the file's shots are decoded. A file with fewer shots than ranks is rejected on every rank. Move the test config helper to module level, add negative tests for the two ValueError paths and compare a two-rank gloo run against one process, and document the multi-GPU behaviour in the README. Signed-off-by: Dezhen Lu --- README.md | 19 +++ code/data/datapipe_stim.py | 66 ++++++-- code/evaluation/logical_error_rate.py | 74 ++++++--- code/tests/test_offline_stim_decoding.py | 199 +++++++++++++++-------- 4 files changed, 250 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 94c6c9b..3e8aa2e 100644 --- a/README.md +++ b/README.md @@ -645,6 +645,25 @@ With that set, `pymatching_only` writes: The directory is created lazily on the first write, so it is safe to point at a path that does not yet exist. +**Multiple GPUs.** `local_run.sh` uses every visible GPU, and every rank +reads the same files. Each rank decodes its own contiguous block of shots, in +rank order, and the reported counts and LER are summed over all ranks: + +* If `test.num_samples // world_size` is at least `file shots // world_size` + (for example when `test.num_samples` is the file size or larger), the whole + file is decoded. The remainder `file shots % world_size` goes to the first + ranks, one extra shot each. When `test.num_samples` is at least the file + size, the result therefore matches a single-GPU run on the same file. +* Otherwise each rank decodes `test.num_samples // world_size` shots from the + front of the file, and rank 0 prints how many of the file's shots are + decoded. +* A file with fewer shots than ranks is rejected; run with fewer GPUs + (`GPUS=1`). + +The `PREDECODER_DECODE_OUTPUT_DIR` arrays are gathered on rank 0 in rank +order before they are written, so each array has one row per decoded shot, in +file order, and covers the same shots as the reported counts. + #### Smoke script ```bash diff --git a/code/data/datapipe_stim.py b/code/data/datapipe_stim.py index 40f05fe..1b03f28 100644 --- a/code/data/datapipe_stim.py +++ b/code/data/datapipe_stim.py @@ -214,6 +214,39 @@ def __getitem__(self, idx): ) +def _stim_file_shard( + *, file_shots: int, requested_per_rank: int, rank: int, world_size: int, samples_path +) -> tuple: + """Return the ``[start, stop)`` rows of a Stim sample file that ``rank`` decodes. + + Callers pass the per-rank shot count (``num_samples // world_size``) and every + rank opens the same file, so each rank gets a disjoint contiguous block, in rank + order. If the request does not cap the block, the whole file is split and the + first ``file_shots % world_size`` ranks take one extra shot; otherwise every rank + takes ``requested_per_rank`` shots from the front of the file. The checks only + depend on values that are equal on every rank, so all ranks raise together. + """ + world_size = max(int(world_size), 1) + rank = int(rank) + if not 0 <= rank < world_size: + raise ValueError(f"rank={rank} is out of range for world_size={world_size}") + if world_size > 1 and file_shots < world_size: + raise ValueError( + f"Stim sample file {samples_path} has {file_shots} shots, " + f"too few to give each of {world_size} ranks at least one shot" + ) + base, remainder = divmod(file_shots, world_size) + if requested_per_rank <= 0 or requested_per_rank >= base: + start = rank * base + min(rank, remainder) + return start, start + base + (1 if rank < remainder else 0) + if rank == 0 and world_size > 1: + print( + f"[Stim samples] Decoding {requested_per_rank * world_size} of {file_shots} " + f"shots in {samples_path} ({requested_per_rank} per rank on {world_size} ranks)" + ) + return rank * requested_per_rank, (rank + 1) * requested_per_rank + + class QCDataPipePreDecoder_Memory_from_stim_file(Dataset): """ Datapipe for offline inference from Stim detector-sample files. @@ -241,9 +274,13 @@ class QCDataPipePreDecoder_Memory_from_stim_file(Dataset): strict_noise: ``True`` (default) raises on noise-fingerprint drift; ``False`` downgrades the failure to a :class:`UserWarning`. rank, world_size: Distributed shard of the file to expose. Every rank - reads the same file, so each rank keeps its own contiguous block of - ``min(num_samples, file_shots // world_size)`` shots and no shot is - decoded twice. The defaults (``0``, ``1``) keep the whole file. + reads the same file and keeps its own contiguous block, in rank + order, so no shot is decoded twice. ``num_samples`` is the per-rank + count. When it is at least ``file_shots // world_size`` (or not + positive), the whole file is split: the first + ``file_shots % world_size`` ranks take one extra shot. Otherwise each + rank takes ``num_samples`` shots from the front of the file. The + defaults (``0``, ``1``) keep the previous single-process behaviour. """ def __init__( @@ -316,21 +353,14 @@ def __init__( noise_model_label=noise_label, strict_noise=bool(strict_noise), ) - # Callers pass the per-rank shot count (num_samples // world_size), and - # every rank opens the same file: give each rank a disjoint block. - world_size = max(int(world_size), 1) - rank = int(rank) - if not 0 <= rank < world_size: - raise ValueError(f"rank={rank} is out of range for world_size={world_size}") - shots_per_rank = dets_and_obs.shape[0] // world_size - if self.requested_num_samples > 0: - shots_per_rank = min(shots_per_rank, self.requested_num_samples) - if world_size > 1 and shots_per_rank == 0: - raise ValueError( - f"Stim sample file {samples_path} has {dets_and_obs.shape[0]} shots, " - f"too few to give each of {world_size} ranks at least one shot" - ) - dets_and_obs = dets_and_obs[rank * shots_per_rank:(rank + 1) * shots_per_rank] + start, stop = _stim_file_shard( + file_shots=int(dets_and_obs.shape[0]), + requested_per_rank=self.requested_num_samples, + rank=rank, + world_size=world_size, + samples_path=samples_path, + ) + dets_and_obs = dets_and_obs[start:stop] self.samples_path = samples_path self.metadata_path = metadata_path diff --git a/code/evaluation/logical_error_rate.py b/code/evaluation/logical_error_rate.py index ec2d523..ad35267 100644 --- a/code/evaluation/logical_error_rate.py +++ b/code/evaluation/logical_error_rate.py @@ -150,6 +150,25 @@ def _save_decode_array(output_dir: Optional[Path], basis: str, name: str, arr) - np.save(output_dir / f"{basis}_{name}.npy", np.asarray(arr)) +def _gather_decode_rows(dist, arr): + """Collect a per-rank decode-output array on rank 0, in rank order. + + Every rank decodes its own block of shots and the counts are all-reduced, so + the saved arrays must cover all ranks' shots too. ``arr`` may be ``None`` on a + rank that decoded nothing. This is a collective: call it on every rank. Returns + the concatenated rows on rank 0 (``None`` if no rank had any) and ``None`` on + the other ranks. + """ + if dist.world_size <= 1: + return arr + parts = [None] * dist.world_size if dist.rank == 0 else None + torch.distributed.gather_object(arr, parts, dst=0) + if dist.rank != 0: + return None + parts = [part for part in parts if part is not None] + return np.concatenate(parts, axis=0) if parts else None + + def _parse_quant_format(rank: int = 0) -> str: """Read and validate the QUANT_FORMAT environment variable. @@ -1058,9 +1077,14 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic baseline_pred = np.asarray(baseline_pred, dtype=np.uint8).reshape(-1, circuit.num_observables) num_pymatch_errors = int((baseline_pred != stim_obs).sum()) - if dist.rank == 0: - _save_decode_array(decode_output_dir, basis, "observables", stim_obs) - _save_decode_array(decode_output_dir, basis, "pymatching_predictions", baseline_pred) + if decode_output_dir is not None: + all_obs = _gather_decode_rows(dist, stim_obs) + all_baseline_pred = _gather_decode_rows(dist, baseline_pred) + if dist.rank == 0: + _save_decode_array(decode_output_dir, basis, "observables", all_obs) + _save_decode_array( + decode_output_dir, basis, "pymatching_predictions", all_baseline_pred + ) if decode_mode == "pymatching_only": num_shots = int(stim_obs.shape[0]) @@ -1388,8 +1412,9 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic ) / stim_dets.size if stim_dets.size > 0 else 0.0 floor_time_per_round = None detector_shape = None - final_prediction_rows = [] if dist.rank == 0 and decode_output_dir is not None else None - residual_rows = [] if dist.rank == 0 and decode_output_dir is not None else None + # Collected on every rank; gathered on rank 0 in rank order before saving. + final_prediction_rows = [] if decode_output_dir is not None else None + residual_rows = [] if decode_output_dir is not None else None t_start = time.perf_counter() t_model_time = 0.0 @@ -1583,21 +1608,30 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic total_samples = int(t_n.item()) num_pymatch_errors = int(t_pymatch.item()) - if dist.rank == 0: - if final_prediction_rows: - _save_decode_array( - decode_output_dir, - basis, - "ising_decoding_pymatching_predictions", - np.concatenate(final_prediction_rows, axis=0), - ) - if residual_rows: - _save_decode_array( - decode_output_dir, - basis, - "predecoder_residual_detectors", - np.concatenate(residual_rows, axis=0), - ) + if decode_output_dir is not None: + all_final_predictions = _gather_decode_rows( + dist, + np.concatenate(final_prediction_rows, axis=0) if final_prediction_rows else None, + ) + all_residuals = _gather_decode_rows( + dist, + np.concatenate(residual_rows, axis=0) if residual_rows else None, + ) + if dist.rank == 0: + if all_final_predictions is not None: + _save_decode_array( + decode_output_dir, + basis, + "ising_decoding_pymatching_predictions", + all_final_predictions, + ) + if all_residuals is not None: + _save_decode_array( + decode_output_dir, + basis, + "predecoder_residual_detectors", + all_residuals, + ) # Latency: single-shot (batch_size=1, matcher.decode) on a small subset on rank 0 only, # timed after the main loop for a clean CPU state. diff --git a/code/tests/test_offline_stim_decoding.py b/code/tests/test_offline_stim_decoding.py index 7047faa..b231a6e 100644 --- a/code/tests/test_offline_stim_decoding.py +++ b/code/tests/test_offline_stim_decoding.py @@ -772,36 +772,37 @@ def test_helper_handles_t_equals_one(self): self.assertTrue(torch.all(train_x[:, 2] == 0)) -class TestOfflineStimLER(unittest.TestCase): - - def _base_cfg(self, num_samples: int): - return SimpleNamespace( - code="surface", - datapipe="memory", - distance=3, +def _base_cfg(num_samples: int, p_error: float = 0.003, meas_basis_test: str = "both"): + return SimpleNamespace( + code="surface", + datapipe="memory", + distance=3, + n_rounds=3, + data=SimpleNamespace( + error_mode="circuit_level_surface_custom", + code_rotation="XV", + ), + test=SimpleNamespace( + num_samples=num_samples, n_rounds=3, - data=SimpleNamespace( - error_mode="circuit_level_surface_custom", - code_rotation="XV", - ), - test=SimpleNamespace( - num_samples=num_samples, - n_rounds=3, - p_error=0.003, - meas_basis_test="both", - noise_model="none", - latency_num_samples=0, - th_data=0.0, - th_syn=0.0, - sampling_mode="threshold", - temperature=1.0, - dataloader={ - "batch_size": 16, - "num_workers": 0, - "persistent_workers": False, - }, - ), - ) + p_error=p_error, + meas_basis_test=meas_basis_test, + noise_model="none", + latency_num_samples=0, + th_data=0.0, + th_syn=0.0, + sampling_mode="threshold", + temperature=1.0, + dataloader={ + "batch_size": 16, + "num_workers": 0, + "persistent_workers": False, + }, + ), + ) + + +class TestOfflineStimLER(unittest.TestCase): def _run_file_ler(self, root: Path, *, decode_mode: str, model: nn.Module, cfg): old_env = { @@ -834,7 +835,7 @@ def test_pymatching_only_ler_matches_golden_x_and_z(self): root, decode_mode="pymatching_only", model=_UnusedModel(), - cfg=self._base_cfg(96), + cfg=_base_cfg(96), ) for basis in ("X", "Z"): @@ -858,7 +859,7 @@ def test_ising_decoding_pymatching_smoke_matches_baseline_for_zero_correction_mo root, decode_mode="ising_decoding_pymatching", model=_ZeroCorrectionModel(), - cfg=self._base_cfg(32), + cfg=_base_cfg(32), ) for basis in ("X", "Z"): @@ -877,8 +878,7 @@ def test_pymatching_only_single_basis_x_branch(self): root = Path(tmp) artifact = _write_artifact(root, basis="X", shots=64) - cfg = self._base_cfg(64) - cfg.test.meas_basis_test = "X" + cfg = _base_cfg(64, meas_basis_test="X") result = self._run_file_ler( root, @@ -893,36 +893,6 @@ def test_pymatching_only_single_basis_x_branch(self): self.assertEqual(observed, artifact.ler) -def _distributed_ler_cfg(num_samples: int, p_error: float, basis: str): - return SimpleNamespace( - code="surface", - datapipe="memory", - distance=3, - n_rounds=3, - data=SimpleNamespace( - error_mode="circuit_level_surface_custom", - code_rotation="XV", - ), - test=SimpleNamespace( - num_samples=num_samples, - n_rounds=3, - p_error=p_error, - meas_basis_test=basis, - noise_model="none", - latency_num_samples=0, - th_data=0.0, - th_syn=0.0, - sampling_mode="threshold", - temperature=1.0, - dataloader={ - "batch_size": 16, - "num_workers": 0, - "persistent_workers": False, - }, - ), - ) - - def _distributed_ler_worker(rank, world_size, init_file, root, shots, result_path): torch.distributed.init_process_group( backend="gloo", init_method=f"file://{init_file}", rank=rank, world_size=world_size @@ -938,8 +908,9 @@ def _distributed_ler_worker(rank, world_size, init_file, root, shots, result_pat ("ising_decoding_pymatching", _ZeroCorrectionModel()), ): os.environ["PREDECODER_DECODE_MODE"] = decode_mode + os.environ["PREDECODER_DECODE_OUTPUT_DIR"] = str(Path(root) / "out" / decode_mode) results[decode_mode] = count_logical_errors_with_errorbar( - model, dist.device, dist, _distributed_ler_cfg(shots, 0.02, "both") + model, dist.device, dist, _base_cfg(shots, p_error=0.02) ) if rank == 0: Path(result_path).write_text(json.dumps(results)) @@ -957,7 +928,7 @@ def _shard(self, root: Path, num_samples: int, rank: int, world_size: int): mock.patch("torch.distributed.is_initialized", return_value=True), \ mock.patch("torch.distributed.get_rank", return_value=rank), \ mock.patch("torch.distributed.get_world_size", return_value=world_size): - cfg = _distributed_ler_cfg(num_samples, 0.02, "X") + cfg = _base_cfg(num_samples, p_error=0.02, meas_basis_test="X") return DatapipeFactory.create_datapipe_inference(cfg).dets_and_obs def test_ranks_get_disjoint_blocks_of_the_file(self): @@ -979,14 +950,78 @@ def test_file_shorter_than_requested_is_split_across_ranks(self): shards = [self._shard(root, num_samples=1000, rank=r, world_size=2) for r in range(2)] self.assertTrue(torch.equal(torch.cat(shards), full)) + def test_remainder_shots_go_to_the_first_ranks(self): + """10 shots on 4 ranks: 3, 3, 2, 2 in file order, nothing dropped.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=10, p_error=0.02).dets_and_obs + ) + for num_samples in (10 // 4, 1000): + with self.subTest(num_samples=num_samples): + shards = [ + self._shard(root, num_samples=num_samples, rank=r, world_size=4) + for r in range(4) + ] + self.assertEqual([len(shard) for shard in shards], [3, 3, 2, 2]) + self.assertTrue(torch.equal(torch.cat(shards), full)) + + def test_capped_request_takes_equal_blocks_from_the_front(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + full = torch.from_numpy( + _write_artifact(root, basis="X", shots=64, p_error=0.02).dets_and_obs + ) + shards = [self._shard(root, num_samples=8, rank=r, world_size=2) for r in range(2)] + self.assertTrue(torch.equal(torch.cat(shards), full[:16])) + + def test_rank_out_of_range_raises(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_artifact(root, basis="X", shots=16, p_error=0.02) + for rank in (2, -1): + with self.subTest(rank=rank): + with self.assertRaisesRegex(ValueError, "out of range for world_size=2"): + self._shard(root, num_samples=8, rank=rank, world_size=2) + + def test_fewer_shots_than_ranks_raises_on_every_rank(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_artifact(root, basis="X", shots=3, p_error=0.02) + for rank in range(4): + with self.subTest(rank=rank): + with self.assertRaisesRegex(ValueError, "too few to give each of 4 ranks"): + self._shard(root, num_samples=1000, rank=rank, world_size=4) + def test_two_rank_gloo_ler_matches_whole_file(self): - shots = 256 + """Two gloo ranks must report the same counts and write the same decode-output + arrays as one process on the same files. An odd shot count checks that the + remainder shot is decoded.""" + shots = 257 + decode_modes = ( + ("pymatching_only", _UnusedModel()), + ("ising_decoding_pymatching", _ZeroCorrectionModel()), + ) with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) - expected = { - basis: _write_artifact(root, basis=basis, shots=shots, p_error=0.02).errors + artifacts = { + basis: _write_artifact(root, basis=basis, shots=shots, p_error=0.02) for basis in ("X", "Z") } + single = {} + for decode_mode, model in decode_modes: + env = { + "PREDECODER_STIM_SAMPLES_DIR": str(root), + "PREDECODER_TORCH_COMPILE": "0", + "PREDECODER_INFERENCE_NUM_WORKERS": "0", + "PREDECODER_DECODE_MODE": decode_mode, + "PREDECODER_DECODE_OUTPUT_DIR": str(root / "single" / decode_mode), + } + with mock.patch.dict(os.environ, env): + single[decode_mode] = count_logical_errors_with_errorbar( + model, torch.device("cpu"), _DummyDist(), _base_cfg(shots, p_error=0.02) + ) + result_path = root / "results.json" torch.multiprocessing.spawn( _distributed_ler_worker, @@ -995,12 +1030,36 @@ def test_two_rank_gloo_ler_matches_whole_file(self): join=True, ) results = json.loads(result_path.read_text()) + self.assertEqual(set(results), {mode for mode, _ in decode_modes}) + array_names = { + "pymatching_only": ("observables", "pymatching_predictions"), + "ising_decoding_pymatching": + ( + "observables", + "pymatching_predictions", + "ising_decoding_pymatching_predictions", + "predecoder_residual_detectors", + ), + } for decode_mode, result in results.items(): for basis in ("X", "Z"): with self.subTest(decode_mode=decode_mode, basis=basis): + expected = single[decode_mode][basis] self.assertEqual(result[basis]["num shots"], shots) - self.assertEqual(result[basis]["pymatch flips"], expected[basis]) - self.assertEqual(result[basis]["logical errors"], expected[basis]) + for key in ("num shots", "pymatch flips", "logical errors"): + self.assertEqual(result[basis][key], expected[key]) + for name in array_names[decode_mode]: + arr = np.load(root / "out" / decode_mode / f"{basis}_{name}.npy") + self.assertEqual(arr.shape[0], shots) + np.testing.assert_array_equal( + arr, + np.load(root / "single" / decode_mode / f"{basis}_{name}.npy"), + ) + num_obs = artifacts[basis].predictions.shape[1] + np.testing.assert_array_equal( + np.load(root / "out" / decode_mode / f"{basis}_observables.npy"), + artifacts[basis].dets_and_obs[:, -num_obs:], + ) if __name__ == "__main__": From 928fb47b9fc986ca674a5d8598079fb7f33b096f Mon Sep 17 00:00:00 2001 From: 0z5a Date: Tue, 17 Nov 2026 20:20:02 +0100 Subject: [PATCH 3/3] Cap the global Stim shot count before distributing rows Signed-off-by: 0z5a --- README.md | 17 ++++------ code/data/datapipe_stim.py | 40 +++++++++--------------- code/data/factory.py | 11 ++++--- code/evaluation/failure_analysis.py | 4 ++- code/evaluation/logical_error_rate.py | 8 +++-- code/tests/test_offline_stim_decoding.py | 21 ++++++++----- 6 files changed, 50 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 3e8aa2e..9db84e7 100644 --- a/README.md +++ b/README.md @@ -649,16 +649,11 @@ a path that does not yet exist. reads the same files. Each rank decodes its own contiguous block of shots, in rank order, and the reported counts and LER are summed over all ranks: -* If `test.num_samples // world_size` is at least `file shots // world_size` - (for example when `test.num_samples` is the file size or larger), the whole - file is decoded. The remainder `file shots % world_size` goes to the first - ranks, one extra shot each. When `test.num_samples` is at least the file - size, the result therefore matches a single-GPU run on the same file. -* Otherwise each rank decodes `test.num_samples // world_size` shots from the - front of the file, and rank 0 prints how many of the file's shots are - decoded. -* A file with fewer shots than ranks is rejected; run with fewer GPUs - (`GPUS=1`). +* A positive `test.num_samples` limits the total number of shots per file. + Each rank decodes an equal share of the selected prefix, with one extra + shot on each of the first ranks when the total is uneven. +* The selected prefix must have at least one shot per rank. Use fewer GPUs + for smaller samples. The `PREDECODER_DECODE_OUTPUT_DIR` arrays are gathered on rank 0 in rank order before they are written, so each array has one row per decoded shot, in @@ -1214,7 +1209,7 @@ Key scalars (as shown in TensorBoard): - **`Metrics/LER`**: Logical Error Rate on the evaluation target (computed during training-time evaluation). Lower is better. - Averaging: computed over `cfg.test.num_samples` Monte Carlo shots **per basis** (X and Z). - Default: `cfg.test.num_samples = 262144` (hardcoded for the current public release). - - Distributed: each rank uses `cfg.test.num_samples // world_size` shots per basis (any remainder is dropped). + - Distributed on generated samples: each rank uses `cfg.test.num_samples // world_size` shots per basis (any remainder is dropped). - **`Metrics/LER_Reduction_Factor`**: Ratio of post-predecoder LER to baseline LER (a “relative improvement” factor). `>1` means improvement. If both are 0, we log `1.0`. - Averaging: derived from the same LER evaluation run (same shot count as `Metrics/LER`). - **`Metrics/PyMatching_Speedup`**: Average PyMatching speedup from the pre-decoder: `latency_baseline / latency_after`. `>1` means faster decoding of PyMatching after pre-decoding. diff --git a/code/data/datapipe_stim.py b/code/data/datapipe_stim.py index 1b03f28..5bbf64b 100644 --- a/code/data/datapipe_stim.py +++ b/code/data/datapipe_stim.py @@ -215,36 +215,28 @@ def __getitem__(self, idx): def _stim_file_shard( - *, file_shots: int, requested_per_rank: int, rank: int, world_size: int, samples_path + *, file_shots: int, requested_total: int, rank: int, world_size: int, samples_path ) -> tuple: """Return the ``[start, stop)`` rows of a Stim sample file that ``rank`` decodes. - Callers pass the per-rank shot count (``num_samples // world_size``) and every - rank opens the same file, so each rank gets a disjoint contiguous block, in rank - order. If the request does not cap the block, the whole file is split and the - first ``file_shots % world_size`` ranks take one extra shot; otherwise every rank - takes ``requested_per_rank`` shots from the front of the file. The checks only - depend on values that are equal on every rank, so all ranks raise together. + Apply the global shot limit before splitting the selected prefix of the file. + The first ranks take one extra row when the total is uneven. """ world_size = max(int(world_size), 1) rank = int(rank) if not 0 <= rank < world_size: raise ValueError(f"rank={rank} is out of range for world_size={world_size}") - if world_size > 1 and file_shots < world_size: + total = min(requested_total, file_shots) if requested_total > 0 else file_shots + if world_size > 1 and total < world_size: raise ValueError( - f"Stim sample file {samples_path} has {file_shots} shots, " + f"Stim sample file {samples_path} has {total} selected shots, " f"too few to give each of {world_size} ranks at least one shot" ) - base, remainder = divmod(file_shots, world_size) - if requested_per_rank <= 0 or requested_per_rank >= base: - start = rank * base + min(rank, remainder) - return start, start + base + (1 if rank < remainder else 0) - if rank == 0 and world_size > 1: - print( - f"[Stim samples] Decoding {requested_per_rank * world_size} of {file_shots} " - f"shots in {samples_path} ({requested_per_rank} per rank on {world_size} ranks)" - ) - return rank * requested_per_rank, (rank + 1) * requested_per_rank + if rank == 0 and world_size > 1 and total < file_shots: + print(f"[Stim samples] Decoding {total} of {file_shots} shots in {samples_path}") + count, remainder = divmod(total, world_size) + start = rank * count + min(rank, remainder) + return start, start + count + int(rank < remainder) class QCDataPipePreDecoder_Memory_from_stim_file(Dataset): @@ -275,12 +267,8 @@ class QCDataPipePreDecoder_Memory_from_stim_file(Dataset): ``False`` downgrades the failure to a :class:`UserWarning`. rank, world_size: Distributed shard of the file to expose. Every rank reads the same file and keeps its own contiguous block, in rank - order, so no shot is decoded twice. ``num_samples`` is the per-rank - count. When it is at least ``file_shots // world_size`` (or not - positive), the whole file is split: the first - ``file_shots % world_size`` ranks take one extra shot. Otherwise each - rank takes ``num_samples`` shots from the front of the file. The - defaults (``0``, ``1``) keep the previous single-process behaviour. + order. A positive ``num_samples`` limits the total across all ranks. + The first ranks receive one extra row when the split is uneven. """ def __init__( @@ -355,7 +343,7 @@ def __init__( ) start, stop = _stim_file_shard( file_shots=int(dets_and_obs.shape[0]), - requested_per_rank=self.requested_num_samples, + requested_total=self.requested_num_samples, rank=rank, world_size=world_size, samples_path=samples_path, diff --git a/code/data/factory.py b/code/data/factory.py index 1f8bbd9..36e2802 100644 --- a/code/data/factory.py +++ b/code/data/factory.py @@ -41,10 +41,10 @@ def create_datapipe(cfg): raise ValueError("Invalid datapipe code") @staticmethod - def create_datapipe_inference(cfg): + def create_datapipe_inference(cfg, *, file_shot_limit=None): """Create datapipe for inference using Stim.""" if cfg.code == "surface": - return DatapipeFactory._create_surface_datapipe_inference(cfg) + return DatapipeFactory._create_surface_datapipe_inference(cfg, file_shot_limit) elif cfg.code == "color": return DatapipeFactory._create_color_datapipe_inference(cfg) else: @@ -74,7 +74,7 @@ def _create_color_datapipe(cfg): raise ValueError(f"Datapipe not implemented: {cfg.datapipe}") @staticmethod - def _create_surface_datapipe_inference(cfg): + def _create_surface_datapipe_inference(cfg, file_shot_limit=None): """ Datapipe for inference using Stim. """ @@ -119,8 +119,11 @@ def _create_surface_datapipe_inference(cfg): # shapes because they only depend on (distance, n_rounds). measure_basis = "X" + num_samples = cfg.test.num_samples dataset_kwargs: dict = {} if stim_samples_dir: + if file_shot_limit is not None: + num_samples = file_shot_limit dataset_cls = QCDataPipePreDecoder_Memory_from_stim_file dataset_kwargs["stim_samples_dir"] = stim_samples_dir # Every rank reads the same file; shard it so ranks decode disjoint shots. @@ -137,7 +140,7 @@ def _create_surface_datapipe_inference(cfg): test_dataset = dataset_cls( distance=cfg.distance, n_rounds=cfg.n_rounds, - num_samples=cfg.test.num_samples, + num_samples=num_samples, error_mode=error_mode_value, p_error=cfg.test.p_error, measure_basis=measure_basis, diff --git a/code/evaluation/failure_analysis.py b/code/evaluation/failure_analysis.py index 0c16e4e..ee63cf7 100644 --- a/code/evaluation/failure_analysis.py +++ b/code/evaluation/failure_analysis.py @@ -932,7 +932,9 @@ def decoder_ablation_study(model, device, dist, cfg): cfg_copy = deepcopy(cfg) cfg_copy.test.num_samples = samples_per_gpu cfg_copy.test.meas_basis_test = basis - test_dataset = DatapipeFactory.create_datapipe_inference(cfg_copy) + test_dataset = DatapipeFactory.create_datapipe_inference( + cfg_copy, file_shot_limit=total_samples + ) finally: torch.set_rng_state(torch_state) if cuda_state is not None: diff --git a/code/evaluation/logical_error_rate.py b/code/evaluation/logical_error_rate.py index ad35267..149864c 100644 --- a/code/evaluation/logical_error_rate.py +++ b/code/evaluation/logical_error_rate.py @@ -1028,7 +1028,9 @@ def run_inference_and_decode_pre_decoder_memory(model, device, dist, cfg) -> dic # Create datapipe with reduced sample count per GPU cfg_copy = deepcopy(cfg) cfg_copy.test.num_samples = samples_per_gpu - test_dataset = DatapipeFactory.create_datapipe_inference(cfg_copy) + test_dataset = DatapipeFactory.create_datapipe_inference( + cfg_copy, file_shot_limit=total_samples + ) finally: # Restore random states torch.set_rng_state(torch_state) @@ -1741,7 +1743,9 @@ def compute_syndrome_density_reduction(model, device, dist, cfg) -> dict: from copy import deepcopy cfg_copy = deepcopy(cfg) cfg_copy.test.num_samples = samples_per_gpu - test_dataset = DatapipeFactory.create_datapipe_inference(cfg_copy) + test_dataset = DatapipeFactory.create_datapipe_inference( + cfg_copy, file_shot_limit=total_samples + ) if rank == 0: print(f"[Syndrome Density] Each GPU processes {len(test_dataset)} samples") diff --git a/code/tests/test_offline_stim_decoding.py b/code/tests/test_offline_stim_decoding.py index b231a6e..0a44aca 100644 --- a/code/tests/test_offline_stim_decoding.py +++ b/code/tests/test_offline_stim_decoding.py @@ -920,7 +920,7 @@ def _distributed_ler_worker(rank, world_size, init_file, root, shots, result_pat class TestOfflineStimDistributed(unittest.TestCase): """Under torchrun every rank opens the same ``.dets`` file. The LER loop - asks each rank for ``num_samples // world_size`` shots and all-reduces the + splits the requested shots across ranks and all-reduces the counts, so each rank must decode its own block of the file.""" def _shard(self, root: Path, num_samples: int, rank: int, world_size: int): @@ -937,7 +937,7 @@ def test_ranks_get_disjoint_blocks_of_the_file(self): full = torch.from_numpy( _write_artifact(root, basis="X", shots=64, p_error=0.02).dets_and_obs ) - shards = [self._shard(root, num_samples=32, rank=r, world_size=2) for r in range(2)] + shards = [self._shard(root, num_samples=64, rank=r, world_size=2) for r in range(2)] self.assertTrue(torch.equal(shards[0], full[:32])) self.assertTrue(torch.equal(shards[1], full[32:])) @@ -957,7 +957,7 @@ def test_remainder_shots_go_to_the_first_ranks(self): full = torch.from_numpy( _write_artifact(root, basis="X", shots=10, p_error=0.02).dets_and_obs ) - for num_samples in (10 // 4, 1000): + for num_samples in (10, 1000): with self.subTest(num_samples=num_samples): shards = [ self._shard(root, num_samples=num_samples, rank=r, world_size=4) @@ -966,14 +966,21 @@ def test_remainder_shots_go_to_the_first_ranks(self): self.assertEqual([len(shard) for shard in shards], [3, 3, 2, 2]) self.assertTrue(torch.equal(torch.cat(shards), full)) - def test_capped_request_takes_equal_blocks_from_the_front(self): + def test_global_cap_is_applied_before_sharding(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) full = torch.from_numpy( - _write_artifact(root, basis="X", shots=64, p_error=0.02).dets_and_obs + _write_artifact(root, basis="X", shots=11, p_error=0.02).dets_and_obs ) - shards = [self._shard(root, num_samples=8, rank=r, world_size=2) for r in range(2)] - self.assertTrue(torch.equal(torch.cat(shards), full[:16])) + for cap, sizes in ((10, [5, 5]), (9, [5, 4]), (0, [6, 5])): + with self.subTest(cap=cap): + shards = [ + self._shard(root, num_samples=cap, rank=r, world_size=2) for r in range(2) + ] + self.assertEqual([len(shard) for shard in shards], sizes) + self.assertTrue(torch.equal(torch.cat(shards), full[:sum(sizes)])) + with self.assertRaisesRegex(ValueError, "too few"): + self._shard(root, num_samples=1, rank=0, world_size=2) def test_rank_out_of_range_raises(self): with tempfile.TemporaryDirectory() as tmp: