From bdcead5aa6d5056f49987143574d495c73cd3613 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 13 Nov 2026 06:51:18 +0000 Subject: [PATCH 1/2] fix(samples): waive simpleP2P without CUDA memory pool support The peer-accessible buffers in this sample are allocated from the device and host memory pools, because peer access is granted with cuMemPoolSetAccess. On configurations without memory pools (for example GPUs in TCC mode on R580), DeviceMemoryResource and PinnedMemoryResource raise CUDA_ERROR_NOT_SUPPORTED, so the sample exited with an unhandled exception before performing any peer transfer. Check memory_pools_supported and host_memory_pools_supported for the selected GPU pair before allocating and return EXIT_WAIVED instead. This matches the existing memory pool waivers in the streamingCopyComputeOverlap and copyImageArraytoGPU samples. Document the prerequisite in the README and add a sample verification test asserting that the capability checks precede the allocations. NVBug 7081727 Signed-off-by: Yimo Jiang --- .../example_tests/test_sample_verification.py | 37 +++++++++++++++++++ samples/cuda_core/simpleP2P/README.md | 4 +- samples/cuda_core/simpleP2P/simpleP2P.py | 12 ++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/cuda_core/tests/example_tests/test_sample_verification.py b/cuda_core/tests/example_tests/test_sample_verification.py index 4130632c8d3..20cf8881bcb 100644 --- a/cuda_core/tests/example_tests/test_sample_verification.py +++ b/cuda_core/tests/example_tests/test_sample_verification.py @@ -99,6 +99,43 @@ def test_unsupported_core_sample_paths_return_waiver( assert len(waiver_returns) == 1 +@pytest.mark.agent_authored(model="deepseek-v4.1-flash-1017") +def test_simple_p2p_gates_allocation_on_memory_pool_support() -> None: + """Allocations must be gated on mempool support instead of raising (NVBug 7081727).""" + sample_path = REPO_ROOT / "samples" / "cuda_core" / "simpleP2P" / "simpleP2P.py" + tree = ast.parse(sample_path.read_text(encoding="utf-8"), filename=str(sample_path)) + gated_attributes = {"memory_pools_supported", "host_memory_pools_supported"} + + gates = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.If) + and isinstance(node.test, ast.UnaryOp) + and isinstance(node.test.op, ast.Not) + and isinstance(node.test.operand, ast.Attribute) + and node.test.operand.attr in gated_attributes + and any( + isinstance(statement, ast.Return) + and isinstance(statement.value, ast.Name) + and statement.value.id == "EXIT_WAIVED" + for statement in node.body + ) + ] + assert {node.test.operand.attr for node in gates} == gated_attributes + + allocation_lines = [ + node.lineno + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in {"DeviceMemoryResource", "PinnedMemoryResource"} + ] + assert allocation_lines + assert max(node.lineno for node in gates) < min(allocation_lines), ( + "memory pool support must be checked before allocating" + ) + + @pytest.mark.agent_authored(model="gpt-5") def test_verify_array_result_or_raise_enforces_match(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.syspath_prepend(str(UTILITIES_DIR)) diff --git a/samples/cuda_core/simpleP2P/README.md b/samples/cuda_core/simpleP2P/README.md index 3541df9dbb6..b37a9f929df 100644 --- a/samples/cuda_core/simpleP2P/README.md +++ b/samples/cuda_core/simpleP2P/README.md @@ -67,6 +67,7 @@ This sample demonstrates peer-to-peer (P2P) memory access between multiple GPUs ### Requirements - **Two or more GPUs**: System must have multiple CUDA-capable GPUs - **P2P support**: GPUs must be P2P-capable (check with `can_access_peer()`) +- **Memory pools**: The GPUs must support device and host memory pools (check with `device.properties.memory_pools_supported` and `device.properties.host_memory_pools_supported`) - **PCIe topology**: Usually requires GPUs on same PCIe root complex - **Platform support**: Not available on Mac OSX, limited on ARM platforms @@ -84,8 +85,9 @@ This sample demonstrates peer-to-peer (P2P) memory access between multiple GPUs 3. **CUDA Toolkit 13.0+** installed on your system 4. **Python 3.11 or newer** 5. **Proper PCIe topology** (GPUs should be on same PCIe root complex for best performance) +6. **CUDA memory pools** support on both GPUs (device and host memory pools); used to allocate the peer-accessible buffers -**Note**: This sample will gracefully exit if fewer than 2 GPUs are detected or if P2P is not supported between any GPU pair. +**Note**: This sample will gracefully exit if fewer than 2 GPUs are detected, if P2P is not supported between any GPU pair, or if the GPUs do not support CUDA memory pools (for example, GPUs in TCC mode). **Install packages:** ```bash diff --git a/samples/cuda_core/simpleP2P/simpleP2P.py b/samples/cuda_core/simpleP2P/simpleP2P.py index c0f5678d856..b3f14727211 100644 --- a/samples/cuda_core/simpleP2P/simpleP2P.py +++ b/samples/cuda_core/simpleP2P/simpleP2P.py @@ -152,6 +152,18 @@ def run(num_elements=1024 * 1024 * 16): print(f"\nUsing GPU{gpuid[0]} ({dev0.name}) and GPU{gpuid[1]} ({dev1.name})") + # The buffers below are allocated from the device and host memory pools, so + # that peer access can be granted with cuMemPoolSetAccess. Configurations + # without memory pools (for example devices in TCC mode) cannot run this + # sample, so waive instead of failing while allocating. + for device, label in ((dev0, f"GPU{gpuid[0]}"), (dev1, f"GPU{gpuid[1]}")): + if not device.properties.memory_pools_supported: + print(f"\nDeviceMemoryResource requires CUDA memory pools, which are not supported on {label}. Waiving.") + return EXIT_WAIVED + if not device.properties.host_memory_pools_supported: + print(f"\nPinnedMemoryResource requires host memory pools, which are not supported on {label}. Waiving.") + return EXIT_WAIVED + # Allocate buffers with P2P access buf_size = num_elements * np.dtype(np.float32).itemsize print(f"\nAllocating buffers ({int(buf_size / 1024 / 1024)}MB on GPU{gpuid[0]}, GPU{gpuid[1]} and CPU Host)...") From e6019e3323cce880a06a2c1166cacf7220142142 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 13 Nov 2026 07:05:24 +0000 Subject: [PATCH 2/2] test(samples): assert simpleP2P waiver behavior across selected GPUs Replace the AST-only check with behavioral parametrization over each selected GPU and each memory pool capability, verifying EXIT_WAIVED is returned and no memory resource is constructed. Signed-off-by: Yimo Jiang --- .../example_tests/test_sample_verification.py | 105 ++++++++++++------ 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/cuda_core/tests/example_tests/test_sample_verification.py b/cuda_core/tests/example_tests/test_sample_verification.py index 20cf8881bcb..464b8284575 100644 --- a/cuda_core/tests/example_tests/test_sample_verification.py +++ b/cuda_core/tests/example_tests/test_sample_verification.py @@ -4,7 +4,10 @@ from __future__ import annotations import ast +import importlib.util +import sys from pathlib import Path +from types import SimpleNamespace import numpy as np import pytest @@ -99,41 +102,81 @@ def test_unsupported_core_sample_paths_return_waiver( assert len(waiver_returns) == 1 -@pytest.mark.agent_authored(model="deepseek-v4.1-flash-1017") -def test_simple_p2p_gates_allocation_on_memory_pool_support() -> None: - """Allocations must be gated on mempool support instead of raising (NVBug 7081727).""" +def _load_simple_p2p(monkeypatch: pytest.MonkeyPatch): sample_path = REPO_ROOT / "samples" / "cuda_core" / "simpleP2P" / "simpleP2P.py" - tree = ast.parse(sample_path.read_text(encoding="utf-8"), filename=str(sample_path)) - gated_attributes = {"memory_pools_supported", "host_memory_pools_supported"} + spec = importlib.util.spec_from_file_location("_test_simple_p2p", sample_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + with monkeypatch.context() as import_ctx: + import_ctx.setattr(sys, "path", list(sys.path)) + spec.loader.exec_module(module) + return module - gates = [ - node - for node in ast.walk(tree) - if isinstance(node, ast.If) - and isinstance(node.test, ast.UnaryOp) - and isinstance(node.test.op, ast.Not) - and isinstance(node.test.operand, ast.Attribute) - and node.test.operand.attr in gated_attributes - and any( - isinstance(statement, ast.Return) - and isinstance(statement.value, ast.Name) - and statement.value.id == "EXIT_WAIVED" - for statement in node.body + +@pytest.mark.parametrize( + ("unsupported_gpu", "unsupported_capability"), + [ + (0, "memory_pools_supported"), + (1, "memory_pools_supported"), + (0, "host_memory_pools_supported"), + (1, "host_memory_pools_supported"), + ], +) +@pytest.mark.agent_authored(model="gemini-4-carbon-high") +def test_simple_p2p_waives_before_allocation_without_memory_pool_support( + monkeypatch: pytest.MonkeyPatch, + unsupported_gpu: int, + unsupported_capability: str, +) -> None: + """Missing device or host mempool support on any selected GPU must waive before allocating (NVBug 7081727).""" + module = _load_simple_p2p(monkeypatch) + + # GPU 0 is non-P2P so the sample selects the (1, 2) pair rather than hardcoded ordinals (0, 1). + p2p_pair = (1, 2) + target_device_id = p2p_pair[unsupported_gpu] + devices = [] + for device_id in range(3): + props = { + "memory_pools_supported": True, + "host_memory_pools_supported": True, + } + if device_id == target_device_id: + props[unsupported_capability] = False + devices.append( + SimpleNamespace( + device_id=device_id, + name=f"FakeGPU{device_id}", + properties=SimpleNamespace(**props), + can_access_peer=lambda other, dev_id=device_id: ( + dev_id in p2p_pair and other.device_id in p2p_pair and dev_id != other.device_id + ), + set_current=lambda: None, + ) ) - ] - assert {node.test.operand.attr for node in gates} == gated_attributes - allocation_lines = [ - node.lineno - for node in ast.walk(tree) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id in {"DeviceMemoryResource", "PinnedMemoryResource"} - ] - assert allocation_lines - assert max(node.lineno for node in gates) < min(allocation_lines), ( - "memory pool support must be checked before allocating" - ) + class _FakeDeviceFactory: + def __new__(cls, device_id: int): + return devices[int(device_id)] + + @staticmethod + def get_all_devices(): + return list(devices) + + constructed_resources: list[str] = [] + + def _record_construction(kind: str): + def _fail(*args, **kwargs): + constructed_resources.append(kind) + raise AssertionError(f"{kind} must not be constructed when memory pools are unsupported") + + return _fail + + monkeypatch.setattr(module, "Device", _FakeDeviceFactory) + monkeypatch.setattr(module, "DeviceMemoryResource", _record_construction("DeviceMemoryResource")) + monkeypatch.setattr(module, "PinnedMemoryResource", _record_construction("PinnedMemoryResource")) + + assert module.run(num_elements=16) == module.EXIT_WAIVED + assert constructed_resources == [] @pytest.mark.agent_authored(model="gpt-5")