diff --git a/cuda_core/tests/example_tests/test_sample_verification.py b/cuda_core/tests/example_tests/test_sample_verification.py index 4130632c8d3..464b8284575 100644 --- a/cuda_core/tests/example_tests/test_sample_verification.py +++ b/cuda_core/tests/example_tests/test_sample_verification.py @@ -4,7 +4,10 @@ from __future__ import annotations import ast +import importlib.util +import sys from pathlib import Path +from types import SimpleNamespace import numpy as np import pytest @@ -99,6 +102,83 @@ def test_unsupported_core_sample_paths_return_waiver( assert len(waiver_returns) == 1 +def _load_simple_p2p(monkeypatch: pytest.MonkeyPatch): + sample_path = REPO_ROOT / "samples" / "cuda_core" / "simpleP2P" / "simpleP2P.py" + spec = importlib.util.spec_from_file_location("_test_simple_p2p", sample_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + with monkeypatch.context() as import_ctx: + import_ctx.setattr(sys, "path", list(sys.path)) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("unsupported_gpu", "unsupported_capability"), + [ + (0, "memory_pools_supported"), + (1, "memory_pools_supported"), + (0, "host_memory_pools_supported"), + (1, "host_memory_pools_supported"), + ], +) +@pytest.mark.agent_authored(model="gemini-4-carbon-high") +def test_simple_p2p_waives_before_allocation_without_memory_pool_support( + monkeypatch: pytest.MonkeyPatch, + unsupported_gpu: int, + unsupported_capability: str, +) -> None: + """Missing device or host mempool support on any selected GPU must waive before allocating (NVBug 7081727).""" + module = _load_simple_p2p(monkeypatch) + + # GPU 0 is non-P2P so the sample selects the (1, 2) pair rather than hardcoded ordinals (0, 1). + p2p_pair = (1, 2) + target_device_id = p2p_pair[unsupported_gpu] + devices = [] + for device_id in range(3): + props = { + "memory_pools_supported": True, + "host_memory_pools_supported": True, + } + if device_id == target_device_id: + props[unsupported_capability] = False + devices.append( + SimpleNamespace( + device_id=device_id, + name=f"FakeGPU{device_id}", + properties=SimpleNamespace(**props), + can_access_peer=lambda other, dev_id=device_id: ( + dev_id in p2p_pair and other.device_id in p2p_pair and dev_id != other.device_id + ), + set_current=lambda: None, + ) + ) + + class _FakeDeviceFactory: + def __new__(cls, device_id: int): + return devices[int(device_id)] + + @staticmethod + def get_all_devices(): + return list(devices) + + constructed_resources: list[str] = [] + + def _record_construction(kind: str): + def _fail(*args, **kwargs): + constructed_resources.append(kind) + raise AssertionError(f"{kind} must not be constructed when memory pools are unsupported") + + return _fail + + monkeypatch.setattr(module, "Device", _FakeDeviceFactory) + monkeypatch.setattr(module, "DeviceMemoryResource", _record_construction("DeviceMemoryResource")) + monkeypatch.setattr(module, "PinnedMemoryResource", _record_construction("PinnedMemoryResource")) + + assert module.run(num_elements=16) == module.EXIT_WAIVED + assert constructed_resources == [] + + @pytest.mark.agent_authored(model="gpt-5") def test_verify_array_result_or_raise_enforces_match(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.syspath_prepend(str(UTILITIES_DIR)) diff --git a/samples/cuda_core/simpleP2P/README.md b/samples/cuda_core/simpleP2P/README.md index 3541df9dbb6..b37a9f929df 100644 --- a/samples/cuda_core/simpleP2P/README.md +++ b/samples/cuda_core/simpleP2P/README.md @@ -67,6 +67,7 @@ This sample demonstrates peer-to-peer (P2P) memory access between multiple GPUs ### Requirements - **Two or more GPUs**: System must have multiple CUDA-capable GPUs - **P2P support**: GPUs must be P2P-capable (check with `can_access_peer()`) +- **Memory pools**: The GPUs must support device and host memory pools (check with `device.properties.memory_pools_supported` and `device.properties.host_memory_pools_supported`) - **PCIe topology**: Usually requires GPUs on same PCIe root complex - **Platform support**: Not available on Mac OSX, limited on ARM platforms @@ -84,8 +85,9 @@ This sample demonstrates peer-to-peer (P2P) memory access between multiple GPUs 3. **CUDA Toolkit 13.0+** installed on your system 4. **Python 3.11 or newer** 5. **Proper PCIe topology** (GPUs should be on same PCIe root complex for best performance) +6. **CUDA memory pools** support on both GPUs (device and host memory pools); used to allocate the peer-accessible buffers -**Note**: This sample will gracefully exit if fewer than 2 GPUs are detected or if P2P is not supported between any GPU pair. +**Note**: This sample will gracefully exit if fewer than 2 GPUs are detected, if P2P is not supported between any GPU pair, or if the GPUs do not support CUDA memory pools (for example, GPUs in TCC mode). **Install packages:** ```bash diff --git a/samples/cuda_core/simpleP2P/simpleP2P.py b/samples/cuda_core/simpleP2P/simpleP2P.py index c0f5678d856..b3f14727211 100644 --- a/samples/cuda_core/simpleP2P/simpleP2P.py +++ b/samples/cuda_core/simpleP2P/simpleP2P.py @@ -152,6 +152,18 @@ def run(num_elements=1024 * 1024 * 16): print(f"\nUsing GPU{gpuid[0]} ({dev0.name}) and GPU{gpuid[1]} ({dev1.name})") + # The buffers below are allocated from the device and host memory pools, so + # that peer access can be granted with cuMemPoolSetAccess. Configurations + # without memory pools (for example devices in TCC mode) cannot run this + # sample, so waive instead of failing while allocating. + for device, label in ((dev0, f"GPU{gpuid[0]}"), (dev1, f"GPU{gpuid[1]}")): + if not device.properties.memory_pools_supported: + print(f"\nDeviceMemoryResource requires CUDA memory pools, which are not supported on {label}. Waiving.") + return EXIT_WAIVED + if not device.properties.host_memory_pools_supported: + print(f"\nPinnedMemoryResource requires host memory pools, which are not supported on {label}. Waiving.") + return EXIT_WAIVED + # Allocate buffers with P2P access buf_size = num_elements * np.dtype(np.float32).itemsize print(f"\nAllocating buffers ({int(buf_size / 1024 / 1024)}MB on GPU{gpuid[0]}, GPU{gpuid[1]} and CPU Host)...")