Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions cuda_core/tests/example_tests/test_sample_verification.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
from __future__ import annotations

import ast
import importlib.util
import sys
from pathlib import Path
from types import SimpleNamespace

import numpy as np
import pytest
Expand Down Expand Up @@ -99,6 +102,83 @@ def test_unsupported_core_sample_paths_return_waiver(
assert len(waiver_returns) == 1


def _load_simple_p2p(monkeypatch: pytest.MonkeyPatch):
sample_path = REPO_ROOT / "samples" / "cuda_core" / "simpleP2P" / "simpleP2P.py"
spec = importlib.util.spec_from_file_location("_test_simple_p2p", sample_path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
with monkeypatch.context() as import_ctx:
import_ctx.setattr(sys, "path", list(sys.path))
spec.loader.exec_module(module)
return module


@pytest.mark.parametrize(
("unsupported_gpu", "unsupported_capability"),
[
(0, "memory_pools_supported"),
(1, "memory_pools_supported"),
(0, "host_memory_pools_supported"),
(1, "host_memory_pools_supported"),
],
)
@pytest.mark.agent_authored(model="gemini-4-carbon-high")
def test_simple_p2p_waives_before_allocation_without_memory_pool_support(
monkeypatch: pytest.MonkeyPatch,
unsupported_gpu: int,
unsupported_capability: str,
) -> None:
"""Missing device or host mempool support on any selected GPU must waive before allocating (NVBug 7081727)."""
module = _load_simple_p2p(monkeypatch)

# GPU 0 is non-P2P so the sample selects the (1, 2) pair rather than hardcoded ordinals (0, 1).
p2p_pair = (1, 2)
target_device_id = p2p_pair[unsupported_gpu]
devices = []
for device_id in range(3):
props = {
"memory_pools_supported": True,
"host_memory_pools_supported": True,
}
if device_id == target_device_id:
props[unsupported_capability] = False
devices.append(
SimpleNamespace(
device_id=device_id,
name=f"FakeGPU{device_id}",
properties=SimpleNamespace(**props),
can_access_peer=lambda other, dev_id=device_id: (
dev_id in p2p_pair and other.device_id in p2p_pair and dev_id != other.device_id
),
set_current=lambda: None,
)
)

class _FakeDeviceFactory:
def __new__(cls, device_id: int):
return devices[int(device_id)]

@staticmethod
def get_all_devices():
return list(devices)

constructed_resources: list[str] = []

def _record_construction(kind: str):
def _fail(*args, **kwargs):
constructed_resources.append(kind)
raise AssertionError(f"{kind} must not be constructed when memory pools are unsupported")

return _fail

monkeypatch.setattr(module, "Device", _FakeDeviceFactory)
monkeypatch.setattr(module, "DeviceMemoryResource", _record_construction("DeviceMemoryResource"))
monkeypatch.setattr(module, "PinnedMemoryResource", _record_construction("PinnedMemoryResource"))

assert module.run(num_elements=16) == module.EXIT_WAIVED
assert constructed_resources == []


@pytest.mark.agent_authored(model="gpt-5")
def test_verify_array_result_or_raise_enforces_match(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.syspath_prepend(str(UTILITIES_DIR))
Expand Down
4 changes: 3 additions & 1 deletion samples/cuda_core/simpleP2P/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ This sample demonstrates peer-to-peer (P2P) memory access between multiple GPUs
### Requirements
- **Two or more GPUs**: System must have multiple CUDA-capable GPUs
- **P2P support**: GPUs must be P2P-capable (check with `can_access_peer()`)
- **Memory pools**: The GPUs must support device and host memory pools (check with `device.properties.memory_pools_supported` and `device.properties.host_memory_pools_supported`)
- **PCIe topology**: Usually requires GPUs on same PCIe root complex
- **Platform support**: Not available on Mac OSX, limited on ARM platforms

Expand All @@ -84,8 +85,9 @@ This sample demonstrates peer-to-peer (P2P) memory access between multiple GPUs
3. **CUDA Toolkit 13.0+** installed on your system
4. **Python 3.11 or newer**
5. **Proper PCIe topology** (GPUs should be on same PCIe root complex for best performance)
6. **CUDA memory pools** support on both GPUs (device and host memory pools); used to allocate the peer-accessible buffers

**Note**: This sample will gracefully exit if fewer than 2 GPUs are detected or if P2P is not supported between any GPU pair.
**Note**: This sample will gracefully exit if fewer than 2 GPUs are detected, if P2P is not supported between any GPU pair, or if the GPUs do not support CUDA memory pools (for example, GPUs in TCC mode).

**Install packages:**
```bash
Expand Down
12 changes: 12 additions & 0 deletions samples/cuda_core/simpleP2P/simpleP2P.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,18 @@ def run(num_elements=1024 * 1024 * 16):

print(f"\nUsing GPU{gpuid[0]} ({dev0.name}) and GPU{gpuid[1]} ({dev1.name})")

# The buffers below are allocated from the device and host memory pools, so
# that peer access can be granted with cuMemPoolSetAccess. Configurations
# without memory pools (for example devices in TCC mode) cannot run this
# sample, so waive instead of failing while allocating.
for device, label in ((dev0, f"GPU{gpuid[0]}"), (dev1, f"GPU{gpuid[1]}")):
if not device.properties.memory_pools_supported:
print(f"\nDeviceMemoryResource requires CUDA memory pools, which are not supported on {label}. Waiving.")
return EXIT_WAIVED
if not device.properties.host_memory_pools_supported:
print(f"\nPinnedMemoryResource requires host memory pools, which are not supported on {label}. Waiving.")
return EXIT_WAIVED

# Allocate buffers with P2P access
buf_size = num_elements * np.dtype(np.float32).itemsize
print(f"\nAllocating buffers ({int(buf_size / 1024 / 1024)}MB on GPU{gpuid[0]}, GPU{gpuid[1]} and CPU Host)...")
Expand Down
Loading