From 3e8208e69793ff06a8ec675e3cf082eaf9db27a1 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Tue, 10 Nov 2026 15:02:55 -0800 Subject: [PATCH 1/8] [doc-only] docs(cuda.core): fix the floor-move wording; advise against pinning cuda-bindings (#3441) * docs(cuda.core): drop the Breaking Changes section name from the floor policy The release notes no longer have a Breaking Changes section for a floor move; 1.3.0 lists it under Highlights. Say only that the release notes list every move. Co-Authored-By: Claude Opus 5.5 * docs(cuda.core): advise against pinning cuda-bindings next to cuda-core Users who pin cuda-bindings themselves can conflict with the floor of a later cuda-core release. Tell them to depend on cuda-core with the cu12 or cu13 extra and let cuda-core choose the cuda-bindings version. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit 83a34225095969e31b38c619c4b7b7b1ceae16f3) --- .github/RELEASE-core.md | 3 +-- cuda_core/AGENTS.md | 6 +++--- cuda_core/docs/source/install.rst | 5 +++++ cuda_core/docs/source/support.rst | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/RELEASE-core.md b/.github/RELEASE-core.md index 45c49461416..05b058a9813 100644 --- a/.github/RELEASE-core.md +++ b/.github/RELEASE-core.md @@ -101,8 +101,7 @@ cuda-bindings floor" in `cuda_core/AGENTS.md`. Check the floors instead: `cuda-bindings==`, and no "predates ... headers" line. Standard CI does not install the CUDA 12 floor: its rows use the 12.9.x backport build. -- The release notes list every floor move since the last release under - "Breaking Changes". +- The release notes list every floor move since the last release. Update the `cuda-core` pin in `cuda_python/setup.py` to the new minor series (`cuda-core~=X.Y.0`), and plan a `cuda-python` release right after diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index 20295a21dd2..a97f37edcfa 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -83,9 +83,9 @@ bump: pre-commit hook `check-cuda-core-bindings-floor` (`toolshed/check_cuda_core_bindings_floor.py`) checks the pins. It also checks that no documentation page spells a floor out by hand. -3. Add to the release notes a "Breaking Changes" entry that names the new - floors. The support-policy table in `docs/source/support.rst` reads the - floors and the release version at docs-build time. It needs no edit. +3. Add to the release notes an entry that names the new floors. The + support-policy table in `docs/source/support.rst` reads the floors and the + release version at docs-build time. It needs no edit. 4. If the pins moved past what the pixi lock files resolve, refresh the lock files. 5. Pin `cuda-bindings` to match in the conda-forge `cuda-core` feedstock. The diff --git a/cuda_core/docs/source/install.rst b/cuda_core/docs/source/install.rst index e2d74063140..7e67fadad8d 100644 --- a/cuda_core/docs/source/install.rst +++ b/cuda_core/docs/source/install.rst @@ -59,6 +59,10 @@ If you upgrade ``cuda-core`` alone, the installed ``cuda-bindings`` can be older release requires. ``import cuda.core`` then reports the required version and the ``pip`` command that installs it. +Do not pin ``cuda-bindings`` in your own requirements. Depend on ``cuda-core`` with the ``cu12`` +or ``cu13`` extra, and let ``cuda-core`` choose the ``cuda-bindings`` version. A pin of your own +can conflict with the floor of a later ``cuda-core`` release. + Installing from Conda (conda-forge) ----------------------------------- @@ -73,6 +77,7 @@ and likewise use ``cuda-version=13`` for CUDA 13. The conda-forge package depends on ``cuda-bindings`` of the same CUDA major. The ``cuda-bindings`` floor of the release also applies to it. See :ref:`cuda-core-bindings-floor`. +As with ``pip``, do not pin ``cuda-bindings`` yourself. Let ``cuda-core`` choose it. Development environment diff --git a/cuda_core/docs/source/support.rst b/cuda_core/docs/source/support.rst index 4a83d73f766..08670da95a1 100644 --- a/cuda_core/docs/source/support.rst +++ b/cuda_core/docs/source/support.rst @@ -107,7 +107,7 @@ all read them from there. A floor moves only for one of two reasons. A ``cuda-core`` release needs a newer ``cuda-bindings`` API or fix, or its wheels are built against a newer CUDA minor version. A new ``cuda-bindings`` release alone does not move a floor, and a ``cuda-core`` release alone does -not. The :doc:`release notes ` list every move under "Breaking Changes". +not. The :doc:`release notes ` list every move. Python Version Support ---------------------- From dd257d5a2e0f3cad944551ec1f8931c53f2e12cb Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Thu, 12 Nov 2026 08:56:26 -0800 Subject: [PATCH 2/8] Mark cuda-gdb source-line test thread_unsafe (#3433) Signed-off-by: Ralf Juengling (cherry picked from commit 1a22eec83310d7233c8fd2e4179cf59d50e53314) --- cuda_core/tests/test_program.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuda_core/tests/test_program.py b/cuda_core/tests/test_program.py index c045aba0786..6dc422c9c46 100644 --- a/cuda_core/tests/test_program.py +++ b/cuda_core/tests/test_program.py @@ -1263,6 +1263,7 @@ def test_nvrtc_debug_keeps_file_the_caller_named(init_cuda, tmp_path, debug): @pytest.mark.agent_authored(model="cursor-grok-4.6") +@pytest.mark.thread_unsafe(reason="concurrent cuda-gdb sessions on one GPU hang with CUDA_ERROR_INVALID_CONTEXT") def test_cuda_gdb_shows_nvrtc_debug_source_lines(init_cuda): import pathlib From 40782e679efe16dac5507725f322dae03165ad77 Mon Sep 17 00:00:00 2001 From: jpascucci-nv Date: Thu, 12 Nov 2026 20:49:43 -0500 Subject: [PATCH 3/8] fix(cuda.core): serialize pinned-memory kernel-access test on Windows (#3436) * fix(cuda.core): serialize pinned-memory kernel-access test on Windows closes #3426 Add @thread_unsafe_on_windows to the pinned-memory kernel-access test. The test accesses host memory while parallel workers may execute GPU work; synchronizing its own stream does not coordinate those workers. This follows the guard used by similar tests on Windows and WSL. Windows CI is needed to confirm it resolves the reported crashes. The separate OOM failures remain under investigation. Validation: Ruff and git diff --check pass. * fix(cuda.core): report CUDA synchronization errors during test teardown Check the cuCtxSynchronize result in init_cuda so asynchronous errors surface during the triggering test's teardown instead of going unnoticed until a later test. Always attempt context cleanup via finally. Related to #3426. * fix(cuda.core): synchronize the bound context during test teardown Use device.sync() because the test wrapper may pop the current context before init_cuda teardown. This avoids spurious INVALID_CONTEXT errors while preserving synchronization error reporting and context cleanup. Add regression coverage for a missing current context and cleanup after synchronization failures. Related to #3426. * fix(cuda.core): serialize memory-pool watermark reset tests Mark test_mempool_watermark_reset thread_unsafe, matching test_mempool_trim. Both exercise the pool teardown and synchronization sequence associated with the suspected driver race in #3350. * fix(cuda.core): bypass access revocation when destroying host pools Skip peer-access revocation for locally created HOST and HOST_NUMA pools to investigate pinned-pool teardown OOMs and crashes. Continue destroying the pools and preserve cleanup for device, managed, and imported pools. This is a diagnostic workaround pending Windows CI validation. Related to #3426. * fix(cuda.core): preserve the first exception during fixture cleanup Keep setup or synchronization failures primary and attach subsequent cleanup failures as exception notes. Always attempt to pop the context. Add regression coverage for combined failures, including interrupts. * fix(cuda.core): limit host-pool cleanup workaround to pinned allocations HOST and HOST_NUMA also describe managed-pool preferences. Check the allocation type so the pinned-memory workaround preserves managed-pool peer-access revocation. Signed-off-by: Ralf W. Grosse-Kunstleve * test(cuda.core): expose pinned allocation OOM and retain its traceback Signed-off-by: Ralf W. Grosse-Kunstleve * ci: diagnose PR3436 pinned pool teardown on Windows Signed-off-by: Ralf W. Grosse-Kunstleve * ci: move watermark investigation out of PR3436 Signed-off-by: Ralf W. Grosse-Kunstleve * test(cuda.core): restore expected pinned allocation OOM handling Signed-off-by: Ralf W. Grosse-Kunstleve * revert(cuda.core): defer pinned host pool cleanup workaround Signed-off-by: Ralf W. Grosse-Kunstleve --------- Signed-off-by: Ralf W. Grosse-Kunstleve Co-authored-by: Ralf W. Grosse-Kunstleve (cherry picked from commit b71507831f58d331c57eedc3e57023b4b35e9b05) --- cuda_core/tests/conftest.py | 23 +++- .../memory/test_init_cuda_collect_gate.py | 112 +++++++++++++++++- cuda_core/tests/test_memory.py | 2 + 3 files changed, 134 insertions(+), 3 deletions(-) diff --git a/cuda_core/tests/conftest.py b/cuda_core/tests/conftest.py index eeee15dff2a..b667f862d48 100644 --- a/cuda_core/tests/conftest.py +++ b/cuda_core/tests/conftest.py @@ -244,6 +244,7 @@ def init_cuda(request): # live_at_setup stays None until the setup snapshot is recorded. device.set_current() live_at_setup = None + primary_error = None try: # Set option to avoid spin-waiting on synchronization. if int(os.environ.get("CUDA_CORE_TEST_BLOCKING_SYNC", 0)) != 0: @@ -275,6 +276,9 @@ def init_cuda(request): # not counted, so no cached-default bookkeeping is needed. live_at_setup = _live_resource_counts() yield device + except BaseException as exc: + primary_error = exc + raise finally: # pytest holds fixture values in item.funcargs (a pytest-internal dict) # until all teardown has run; we deliberately drop them so already @@ -298,8 +302,23 @@ def init_cuda(request): # With pytest-run-parallel this runs after worker join. This also runs # when setup raises, so an escalated leftover warning cannot leave the # context current. - driver.cuCtxSynchronize() - _ = _device_unset_current() + # The test wrapper may already have popped the calling thread's + # context; synchronize the fixture device's bound context explicitly. + # Keep the first failure primary; cleanup still runs for interrupts. + try: + device.sync() + except BaseException as exc: + if primary_error is None: + primary_error = exc + raise + primary_error.add_note(f"CUDA fixture synchronization failed: {type(exc).__name__}: {exc}") + finally: + try: + _ = _device_unset_current() + except BaseException as exc: + if primary_error is None: + raise + primary_error.add_note(f"CUDA fixture context pop failed: {type(exc).__name__}: {exc}") # After the cleanup, so an escalated warning cannot skip it. if live_after is not None and any(now < snap for now, snap in zip(live_after, live_at_setup)): released = tuple(s - n for s, n in zip(live_at_setup, live_after)) diff --git a/cuda_core/tests/memory/test_init_cuda_collect_gate.py b/cuda_core/tests/memory/test_init_cuda_collect_gate.py index 3455ee8e21f..1f7122694e4 100644 --- a/cuda_core/tests/memory/test_init_cuda_collect_gate.py +++ b/cuda_core/tests/memory/test_init_cuda_collect_gate.py @@ -33,7 +33,8 @@ UnreleasedMemoryResourceWarning, ) -from cuda.core._utils.cuda_utils import driver, handle_return +from cuda.core import Device +from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return class _ScriptedCounts: @@ -250,3 +251,112 @@ def test_teardown_warns_on_baseline_release(request, monkeypatch): pytest.raises(StopIteration), ): next(gen) # teardown: live_after == (0, 0) -> baseline-release warning + + +@_THREAD_UNSAFE_MARKER +@pytest.mark.agent_authored(model="gpt-6") +@pytest.mark.parametrize("clear_current", [False, True]) +def test_teardown_synchronizes_fixture_context(request, monkeypatch, clear_current): + """Teardown works even after the test wrapper has popped the current context.""" + func = _init_cuda_func(request) + _patch_conftest_global(monkeypatch, func, "_mr_expected_live", None) + _patch_conftest_global(monkeypatch, func, "_live_resource_counts", _ScriptedCounts((0, 0))) + + saved = _pop_context_stack() + gen = func(_FakeRequest()) + try: + next(gen) + if clear_current: + handle_return(driver.cuCtxPopCurrent()) + assert current_context_handle() == 0 + _finish_teardown(gen) + assert current_context_handle() == 0 + finally: + gen.close() + _push_context_stack(saved) + + +@_THREAD_UNSAFE_MARKER +@pytest.mark.agent_authored(model="gpt-6") +def test_teardown_sync_failure_still_pops_context(request, monkeypatch): + """A synchronization error propagates without leaving the context current.""" + func = _init_cuda_func(request) + _patch_conftest_global(monkeypatch, func, "_mr_expected_live", None) + _patch_conftest_global(monkeypatch, func, "_live_resource_counts", _ScriptedCounts((0, 0))) + failure = CUDAError("injected synchronization failure") + + class FailingSyncDevice: + device_id = 0 + + def set_current(self): + Device(0).set_current() + + def sync(self): + raise failure + + _patch_conftest_global(monkeypatch, func, "Device", lambda _: FailingSyncDevice()) + saved = _pop_context_stack() + gen = func(_FakeRequest()) + try: + next(gen) + with pytest.raises(CUDAError) as excinfo: + next(gen) + assert excinfo.value is failure + assert current_context_handle() == 0 + finally: + gen.close() + _push_context_stack(saved) + + +@_THREAD_UNSAFE_MARKER +@pytest.mark.agent_authored(model="gpt-6") +@pytest.mark.parametrize( + "fail_setup, fail_sync, fail_pop", + [(True, True, True), (True, False, True), (True, True, False), (False, True, True), (False, False, True)], +) +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_cleanup_preserves_first_failure(request, monkeypatch, fail_setup, fail_sync, fail_pop, error_type): + """Later cleanup errors annotate the first failure, and pop is always attempted.""" + func = _init_cuda_func(request) + _patch_conftest_global(monkeypatch, func, "_mr_expected_live", None) + monkeypatch.setenv("CUDA_CORE_TEST_BLOCKING_SYNC", "0") + setup_error = error_type("injected setup failure") + sync_error = error_type("injected synchronization failure") + pop_error = error_type("injected context pop failure") + state = {"current": False} + + class FakeDevice: + def set_current(self): + state["current"] = True + + def sync(self): + if fail_sync: + raise sync_error + + def counts(): + if fail_setup: + raise setup_error + return (0, 0) + + def pop(): + state["current"] = False + if fail_pop: + raise pop_error + + _patch_conftest_global(monkeypatch, func, "Device", lambda _: FakeDevice()) + _patch_conftest_global(monkeypatch, func, "_live_resource_counts", counts) + _patch_conftest_global(monkeypatch, func, "_device_unset_current", pop) + gen = func(_FakeRequest()) + if not fail_setup: + next(gen) + with pytest.raises(error_type) as excinfo: + next(gen) + expected = setup_error if fail_setup else sync_error if fail_sync else pop_error + assert excinfo.value is expected + assert not state["current"] + notes = getattr(expected, "__notes__", []) + assert len(notes) == int(fail_setup and fail_sync) + int(fail_pop and (fail_setup or fail_sync)) + if fail_setup and fail_sync: + assert any(str(sync_error) in note for note in notes) + if fail_pop and (fail_setup or fail_sync): + assert any(str(pop_error) in note for note in notes) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index fbafb1b2207..5d4aab75e42 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -1246,6 +1246,7 @@ def test_pinned_memory_resource_initialization(init_cuda): buffer.close() +@thread_unsafe_on_windows @pytest.mark.human_reviewed @pytest.mark.parametrize("pool", ["current", "new", "new_numa"]) def test_pinned_memory_resource_kernel_access(init_cuda, pool): @@ -2972,6 +2973,7 @@ def test_mempool_release_threshold_accepts_index_types(memory_resource_factory, mr.close() +@pytest.mark.thread_unsafe(reason="CUDA legacy-default-stream mempool teardown/sync race (gh-3350)") @pytest.mark.agent_authored(model="claude-opus-5-5") @pytest.mark.parametrize("kind", ["reserved", "used"]) def test_mempool_watermark_reset(memory_resource_factory, kind): From 071268a20695ef285fbe39d2fb96149cb681f984 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 16 Nov 2026 10:31:41 -0800 Subject: [PATCH 4/8] [no-ci] ci: exclude unavailable historical GitHub references (#3471) * ci: exclude unavailable historical GitHub references Signed-off-by: Ralf W. Grosse-Kunstleve * ci: require manual copy-pr-bot approval Signed-off-by: Ralf W. Grosse-Kunstleve --------- Signed-off-by: Ralf W. Grosse-Kunstleve (cherry picked from commit 3991a6973dde813ef761983d9752713b592b1c83) --- lychee.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lychee.toml b/lychee.toml index 4ad1995e377..dc179508069 100644 --- a/lychee.toml +++ b/lychee.toml @@ -12,4 +12,8 @@ exclude = [ '^file://.*/cuda-bindings/latest/module/(driver|runtime)\.html#id[0-9]+$', # Preferred Networks rejects hosted-runner GETs, but the URL is browser reachable. '^https://www\.preferred\.jp/en/?$', + # Previously valid GitHub references became unavailable in November 2026. + '^https://github\.com/NVIDIA/cuda-python/pull/2236$', + '^https://github\.com/NVIDIA/cuda-python/issues/2344$', + '^https://github\.com/NVIDIA/cuda-python/issues/2345$', ] From 624896f4d56841fb809bd3a74813317842cc8938 Mon Sep 17 00:00:00 2001 From: B Sainath Reddy Date: Tue, 17 Nov 2026 04:57:03 +0530 Subject: [PATCH 5/8] cuda.core: map ml_dtypes bfloat16 to kDLBfloat in DLPack export (#3359) _smv_dtype_numpy_to_dlpack dispatched on numpy dtype kind, but ml_dtypes.bfloat16 has kind "V", so bf16 views fell through to the unsupported-dtype BufferError and could not be re-exported, even though the DLPack import path already accepts kDLBfloat and _tensor_map.pyx maps bfloat16. Detect bfloat16 up front and export it as kDLBfloat (16 bits). The new round-trip test goes through StridedMemoryView.from_dlpack so it cannot skip on stacks where NumPy's own DLPack support for extended dtypes is unavailable, unlike the existing parametrized round-trip test's NumPy-consumer probe. Signed-off-by: Sainath-Reddy7 <180950590+Sainath-Reddy7@users.noreply.github.com> Co-authored-by: Sainath-Reddy7 <180950590+Sainath-Reddy7@users.noreply.github.com> Co-authored-by: Sebastian Berg (cherry picked from commit d1d14572145c14a74e69594e6bc17bdd8cc08d23) --- cuda_core/cuda/core/_memoryview.pyx | 4 ++++ cuda_core/tests/test_utils_dlpack.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/cuda_core/cuda/core/_memoryview.pyx b/cuda_core/cuda/core/_memoryview.pyx index fd73cbb77ef..8e0d7ee13f3 100644 --- a/cuda_core/cuda/core/_memoryview.pyx +++ b/cuda_core/cuda/core/_memoryview.pyx @@ -695,6 +695,10 @@ cdef inline int _smv_dtype_numpy_to_dlpack(object dtype_obj, DLDataType* out_dty if bits not in (64, 128): raise BufferError(f"Unsupported complex dtype: {np_dtype}") code = kDLComplex + elif np_dtype.type is bfloat16: + # ml_dtypes bfloat16 has kind "V", so the kind chain cannot reach it; + # the byte-order check above has already run for it. + code = kDLBfloat else: raise BufferError(f"Unsupported dtype for DLPack export: {np_dtype}") diff --git a/cuda_core/tests/test_utils_dlpack.py b/cuda_core/tests/test_utils_dlpack.py index 9489c977d3f..5efb6b42df4 100644 --- a/cuda_core/tests/test_utils_dlpack.py +++ b/cuda_core/tests/test_utils_dlpack.py @@ -73,6 +73,30 @@ def test_dlpack_export_roundtrip_dtypes(dtype): _assert_dlpack_export_roundtrip(np.zeros((2, 3), dtype=dtype)) +@pytest.mark.agent_authored(model="glm-5.3") +def test_dlpack_export_bfloat16_roundtrip(init_cuda): + """bfloat16 views export as kDLBfloat and re-import as bfloat16 (#3337). + + This test becomes unnecessary when `_assert_dlpack_export_roundtrip` + starts testing bfloat16, which should be `ml_dtypes>0.6.0`. + """ + if ml_dtypes is None: + pytest.skip("ml_dtypes is not installed") + src = np.zeros((2, 3), dtype=np.uint16) + view = StridedMemoryView.from_any_interface(src, stream_ptr=-1).view(dtype=np.dtype(ml_dtypes.bfloat16)) + imported = StridedMemoryView.from_dlpack(view, stream_ptr=-1) + assert imported.dtype == np.dtype(ml_dtypes.bfloat16) + assert imported.shape == (2, 3) + assert imported.ptr == view.ptr + # Pin the exported type code and bit width directly: the importer maps + # kDLBfloat to bfloat16 without checking bits (kDLBfloat = 4 in dlpack.h). + capsule = view.__dlpack__() + dlm = ctypes.cast(_PyCapsule_GetPointer(capsule, b"dltensor"), ctypes.POINTER(_DLManagedTensor)) + exported_dtype = dlm.contents.dl_tensor.dtype + assert exported_dtype.code == 4 + assert exported_dtype.bits == 16 + + @pytest.mark.parametrize( "shape", [pytest.param((), id="scalar"), pytest.param((0, 3), id="empty")], From 909a4b493ddfcfadbc02bf4ea4b44d5c18f6d5b8 Mon Sep 17 00:00:00 2001 From: Rob Parolin Date: Mon, 16 Nov 2026 14:01:56 -0800 Subject: [PATCH 6/8] fix(cuda.core): make IPC imports wait for a pending release of the same descriptor (#3450) * fix(cuda.core): make IPC imports wait for a pending release of the same descriptor deviceptr_import_ipc() shares one handle per descriptor because the driver's first free of an imported pointer unmaps every import of it (nvbug 5570902). The handle's deleter took ipc_import_mutex only after the last reference was gone. In that gap an import found the cache entry expired, re-imported the still-mapped pointer and registered a new entry. The deleter then erased that new entry and freed the pointer, which unmapped it under the new Buffer. The cache entry now outlives the last reference and only its own deleter erases it, under the mutex, together with the free. An import that finds an expired entry waits on a condition variable until that release is done, then imports fresh. Co-Authored-By: Claude Opus 5.5 * Use CTAD for the ipc_import_mutex locks Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit 2f1185ea1625b44704a8ed4a4470118b54959af8) --- cuda_core/cuda/core/_cpp/rt/api.hpp | 4 +- cuda_core/cuda/core/_cpp/rt/memory.cpp | 43 +++++-- .../test_ipc_import_release_race.py | 113 ++++++++++++++++++ 3 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 cuda_core/tests/memory_ipc/test_ipc_import_release_race.py diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp index 4abf85bd5dc..40642ab2f26 100644 --- a/cuda_core/cuda/core/_cpp/rt/api.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -229,7 +229,9 @@ DevicePtrHandle deviceptr_create_mapped_graphics( // Import a device pointer from IPC via cuMemPoolImportPointer. // When the last reference is released, cuMemFreeAsync is called on the stored stream. -// Note: Does not yet implement reference counting for nvbug 5570902. +// Repeat imports of one descriptor share one handle, because the driver's first +// free of an imported pointer unmaps every import of it (nvbug 5570902). An +// import that races the release of the last handle waits until the free is done. // On error, returns empty handle and sets thread-local error (use get_last_error()). DevicePtrHandle deviceptr_import_ipc( const MemoryPoolHandle &h_pool, const void *export_data, const StreamHandle &h_stream diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index fb95140734a..3cf92649dc4 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -9,11 +9,13 @@ #include "error.hpp" #include "internal.hpp" #include +#include #include #include #include #include #include +#include #include #ifndef _WIN32 #include @@ -519,10 +521,21 @@ struct ExportDataKeyHash { } }; +// The handle imported for one descriptor, and the box it aliases. +struct IpcCacheEntry { + std::weak_ptr handle; + const DevicePtrBox *box; +}; + } // namespace -static HandleRegistry ipc_ptr_cache; +// Guarded by ipc_import_mutex. An entry outlives its handle's last reference: +// only that handle's deleter erases it, under the mutex, together with the +// free. An importer that finds an expired entry therefore knows a release of +// the same mapping is in flight and waits on ipc_release_cv until it is done. +static std::unordered_map ipc_ptr_cache; static std::mutex ipc_import_mutex; +static std::condition_variable ipc_release_cv; DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, const StreamHandle &h_stream) @@ -552,10 +565,17 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co CUresult discard_status = CUDA_SUCCESS; CUdeviceptr discarded = 0; { - std::scoped_lock lock(ipc_import_mutex); + std::unique_lock lock(ipc_import_mutex); - if (auto h = ipc_ptr_cache.lookup(key)) { - return h; + for (auto it = ipc_ptr_cache.find(key); it != ipc_ptr_cache.end(); it = ipc_ptr_cache.find(key)) { + if (DevicePtrHandle h = it->second.handle.lock()) { + return h; + } + // Expired but not erased: the last reference is gone and its + // deleter has not freed the mapping yet. A re-import now would + // share that mapping, and the deleter's cuMemFreeAsync would + // unmap it under us (nvbug 5570902). Wait for the release. + ipc_release_cv.wait(lock); } CUdeviceptr ptr; @@ -579,20 +599,29 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co GILReleaseGuard gil; const DeallocationStream& stream = b->deallocation; ContextHandle h_dealloc = deallocation_context(stream); + // Explicit template argument: MSVC cannot deduce it here (C2955). std::unique_lock lock(ipc_import_mutex); - ipc_ptr_cache.unregister_handle(key); + // Erase only this box's entry (importers wait while it + // exists, so no other box can own the key yet). + auto entry = ipc_ptr_cache.find(key); + if (entry != ipc_ptr_cache.end() && entry->second.box == b) { + ipc_ptr_cache.erase(entry); + } cleanup_in_context( h_dealloc, "cuMemFreeAsync", handle_bits(b->resource), [&]() noexcept { return p_cuMemFreeAsync(b->resource, as_cu(stream.h_stream)); // raw: under ipc_import_mutex }, - [&]() noexcept { lock.release()->unlock(); }); + [&]() noexcept { + ipc_release_cv.notify_all(); + lock.release()->unlock(); + }); delete b; } ); // clang-format on DevicePtrHandle h(box, &box->resource); - ipc_ptr_cache.register_handle(key, h); + ipc_ptr_cache[key] = IpcCacheEntry{h, box.get()}; return h; } diff --git a/cuda_core/tests/memory_ipc/test_ipc_import_release_race.py b/cuda_core/tests/memory_ipc/test_ipc_import_release_race.py new file mode 100644 index 00000000000..7e6a19b977a --- /dev/null +++ b/cuda_core/tests/memory_ipc/test_ipc_import_release_race.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stress test: importing a descriptor while its last import is being released. + +``deviceptr_import_ipc()`` shares one handle per descriptor, because the +driver's first free of an imported pointer unmaps every import of it +(nvbug 5570902). Releasing the last handle frees the mapping after its +reference count reaches zero, so an import can arrive while that free is still +pending. The import must wait for the free, not re-import the still-mapped +pointer and lose it to the pending free. + +Threads in one child import, use and close the same descriptor in a loop, so +the last handle is released again and again while other threads import. A +lost mapping shows up as a failed copy; a missed wakeup shows up as a hang, +which the parent's join timeout catches. +""" + +import contextlib +import multiprocessing as mp +import threading +import time + +import pytest +from helpers.child_processes import child_timeout_sec, kill_subprocesses + +from cuda.core import Buffer, Device + +CHILD_TIMEOUT_SEC = child_timeout_sec() +NBYTES = 4096 +THREADS = 4 +DURATION_SEC = 1.0 + +# these tests spawn new processes and files which fails for very many threads; +# child processes and IPC fixtures are shared across workers (#2784); cap at two threads +pytestmark = pytest.mark.parallel_threads_limit(2) + + +def child_main(queue): + device = Device() + device.set_current() + mr = queue.get() + descriptor = queue.get() + # A VirtualMemoryResource rounds the allocation up to its granularity. + nbytes = queue.get() + + barrier = threading.Barrier(THREADS) + errors = [] + iterations = [0] * THREADS + + def worker(index): + Device().set_current() + stream = Device().create_stream() + local = Device().memory_resource.allocate(nbytes, stream=stream) + barrier.wait() + deadline = time.monotonic() + DURATION_SEC + try: + while time.monotonic() < deadline and not errors: + buffer = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) + local.copy_from(buffer, stream=stream) + stream.sync() + buffer.close() + iterations[index] += 1 + except Exception as e: + errors.append(e) + finally: + local.close(stream) + stream.sync() + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(THREADS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + if errors: + raise errors[0] + assert all(iterations), iterations + device.sync() + + +class TestIpcImportReleaseRace: + """Imports that race the release of the last handle must stay usable and must not hang.""" + + @pytest.fixture(autouse=True) + def _set_start_method(self): + # Ensure spawn is used for multiprocessing + with contextlib.suppress(RuntimeError): + mp.set_start_method("spawn", force=True) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_main(self, ipc_device, ipc_memory_resource): + ipc_device.set_current() + mr = ipc_memory_resource + + stream = ipc_device.default_stream + buffer = mr.allocate(NBYTES, stream=stream) + stream.sync() + + try: + queue = mp.Queue() + process = mp.Process(target=child_main, args=(queue,)) + process.start() + queue.put(mr) + queue.put(buffer.ipc_descriptor) + queue.put(buffer.size) + + process.join(timeout=CHILD_TIMEOUT_SEC) + survivors = kill_subprocesses(process) + assert not survivors, "an import racing the last release did not return" + assert process.exitcode == 0, f"child process failed with exit code {process.exitcode}" + finally: + buffer.close(stream) From 8e7cb056ad271abf07dd07c79501faa7b466f86f Mon Sep 17 00:00:00 2001 From: Rui Luo Date: Tue, 17 Nov 2026 08:16:38 +0800 Subject: [PATCH 7/8] test(cuda.core): skip HOST graph allocation before CUDA 13 (#3446) (cherry picked from commit 11cc66621809b3d4eef8bd4702ec082ac15d1576) --- cuda_core/tests/graph/test_graph_definition.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index cd4f5038bd7..0d4f2c8f843 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -927,6 +927,11 @@ def test_alloc_memory_type_host(init_cuda): """HOST graph alloc nodes reconstruct memory_type from the driver, not the Python argument.""" skip_if_mempool_unsupported() from cuda.core._utils.cuda_utils import CUDAError + from cuda.core._utils.version import driver_version + + # CUDA 12 rejects HOST pool properties before graph support is checked. + if driver_version() < (13, 0, 0): + pytest.skip("CUDA drivers before 13.0 reject HOST allocation properties") g = GraphDefinition() try: From c18d573517f40551e2502ba1b19afd9a72460947 Mon Sep 17 00:00:00 2001 From: Rui Luo Date: Tue, 17 Nov 2026 08:17:45 +0800 Subject: [PATCH 8/8] test(cuda.core): query green stream contexts with v2 (#3448) Signed-off-by: Rui Luo (cherry picked from commit 7076a67d91ee4eb6addbce87ba9fcf153d5161a5) --- cuda_core/tests/helpers/contexts.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/helpers/contexts.py b/cuda_core/tests/helpers/contexts.py index f613f0ac336..d1e73de926a 100644 --- a/cuda_core/tests/helpers/contexts.py +++ b/cuda_core/tests/helpers/contexts.py @@ -71,9 +71,17 @@ def assert_device_operations_use_bound_context(device): # Live check: query the driver directly, rather than comparing cached # metadata, to confirm the stream was actually created in the bound # context rather than whatever was ambient. - driver_stream_ctx = handle_return(driver.cuStreamGetCtx(stream.handle)) + if bound_context.is_green: + # The legacy query rejects green streams on CUDA 12.9. In v2, + # pCtx is the primary context; pGreenCtx identifies the green context. + _, driver_green_ctx = handle_return(driver.cuStreamGetCtx_v2(stream.handle)) + assert int(driver_green_ctx) != 0, "The stream must be associated with a green context." + driver_stream_ctx = handle_return(driver.cuCtxFromGreenCtx(driver_green_ctx)) + else: + # Keep the legacy query for regular contexts on older drivers. + driver_stream_ctx = handle_return(driver.cuStreamGetCtx(stream.handle)) assert int(driver_stream_ctx) == int(bound_context.handle), ( - "cuStreamGetCtx reports a context other than the one the stream was created in." + "The driver reports a context other than the one the stream was created in." ) event = device.create_event()