From 37ca7465cc0f855ccd6625b8e5e7f23c000c0b73 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 5 Aug 2026 08:07:57 -0400 Subject: [PATCH] Fix nvbug6550424: Don't check NVML init behavior on CTK >= 13.4 --- cuda_bindings/tests/nvml/test_init.py | 2 + cuda_bindings/tests/test_cuda.py | 49 +++++++++---------- cuda_bindings/tests/test_cudart.py | 43 +++++++--------- .../cuda_python_test_helpers/__init__.py | 10 ++++ 4 files changed, 53 insertions(+), 51 deletions(-) diff --git a/cuda_bindings/tests/nvml/test_init.py b/cuda_bindings/tests/nvml/test_init.py index a47af24dc6a..c56c400a0b9 100644 --- a/cuda_bindings/tests/nvml/test_init.py +++ b/cuda_bindings/tests/nvml/test_init.py @@ -7,6 +7,7 @@ import pytest from cuda.bindings import nvml +from cuda_python_test_helpers import driver_version_less_than def assert_nvml_is_initialized(): @@ -43,6 +44,7 @@ def get_architecture_name(arch): @pytest.mark.skipif(sys.platform == "win32", reason="Test not supported on Windows") @pytest.mark.thread_unsafe(reason="nvml init affects other threads") +@pytest.mark.skipif(not driver_version_less_than(13040), reason="Init behavior changed in CUDA 13.4") def test_init_ref_count(): """ Verifies that we can call NVML shutdown and init(2) multiple times, and that ref counting works diff --git a/cuda_bindings/tests/test_cuda.py b/cuda_bindings/tests/test_cuda.py index e2751df9237..7bef2b844aa 100644 --- a/cuda_bindings/tests/test_cuda.py +++ b/cuda_bindings/tests/test_cuda.py @@ -15,14 +15,7 @@ import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart from cuda.bindings import driver - - -def driverVersionLessThan(target): - (err,) = cuda.cuInit(0) - assert err == cuda.CUresult.CUDA_SUCCESS - err, version = cuda.cuDriverGetVersion() - assert err == cuda.CUresult.CUDA_SUCCESS - return version < target +from cuda_python_test_helpers import driver_version_less_than def supportsMemoryPool(): @@ -265,7 +258,7 @@ def test_cuda_CUstreamBatchMemOpParams(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" ) def test_cuda_memPool_attr(): poolProps = cuda.CUmemPoolProps() @@ -328,7 +321,7 @@ def test_cuda_memPool_attr(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" ) def test_cuda_pointer_attr(): err, ptr = cuda.cuMemAllocManaged(0x1000, cuda.CUmemAttach_flags.CU_MEM_ATTACH_GLOBAL.value) @@ -379,7 +372,7 @@ def test_cuda_pointer_attr(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" ) def test_pointer_get_attributes_device_ordinal(): attributes = [ @@ -457,7 +450,9 @@ def test_cuda_mem_range_attr(device): assert err == cuda.CUresult.CUDA_SUCCESS -@pytest.mark.skipif(driverVersionLessThan(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported") +@pytest.mark.skipif( + driver_version_less_than(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported" +) @pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.") def test_cuda_graphMem_attr(device): err, stream = cuda.cuStreamCreate(0) @@ -516,7 +511,7 @@ def test_cuda_graphMem_attr(device): @pytest.mark.skipif( - driverVersionLessThan(12010) + driver_version_less_than(12010) or not supportsCudaAPI("cuCoredumpSetAttributeGlobal") or not supportsCudaAPI("cuCoredumpGetAttributeGlobal"), reason="Coredump API not present", @@ -566,7 +561,7 @@ def test_get_error_name_and_string(): # TODO: cuStreamGetCaptureInfo_v2 -@pytest.mark.skipif(driverVersionLessThan(11030), reason="Driver too old for cuStreamGetCaptureInfo_v2") +@pytest.mark.skipif(driver_version_less_than(11030), reason="Driver too old for cuStreamGetCaptureInfo_v2") def test_stream_capture(): pass @@ -636,7 +631,7 @@ def test_invalid_repr_attribute(): @pytest.mark.skipif( - driverVersionLessThan(12020) + driver_version_less_than(12020) or not supportsCudaAPI("cuGraphAddNode") or not supportsCudaAPI("cuGraphNodeSetParams") or not supportsCudaAPI("cuGraphExecNodeSetParams"), @@ -748,7 +743,7 @@ def test_graph_poly(): @pytest.mark.skipif( - driverVersionLessThan(12040) or not supportsCudaAPI("cuDeviceGetDevResource"), + driver_version_less_than(12040) or not supportsCudaAPI("cuDeviceGetDevResource"), reason="Polymorphic graph APIs required", ) def test_cuDeviceGetDevResource(device): @@ -768,7 +763,7 @@ def test_cuDeviceGetDevResource(device): @pytest.mark.skipif( - driverVersionLessThan(12030) or not supportsCudaAPI("cuGraphConditionalHandleCreate"), + driver_version_less_than(12030) or not supportsCudaAPI("cuGraphConditionalHandleCreate"), reason="Conditional graph APIs required", ) def test_conditional(ctx): @@ -830,14 +825,14 @@ def test_all_CUresult_codes(): assert num_good >= 76 # CTK 11.0.3_450.51.06 -@pytest.mark.skipif(driverVersionLessThan(12030), reason="Driver too old for cuKernelGetName") +@pytest.mark.skipif(driver_version_less_than(12030), reason="Driver too old for cuKernelGetName") def test_cuKernelGetName_failure(): err, name = cuda.cuKernelGetName(0) assert err == cuda.CUresult.CUDA_ERROR_INVALID_VALUE assert name is None -@pytest.mark.skipif(driverVersionLessThan(12030), reason="Driver too old for cuFuncGetName") +@pytest.mark.skipif(driver_version_less_than(12030), reason="Driver too old for cuFuncGetName") def test_cuFuncGetName_failure(): err, name = cuda.cuFuncGetName(0) assert err == cuda.CUresult.CUDA_ERROR_INVALID_VALUE @@ -845,7 +840,7 @@ def test_cuFuncGetName_failure(): @pytest.mark.skipif( - driverVersionLessThan(12080) or not supportsCudaAPI("cuCheckpointProcessGetState"), + driver_version_less_than(12080) or not supportsCudaAPI("cuCheckpointProcessGetState"), reason="When API was introduced", ) def test_cuCheckpointProcessGetState_failure(): @@ -887,7 +882,7 @@ def test_struct_pointer_comparison(target): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphGetId"), reason="Requires CUDA 13.1+", ) def test_cuGraphGetId(device, ctx): @@ -914,7 +909,7 @@ def test_cuGraphGetId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphExecGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphExecGetId"), reason="Requires CUDA 13.1+", ) def test_cuGraphExecGetId(device, ctx): @@ -1040,7 +1035,7 @@ def test_cuGraphNodeGetDependencies_edgeData_outlives_call(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetLocalId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetLocalId"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetLocalId(device, ctx): @@ -1082,7 +1077,7 @@ def test_cuGraphNodeGetLocalId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetToolsId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetToolsId"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetToolsId(device, ctx): @@ -1111,7 +1106,7 @@ def test_cuGraphNodeGetToolsId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetContainingGraph"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetContainingGraph"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetContainingGraph(device, ctx): @@ -1158,7 +1153,7 @@ def test_cuGraphNodeGetContainingGraph(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuStreamGetDevResource"), + driver_version_less_than(13010) or not supportsCudaAPI("cuStreamGetDevResource"), reason="Requires CUDA 13.1+", ) def test_cuStreamGetDevResource(device, ctx): @@ -1177,7 +1172,7 @@ def test_cuStreamGetDevResource(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuDevSmResourceSplit"), + driver_version_less_than(13010) or not supportsCudaAPI("cuDevSmResourceSplit"), reason="Requires CUDA 13.1+", ) def test_cuDevSmResourceSplit(device, ctx): diff --git a/cuda_bindings/tests/test_cudart.py b/cuda_bindings/tests/test_cudart.py index 0a4e2b8bb82..7b70acdeb46 100644 --- a/cuda_bindings/tests/test_cudart.py +++ b/cuda_bindings/tests/test_cudart.py @@ -12,6 +12,7 @@ import cuda.bindings.runtime as cudart from cuda import pathfinder from cuda.bindings import runtime +from cuda_python_test_helpers import driver_version_less_than def isSuccess(err): @@ -22,12 +23,6 @@ def assertSuccess(err): assert isSuccess(err) -def driverVersionLessThan(target): - err, version = cudart.cudaDriverGetVersion() - assertSuccess(err) - return version < target - - def supportsMemoryPool(): err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) return isSuccess(err) and isSupported @@ -504,7 +499,7 @@ def test_cudart_cudaGetDeviceProperties(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" ) def test_cudart_MemPool_attr(): poolProps = cudart.cudaMemPoolProps() @@ -1445,7 +1440,7 @@ def test_cudart_func_callback(): @pytest.mark.skipif( - driverVersionLessThan(12030) or not supportsCudaAPI("cudaGraphConditionalHandleCreate"), + driver_version_less_than(12030) or not supportsCudaAPI("cudaGraphConditionalHandleCreate"), reason="Conditional graph APIs required", ) def test_cudart_conditional(): @@ -1503,7 +1498,7 @@ def test_getLocalRuntimeVersion(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaGraphGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaGraphGetId"), reason="Requires CUDA 13.1+", ) def test_cudaGraphGetId(): @@ -1530,7 +1525,7 @@ def test_cudaGraphGetId(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaGraphExecGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaGraphExecGetId"), reason="Requires CUDA 13.1+", ) def test_cudaGraphExecGetId(): @@ -1577,7 +1572,7 @@ def test_cudaGraphExecGetId(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaGraphNodeGetLocalId"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaGraphNodeGetLocalId"), reason="Requires CUDA 13.1+", ) def test_cudaGraphNodeGetLocalId(): @@ -1619,7 +1614,7 @@ def test_cudaGraphNodeGetLocalId(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaGraphNodeGetToolsId"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaGraphNodeGetToolsId"), reason="Requires CUDA 13.1+", ) def test_cudaGraphNodeGetToolsId(): @@ -1648,7 +1643,7 @@ def test_cudaGraphNodeGetToolsId(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaGraphNodeGetContainingGraph"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaGraphNodeGetContainingGraph"), reason="Requires CUDA 13.1+", ) def test_cudaGraphNodeGetContainingGraph(): @@ -1695,7 +1690,7 @@ def test_cudaGraphNodeGetContainingGraph(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaStreamGetDevResource"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaStreamGetDevResource"), reason="Requires CUDA 13.1+", ) def test_cudaStreamGetDevResource(): @@ -1714,7 +1709,7 @@ def test_cudaStreamGetDevResource(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaDeviceGetDevResource"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaDeviceGetDevResource"), reason="Requires CUDA 13.1+", ) def test_cudaDeviceGetDevResource(): @@ -1729,7 +1724,7 @@ def test_cudaDeviceGetDevResource(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaDeviceGetExecutionCtx"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaDeviceGetExecutionCtx"), reason="Requires CUDA 13.1+", ) def test_cudaExecutionCtxGetDevResource(): @@ -1747,7 +1742,7 @@ def test_cudaExecutionCtxGetDevResource(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaDeviceGetExecutionCtx"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaDeviceGetExecutionCtx"), reason="Requires CUDA 13.1+", ) def test_cudaExecutionCtxGetDevice(): @@ -1767,7 +1762,7 @@ def test_cudaExecutionCtxGetDevice(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaDeviceGetExecutionCtx"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaDeviceGetExecutionCtx"), reason="Requires CUDA 13.1+", ) def test_cudaExecutionCtxGetId(): @@ -1795,7 +1790,7 @@ def test_cudaExecutionCtxGetId(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaDevSmResourceSplit"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaDevSmResourceSplit"), reason="Requires CUDA 13.1+", ) def test_cudaDevSmResourceSplit(): @@ -1864,7 +1859,7 @@ def test_cudaDevSmResourceSplit(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaDevSmResourceSplitByCount"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaDevSmResourceSplitByCount"), reason="Requires CUDA 13.1+", ) def test_cudaDevSmResourceSplitByCount(): @@ -1887,7 +1882,7 @@ def test_cudaDevSmResourceSplitByCount(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaDevResourceGenerateDesc"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaDevResourceGenerateDesc"), reason="Requires CUDA 13.1+", ) def test_cudaDevResourceGenerateDesc(): @@ -1904,7 +1899,7 @@ def test_cudaDevResourceGenerateDesc(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaGreenCtxCreate"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaGreenCtxCreate"), reason="Requires CUDA 13.1+", ) def test_cudaGreenCtxCreate(): @@ -1935,7 +1930,7 @@ def test_cudaGreenCtxCreate(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaExecutionCtxStreamCreate"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaExecutionCtxStreamCreate"), reason="Requires CUDA 13.1+", ) def test_cudaExecutionCtxStreamCreate(): @@ -1956,7 +1951,7 @@ def test_cudaExecutionCtxStreamCreate(): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cudaGraphConditionalHandleCreate_v2"), + driver_version_less_than(13010) or not supportsCudaAPI("cudaGraphConditionalHandleCreate_v2"), reason="Requires CUDA 13.1+", ) def test_cudaGraphConditionalHandleCreate_v2(): diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/__init__.py b/cuda_python_test_helpers/cuda_python_test_helpers/__init__.py index c67162483f5..7e1e33a428b 100644 --- a/cuda_python_test_helpers/cuda_python_test_helpers/__init__.py +++ b/cuda_python_test_helpers/cuda_python_test_helpers/__init__.py @@ -65,3 +65,13 @@ def under_compute_sanitizer() -> bool: # Another common indicator: sanitizer injectors are configured via env vars. inj = os.environ.get("CUDA_INJECTION64_PATH", "") return "compute-sanitizer" in inj or "cuda-memcheck" in inj + + +def driver_version_less_than(target): + from cuda.bindings import driver + + (err,) = driver.cuInit(0) + assert err == driver.CUresult.CUDA_SUCCESS + err, version = driver.cuDriverGetVersion() + assert err == driver.CUresult.CUDA_SUCCESS + return version < target