Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cuda_bindings/cuda/bindings/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from typing import Any, Callable

from ._ptx_utils import get_minimal_required_cuda_ver_from_ptx_ver, get_ptx_ver
from ._version_check import check_cuda_version_compatibility

_handle_getters: dict[type, Callable[[Any], int]] = {}

Expand Down
75 changes: 75 additions & 0 deletions cuda_bindings/cuda/bindings/utils/_version_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE

import os
import warnings

# Track whether we've already checked version compatibility
_version_compatibility_checked = False


def check_cuda_version_compatibility():
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd definitely add major into the name for clarity, and make it more obvious what the function actually does:

def warn_if_cuda_major_version_mismatch()

Also CUDA_PYTHON_DISABLE_MAJOR_VERSION_WARNING

"""Check if the CUDA driver version is compatible with cuda-bindings compile-time version.
This function compares the CUDA version that cuda-bindings was compiled against
with the CUDA version supported by the installed driver. If the compile-time
major version is greater than the driver's major version, a warning is issued.
The check runs only once per process. Subsequent calls are no-ops.
The warning can be suppressed by setting the environment variable
``CUDA_PYTHON_DISABLE_VERSION_CHECK=1``.
Examples
--------
>>> from cuda.bindings.utils import check_cuda_version_compatibility
>>> check_cuda_version_compatibility() # Issues warning if version mismatch
"""
global _version_compatibility_checked
if _version_compatibility_checked:
return
_version_compatibility_checked = True

# Allow users to suppress the warning
if os.environ.get("CUDA_PYTHON_DISABLE_VERSION_CHECK"):
return

# Import here to avoid circular imports and allow lazy loading
from cuda.bindings import driver

# Get compile-time CUDA version from cuda-bindings
try:
compile_version = driver.CUDA_VERSION # e.g., 13010
except AttributeError:
# Older cuda-bindings may not expose CUDA_VERSION
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm ... this code is in the cuda-bindings sources, it cannot possibly be in older cuda-bindings releases.

I think the try-except can go completely?

return

# Get runtime driver version
err, runtime_version = driver.cuDriverGetVersion()
if err != driver.CUresult.CUDA_SUCCESS:
return # Can't check, skip silently
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not being able to query from the driver version is worthy of a warning to the user instead of silently eating it.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd even assume here that this should never error, and surface an exception if it does, thinking something is very wrong at that point, and flagging it immediately is most helpful.


compile_major = compile_version // 1000
runtime_major = runtime_version // 1000

if compile_major > runtime_major:
compile_minor = (compile_version % 1000) // 10
runtime_minor = (runtime_version % 1000) // 10
warnings.warn(
f"cuda-bindings was built against CUDA {compile_major}.{compile_minor}, "
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be most helpful to be clear that the major version mismatch is the primary concern. The minor version information is more distracting than helpful. Suggested warning message:

cuda-bindings was built for CUDA major version {compile_major}, but the
NVIDIA driver only supports up to CUDA {runtime_major}. Some cuda-bindings
features may not work correctly. Consider updating your NVIDIA driver,
or using a cuda-bindings version built for CUDA {runtime_major}. (Set
CUDA_PYTHON_DISABLE_MAJOR_VERSION_WARNING=1 to suppress this warning.)

f"but the installed driver only supports CUDA {runtime_major}.{runtime_minor}. "
f"Some features may not work correctly. Consider updating your NVIDIA driver. "
f"Set CUDA_PYTHON_DISABLE_VERSION_CHECK=1 to suppress this warning.",
UserWarning,
stacklevel=3,
)


def _reset_version_compatibility_check():
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a real use case where production code would call this function? If it’s only used by tests, then it seems reasonable to rely implicitly on the global and move the function into the test code instead.

"""Reset the version compatibility check flag for testing purposes.
This function is intended for use in tests to allow multiple test runs
to check the warning behavior.
"""
global _version_compatibility_checked
_version_compatibility_checked = False
2 changes: 2 additions & 0 deletions cuda_bindings/docs/source/environment_variables.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ Runtime Environment Variables

- ``CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM`` : When set to 1, the default stream is the per-thread default stream. When set to 0, the default stream is the legacy default stream. This defaults to 0, for the legacy default stream. See `Stream Synchronization Behavior <https://docs.nvidia.com/cuda/cuda-runtime-api/stream-sync-behavior.html>`_ for an explanation of the legacy and per-thread default streams.

- ``CUDA_PYTHON_DISABLE_VERSION_CHECK`` : When set to 1, suppresses the warning that is issued when ``cuda.core`` detects that ``cuda-bindings`` was compiled against a newer CUDA major version than the installed driver supports. This warning helps identify version mismatches that may cause features to not work correctly.


Build-Time Environment Variables
--------------------------------
Expand Down
113 changes: 113 additions & 0 deletions cuda_bindings/tests/test_version_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE

import os
import warnings
from unittest import mock

from cuda.bindings import driver
from cuda.bindings.utils import check_cuda_version_compatibility
from cuda.bindings.utils._version_check import _reset_version_compatibility_check


class TestVersionCompatibilityCheck:
"""Tests for CUDA version compatibility check function."""

def setup_method(self):
"""Reset the version compatibility check flag before each test."""
_reset_version_compatibility_check()

def teardown_method(self):
"""Reset the version compatibility check flag after each test."""
_reset_version_compatibility_check()

def test_no_warning_when_driver_newer(self):
"""No warning should be issued when driver version >= compile version."""
# Mock compile version 12.9 and driver version 13.0
with (
mock.patch.object(driver, "CUDA_VERSION", 12090),
mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 13000)),
warnings.catch_warnings(record=True) as w,
):
warnings.simplefilter("always")
check_cuda_version_compatibility()
assert len(w) == 0

def test_no_warning_when_same_major_version(self):
"""No warning should be issued when major versions match."""
# Mock compile version 12.9 and driver version 12.8
with (
mock.patch.object(driver, "CUDA_VERSION", 12090),
mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
warnings.catch_warnings(record=True) as w,
):
warnings.simplefilter("always")
check_cuda_version_compatibility()
assert len(w) == 0

def test_warning_when_compile_major_newer(self):
"""Warning should be issued when compile major version > driver major version."""
# Mock compile version 13.0 and driver version 12.8
with (
mock.patch.object(driver, "CUDA_VERSION", 13000),
mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
warnings.catch_warnings(record=True) as w,
):
warnings.simplefilter("always")
check_cuda_version_compatibility()
assert len(w) == 1
assert issubclass(w[0].category, UserWarning)
assert "cuda-bindings was built against CUDA 13.0" in str(w[0].message)
assert "driver only supports CUDA 12.8" in str(w[0].message)

def test_warning_only_issued_once(self):
"""Warning should only be issued once per process."""
with (
mock.patch.object(driver, "CUDA_VERSION", 13000),
mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
warnings.catch_warnings(record=True) as w,
):
warnings.simplefilter("always")
check_cuda_version_compatibility()
check_cuda_version_compatibility()
check_cuda_version_compatibility()
# Only one warning despite multiple calls
assert len(w) == 1

def test_warning_suppressed_by_env_var(self):
"""Warning should be suppressed when CUDA_PYTHON_DISABLE_VERSION_CHECK is set."""
with (
mock.patch.object(driver, "CUDA_VERSION", 13000),
mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
mock.patch.dict(os.environ, {"CUDA_PYTHON_DISABLE_VERSION_CHECK": "1"}),
warnings.catch_warnings(record=True) as w,
):
warnings.simplefilter("always")
check_cuda_version_compatibility()
assert len(w) == 0

def test_silent_when_driver_version_fails(self):
"""Should silently skip if cuDriverGetVersion fails."""
with (
mock.patch.object(driver, "CUDA_VERSION", 13000),
mock.patch.object(
driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_ERROR_NOT_INITIALIZED, 0)
),
warnings.catch_warnings(record=True) as w,
):
warnings.simplefilter("always")
check_cuda_version_compatibility()
assert len(w) == 0

def test_silent_when_cuda_version_not_available(self):
"""Should silently skip if CUDA_VERSION attribute is not available."""
# Simulate older cuda-bindings without CUDA_VERSION
original = driver.CUDA_VERSION
try:
del driver.CUDA_VERSION
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
check_cuda_version_compatibility()
assert len(w) == 0
finally:
driver.CUDA_VERSION = original
8 changes: 8 additions & 0 deletions cuda_core/cuda/core/_device.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ from cuda.core._event import Event, EventOptions
from cuda.core._graph import GraphBuilder
from cuda.core._stream import IsStreamT, Stream, StreamOptions
from cuda.core._utils.clear_error_support import assert_type
try:
from cuda.bindings.utils import check_cuda_version_compatibility
except ImportError:
# Older cuda-bindings versions may not have this function
check_cuda_version_compatibility = None
from cuda.core._utils.cuda_utils import (
ComputeCapability,
CUDAError,
Expand Down Expand Up @@ -963,6 +968,9 @@ class Device:
with _lock, nogil:
HANDLE_RETURN(cydriver.cuInit(0))
_is_cuInit = True
# Check version compatibility after CUDA is initialized
if check_cuda_version_compatibility is not None:
check_cuda_version_compatibility()

# important: creating a Device instance does not initialize the GPU!
cdef cydriver.CUdevice dev
Expand Down
Loading