This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-tirx.git
The following commit(s) were added to refs/heads/main by this push:
new 6569e2a18 [FEAT] Allocate CUDA device memory in a locality domain
(#242)
6569e2a18 is described below
commit 6569e2a181d686b2e0d1ba36c84786ff7c449c2d
Author: Hongyi Jin <[email protected]>
AuthorDate: Fri Nov 13 20:59:21 2026 -0500
[FEAT] Allocate CUDA device memory in a locality domain (#242)
CUDA 13.4 exposes **locality domains**. On GPUs such as Rubin, the SMs
and device memory are split into domains, and an SM reads memory in its
own domain faster than memory in another domain. This PR lets TIRx place
tensors in a chosen domain and lets users query the domain topology.
## User-facing API
```python
from tvm_tirx.backend.cuda.runtime import locality_domain
dev = tvm_tirx.cuda(0) # or just the
device id, 0
locality_domain.get_num_locality_domains(dev) # 2 on VR200; 1 on
devices without domains
locality_domain.is_localization_supported(dev) # True when there
are 2+ domains
locality_domain.get_num_sms_per_locality_domain(dev) # SMs guaranteed
to each domain
x = tvm_tirx.runtime.empty((m,), "float32", dev,
mem_scope="global.locality_domain.1")
y = tvm_tirx.runtime.tensor(arr, dev,
mem_scope=locality_domain.locality_domain_scope(0))
z = w.copyto(dev, mem_scope=locality_domain.locality_domain_scope(1))
locality_domain.get_locality_domain_id(x) # 1; None for
memory that is not localized
```
- The query names follow PyTorch
(`torch.cuda.green_contexts.get_num_locality_domains`,
`is_localization_supported`, and the `locality_domain_id` of green
contexts).
- Inside a kernel, `T.cuda.mov_sreg(32, "locality_domain_id")` returns
the domain the CTA runs on (PTX ISA 9.5). A kernel can use it to work on
the data stored in its own domain.
- Every domain's memory is accessible from every SM; only the access
speed differs.
- Localized tensors are ordinary CUDA tensors: they can be passed to
kernels and shared through DLPack (e.g. with torch).
---
docs/reference/python/backend.cuda.rst | 2 +
.../backend.cuda.runtime.locality_domain.rst | 36 ++
python/tvm_tirx/backend/cuda/__init__.py | 2 +
python/tvm_tirx/backend/cuda/runtime/__init__.py | 19 ++
python/tvm_tirx/backend/cuda/runtime/_ffi_api.py | 22 ++
.../backend/cuda/runtime/locality_domain.py | 150 ++++++++
python/tvm_tirx/testing/env.py | 26 ++
src/backend/cuda/runtime/cuda_device_api.cc | 269 ++++++++++++++-
tests/python/backend/cuda/test_locality_domain.py | 378 +++++++++++++++++++++
9 files changed, 896 insertions(+), 8 deletions(-)
diff --git a/docs/reference/python/backend.cuda.rst
b/docs/reference/python/backend.cuda.rst
index c66adc94d..d8cc773b5 100644
--- a/docs/reference/python/backend.cuda.rst
+++ b/docs/reference/python/backend.cuda.rst
@@ -38,6 +38,7 @@ Namespaces
* :doc:`tvm_tirx.backend.cuda.iket <backend.cuda.iket>`
* :doc:`tvm_tirx.backend.cuda.launch <backend.cuda.launch>`
* :doc:`tvm_tirx.backend.cuda.ptx <backend.cuda.ptx>`
+* :doc:`tvm_tirx.backend.cuda.runtime.locality_domain
<backend.cuda.runtime.locality_domain>`
* :doc:`tvm_tirx.backend.cuda.thread_scope <backend.cuda.thread_scope>`
* :doc:`tvm_tirx.backend.cuda.tile <backend.cuda.tile>`
@@ -53,6 +54,7 @@ Namespaces
backend.cuda.iket
backend.cuda.launch
backend.cuda.ptx
+ backend.cuda.runtime.locality_domain
backend.cuda.thread_scope
backend.cuda.tile
diff --git a/docs/reference/python/backend.cuda.runtime.locality_domain.rst
b/docs/reference/python/backend.cuda.runtime.locality_domain.rst
new file mode 100644
index 000000000..c479fe9ef
--- /dev/null
+++ b/docs/reference/python/backend.cuda.runtime.locality_domain.rst
@@ -0,0 +1,36 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+tvm_tirx.backend.cuda.runtime.locality_domain
+=============================================
+
+.. automodule:: tvm_tirx.backend.cuda.runtime.locality_domain
+ :no-members:
+
+.. currentmodule:: tvm_tirx.backend.cuda.runtime.locality_domain
+
+Functions
+---------
+
+.. autosummary::
+ :toctree: generated/
+
+ get_num_locality_domains
+ is_localization_supported
+ get_num_sms_per_locality_domain
+ get_locality_domain_id
+ locality_domain_scope
diff --git a/python/tvm_tirx/backend/cuda/__init__.py
b/python/tvm_tirx/backend/cuda/__init__.py
index 497426972..3a8fe5f9b 100644
--- a/python/tvm_tirx/backend/cuda/__init__.py
+++ b/python/tvm_tirx/backend/cuda/__init__.py
@@ -37,6 +37,7 @@ _LAZY_SUBMODULES = {
"launch",
"op",
"ptx",
+ "runtime",
"script",
"target_tags",
"tile",
@@ -140,6 +141,7 @@ __all__ = [
"op",
"ptx",
"register_backend",
+ "runtime",
"script",
"script_namespace",
"script_namespaces",
diff --git a/python/tvm_tirx/backend/cuda/runtime/__init__.py
b/python/tvm_tirx/backend/cuda/runtime/__init__.py
new file mode 100644
index 000000000..4a26c1c79
--- /dev/null
+++ b/python/tvm_tirx/backend/cuda/runtime/__init__.py
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""CUDA runtime extensions of TIRx."""
+
+from . import locality_domain
diff --git a/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py
b/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py
new file mode 100644
index 000000000..218e9fb6c
--- /dev/null
+++ b/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""FFI APIs for tvm_tirx.backend.cuda.runtime"""
+
+import tvm_ffi
+
+# Exports functions registered in the runtime.cuda namespace.
+tvm_ffi.init_ffi_api("runtime.cuda", __name__)
diff --git a/python/tvm_tirx/backend/cuda/runtime/locality_domain.py
b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py
new file mode 100644
index 000000000..b8d5b43ea
--- /dev/null
+++ b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py
@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""CUDA locality domains: per-domain device memory and topology queries.
+
+Some GPUs (e.g. Rubin) split their SMs and device memory into locality domains;
+an SM reads memory of its own domain faster than memory of another one. A
+tensor is placed in domain ``k`` by allocating it with the memory scope
+``global.locality_domain.<k>``. On a GPU with multiple locality domains::
+
+ from tvm_tirx.backend.cuda.runtime import locality_domain
+
+ dev = tvm_tirx.cuda(0)
+ for k in range(locality_domain.get_num_locality_domains(dev)):
+ scope = locality_domain.locality_domain_scope(k) #
"global.locality_domain.<k>"
+ x = tvm_tirx.runtime.empty((n,), "float32", dev, mem_scope=scope)
+ assert locality_domain.get_locality_domain_id(x) == k
+
+The query names and their semantics follow ``torch.cuda.green_contexts``
+(``get_num_locality_domains``, ``is_localization_supported``) and torch's
+``locality_domain_id``. Each query wraps the tvm-ffi function
+``runtime.cuda.<Name>`` of the same name, e.g. ``get_num_locality_domains`` is
+``runtime.cuda.GetNumLocalityDomains``.
+
+Like ``cudaMalloc`` memory, a localized tensor is ready on every stream once it
+is allocated, and freeing it waits for the work on every stream of the device.
+
+A kernel reads the domain of the SM it runs on with
+``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work
on
+the data of its own domain. Memory of any domain stays accessible from every
+SM; only the access speed differs. A device without locality domains, or with
+a CUDA build or driver older than 13.4, reports one domain:
+``global.locality_domain.0`` then behaves like ``global``, and
+``get_locality_domain_id`` returns None for its memory.
+"""
+
+import tvm_ffi
+
+from tvm_tirx.runtime import Device
+
+from . import _ffi_api
+
+__all__ = [
+ "MAX_LOCALITY_DOMAINS",
+ "get_locality_domain_id",
+ "get_num_locality_domains",
+ "get_num_sms_per_locality_domain",
+ "is_localization_supported",
+ "locality_domain_scope",
+]
+
+#: Number of ``global.locality_domain.<k>`` memory scopes registered (k <
this).
+MAX_LOCALITY_DOMAINS = 8
+
+
+def _device_id(device: Device | int) -> int:
+ if isinstance(device, int):
+ return device
+ if device.dlpack_device_type() != tvm_ffi.DLDeviceType.kDLCUDA:
+ raise ValueError(f"Expected a CUDA device, got {device}")
+ return device.index
+
+
+def get_num_locality_domains(device: Device | int = 0) -> int:
+ """Return the number of locality domains of a CUDA device.
+
+ A device without locality domains, or a CUDA driver older than 13.4,
+ reports 1.
+
+ Parameters
+ ----------
+ device : tvm_tirx.runtime.Device or int
+ A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id.
+ """
+ return int(_ffi_api.GetNumLocalityDomains(_device_id(device)))
+
+
+def is_localization_supported(device: Device | int = 0) -> bool:
+ """Return whether memory can be localized on a CUDA device.
+
+ True when the CUDA driver supports locality domains and the device has two
+ or more of them.
+
+ Parameters
+ ----------
+ device : tvm_tirx.runtime.Device or int
+ A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id.
+ """
+ return bool(_ffi_api.IsLocalizationSupported(_device_id(device)))
+
+
+def get_num_sms_per_locality_domain(device: Device | int = 0) -> int:
+ """Return the number of SMs guaranteed to each locality domain of a CUDA
device.
+
+ A device without locality domains reports all of its SMs. On a device with
+ domains, SMs beyond ``num_domains * sms_per_domain`` are supplemental SMs;
a
+ kernel tells them apart with
+ ``T.cuda.mov_sreg(32, "locality_domain_supplemental_sm")``.
+
+ Parameters
+ ----------
+ device : tvm_tirx.runtime.Device or int
+ A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id.
+ """
+ return int(_ffi_api.GetNumSMsPerLocalityDomain(_device_id(device)))
+
+
+def get_locality_domain_id(tensor) -> int | None:
+ """Return the locality domain a CUDA tensor's memory belongs to.
+
+ Returns None for memory that is not localized, e.g. memory allocated with
+ the plain ``global`` scope, or with ``global.locality_domain.0`` on a
device
+ that reports a single locality domain.
+
+ Parameters
+ ----------
+ tensor : tvm_tirx.runtime.Tensor or any DLPack-compatible tensor
+ A tensor in CUDA device memory.
+ """
+ domain = _ffi_api.GetLocalityDomainId(tensor)
+ return None if domain is None else int(domain)
+
+
+def locality_domain_scope(locality_domain_id: int) -> str:
+ """Return the memory scope that places an allocation in a locality domain.
+
+ Parameters
+ ----------
+ locality_domain_id : int
+ The locality domain, in ``[0, get_num_locality_domains(device))``.
+ """
+ if not 0 <= locality_domain_id < MAX_LOCALITY_DOMAINS:
+ raise ValueError(
+ f"locality domain {locality_domain_id} is outside the registered
scopes "
+ f"[0, {MAX_LOCALITY_DOMAINS})"
+ )
+ return f"global.locality_domain.{locality_domain_id}"
diff --git a/python/tvm_tirx/testing/env.py b/python/tvm_tirx/testing/env.py
index fbba72148..5f6a8dd5f 100644
--- a/python/tvm_tirx/testing/env.py
+++ b/python/tvm_tirx/testing/env.py
@@ -31,6 +31,7 @@ __all__ = [
"has_cuda_arch",
# version / capability
"has_cuda_compute",
+ "has_cuda_locality_domain_nvrtc_support",
"has_cudagraph",
"has_gpu",
"has_llvm",
@@ -221,6 +222,31 @@ def has_nvcc_version(major: int, minor: int = 0, release:
int = 0) -> bool:
return has_cuda() and _nvcc_version() >= (major, minor, release)
[email protected]
+def has_cuda_locality_domain_nvrtc_support() -> bool:
+ """True if locality-domain kernels can run with the default NVRTC compiler.
+
+ Requires a CUDA runtime and an sm_100+ device with multiple locality
domains,
+ plus loaded NVRTC 13.5+ for the PTX 9.5 locality-domain special registers.
+ This probes kernel-test capability, not just localized allocation support.
+ Missing bindings, libraries, or failed capability queries return False.
+ """
+ try:
+ if not has_cuda_compute(10):
+ return False
+ get_num_domains = tvm_ffi.get_global_func(
+ "runtime.cuda.GetNumLocalityDomains", allow_missing=True
+ )
+ if get_num_domains is None or get_num_domains(0) < 2:
+ return False
+ from cuda.bindings import nvrtc # pylint:
disable=import-outside-toplevel
+
+ status, major, minor = nvrtc.nvrtcVersion()
+ return status == nvrtc.nvrtcResult.NVRTC_SUCCESS and (major, minor) >=
(13, 5)
+ except Exception: # pylint: disable=broad-except
+ return False
+
+
@functools.cache
def _llvm_version_major() -> int:
"""Return the major LLVM version, or 0 if LLVM is unavailable."""
diff --git a/src/backend/cuda/runtime/cuda_device_api.cc
b/src/backend/cuda/runtime/cuda_device_api.cc
index 745d10be5..4828973e8 100644
--- a/src/backend/cuda/runtime/cuda_device_api.cc
+++ b/src/backend/cuda/runtime/cuda_device_api.cc
@@ -26,6 +26,7 @@
#include <tvm/ffi/extra/c_env_api.h>
#include <tvm/ffi/extra/cuda/device_guard.h>
#include <tvm/ffi/function.h>
+#include <tvm/ffi/optional.h>
#include <tvm/ffi/reflection/registry.h>
#include <tvm/runtime/device_api.h>
#include <tvm/runtime/logging.h>
@@ -34,6 +35,12 @@
#include <algorithm>
#include <cstring>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
#include "../../../runtime/workspace_pool.h"
@@ -52,6 +59,182 @@ namespace runtime {
}
#endif
+namespace {
+
+/*!
+ * \brief Device memory localized to one CUDA locality domain.
+ *
+ * A device with several locality domains
(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT)
+ * splits its memory between them; SMs read their own domain's memory faster.
+ * `Empty` places a tensor in domain k for the memory scope
+ * `global.locality_domain.<k>`. The scopes are registered for k in
+ * [0, kMaxLocalityDomains); a device validates k against its own domain count.
+ */
+constexpr int kMaxLocalityDomains = 8;
+constexpr const char* kLocalityDomainScopePrefix = "global.locality_domain.";
+
+/*! \brief Memory-scope attribute: the locality domain k of
`global.locality_domain.<k>`. */
+constexpr ffi::reflection::AttrKey<int64_t>
kLocalityDomainIdAttr{"cuda.locality_domain_id"};
+
+void RegisterLocalityDomainScopes() {
+ for (int k = 0; k < kMaxLocalityDomains; ++k) {
+ std::string name = kLocalityDomainScopePrefix + std::to_string(k);
+ if (!MemoryScope::IsDefined(name)) {
+ ffi::reflection::EnumDef<MemoryScopeObj>(name.c_str())
+ .set_attr(attr::kMemoryAllocProperty,
static_cast<int>(MemoryAllocProperty::kNormal))
+ .set_attr(kLocalityDomainIdAttr, k);
+ }
+ }
+}
+
+/*! \brief The locality domain of a memory scope, or nullopt if it is not a
locality-domain scope.
+ */
+std::optional<int> LocalityDomainOf(const MemoryScope& scope) {
+ static auto column = [] {
+ ffi::reflection::TypeAttrColumn state_column(ffi::type_attr::kEnumState);
+ auto state =
state_column[MemoryScopeObj::RuntimeTypeIndex()].cast<ffi::EnumState>();
+ return state->attrs.at(kLocalityDomainIdAttr.name);
+ }();
+ if (auto value = column.Get(scope)) return
static_cast<int>(value->cast<int64_t>());
+ return std::nullopt;
+}
+
+// Locality domains go through the driver API, not the CUDA runtime: the
+// runtime library loaded at run time may predate CUDA 13.4 (e.g. the one a
+// PyTorch wheel ships) and then knows neither the attributes nor the new
+// cudaPointerAttributes field, while the driver always matches the device.
+#if CUDA_VERSION >= 13040
+/*! \brief Whether the installed driver implements locality domains (CUDA
13.4+). */
+bool DriverHasLocalityDomains() {
+ static const bool supported = [] {
+ int version = 0;
+ return cuDriverGetVersion(&version) == CUDA_SUCCESS && version >= 13040;
+ }();
+ return supported;
+}
+
+int GetDriverAttribute(CUdevice_attribute attribute, int device_id) {
+ CUDA_DRIVER_CALL(cuInit(0));
+ CUdevice device;
+ CUDA_DRIVER_CALL(cuDeviceGet(&device, device_id));
+ int value = 0;
+ CUDA_DRIVER_CALL(cuDeviceGetAttribute(&value, attribute, device));
+ return value;
+}
+#endif
+
+// The functions below back `tvm_tirx.backend.cuda.runtime.locality_domain`;
each
+// one is registered as `runtime.cuda.<Name>` for the Python function of the
same name.
+
+/*! \brief Number of locality domains of a device; 1 when it has none or CUDA
predates them. */
+int GetNumLocalityDomains(int device_id) {
+ // Queried once per device: every localized allocation and free needs it.
+ static std::mutex mutex;
+ static std::vector<int> cache; // 0 = not queried yet
+ std::lock_guard<std::mutex> lock(mutex);
+ TVM_FFI_CHECK(device_id >= 0, ValueError) << "Invalid CUDA device id " <<
device_id;
+ if (static_cast<size_t>(device_id) < cache.size() && cache[device_id] > 0) {
+ return cache[device_id];
+ }
+ int count = 0;
+#if CUDA_VERSION >= 13040
+ if (DriverHasLocalityDomains()) {
+ count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT,
device_id);
+ }
+#endif
+ if (count <= 0) count = 1;
+ if (static_cast<size_t>(device_id) >= cache.size()) cache.resize(device_id +
1, 0);
+ cache[device_id] = count;
+ return count;
+}
+
+/*! \brief Whether memory can be localized on a device: it has two or more
locality domains. */
+bool IsLocalizationSupported(int device_id) { return
GetNumLocalityDomains(device_id) > 1; }
+
+/*! \brief SMs guaranteed to each locality domain; the device's SM count
without domains. */
+int GetNumSMsPerLocalityDomain(int device_id) {
+ int count = 0;
+#if CUDA_VERSION >= 13040
+ if (DriverHasLocalityDomains()) {
+ count =
GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_MULTIPROCESSOR_COUNT,
device_id);
+ }
+#endif
+ if (count <= 0) {
+ TVM_FFI_CHECK_CUDA_ERROR(
+ cudaDeviceGetAttribute(&count, cudaDevAttrMultiProcessorCount,
device_id));
+ }
+ return count;
+}
+
+/*! \brief The locality domain a tensor's memory belongs to, or None if it is
not localized. */
+ffi::Optional<int64_t> GetLocalityDomainId(ffi::TensorView tensor) {
+ TVM_FFI_CHECK(tensor.device().device_type == kDLCUDA, ValueError)
+ << "Expected a tensor in CUDA device memory, got device type " <<
tensor.device().device_type;
+ if (tensor.data_ptr() == nullptr) return std::nullopt;
+#if CUDA_VERSION >= 13040
+ if (DriverHasLocalityDomains()) {
+ // The driver call needs a current context, which a fresh thread may lack.
+ ffi::CUDADeviceGuard device_guard(tensor.device().device_id);
+ TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(tensor.device().device_id));
+ int ordinal = -1;
+ CUDA_DRIVER_CALL(cuPointerGetAttribute(&ordinal,
CU_POINTER_ATTRIBUTE_LOCALITY_DOMAIN_ORDINAL,
+
reinterpret_cast<CUdeviceptr>(tensor.data_ptr())));
+ if (ordinal >= 0) return static_cast<int64_t>(ordinal);
+ }
+#endif
+ return std::nullopt;
+}
+
+#if CUDA_VERSION >= 13040
+/*!
+ * \brief The memory pools of CUDA locality domains, one per (device, domain),
+ * each created on first use.
+ *
+ * Pools live until process exit (they are never destroyed) so that tensors
freed
+ * during static destruction still have a valid pool. The release threshold
stays
+ * at its default of 0: freed memory goes back to the device at the next
+ * synchronization instead of staying reserved by the pool.
+ */
+class LocalityDomainPoolManager {
+ public:
+ static LocalityDomainPoolManager* Global() {
+ // Never destroyed, for the same reason as the pools.
+ static auto* manager = new LocalityDomainPoolManager();
+ return manager;
+ }
+
+ CUmemoryPool GetLocalityDomainPool(int device_id, int locality_domain_id) {
+ TVM_FFI_CHECK(device_id >= 0, ValueError) << "Invalid CUDA device id " <<
device_id;
+ TVM_FFI_CHECK(0 <= locality_domain_id && locality_domain_id <
kMaxLocalityDomains, ValueError)
+ << "Invalid locality domain " << locality_domain_id;
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (static_cast<size_t>(device_id) >= pools_.size()) {
+ pools_.resize(device_id + 1,
std::vector<CUmemoryPool>(kMaxLocalityDomains, nullptr));
+ }
+ CUmemoryPool& pool = pools_[device_id][locality_domain_id];
+ if (pool == nullptr) {
+ CUmemPoolProps props{};
+ props.allocType = CU_MEM_ALLOCATION_TYPE_PINNED;
+ props.handleTypes = CU_MEM_HANDLE_TYPE_NONE;
+ props.location.type = CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN;
+ props.location.localized.deviceId = static_cast<unsigned
char>(device_id);
+ props.location.localized.localityDomainId = static_cast<unsigned
char>(locality_domain_id);
+ CUmemoryPool created = nullptr;
+ CUDA_DRIVER_CALL(cuMemPoolCreate(&created, &props));
+ pool = created;
+ }
+ return pool;
+ }
+
+ private:
+ std::mutex mutex_;
+ /*! \brief pools_[device][domain]; nullptr until created. */
+ std::vector<std::vector<CUmemoryPool>> pools_;
+};
+#endif
+
+} // namespace
+
class CUDADeviceAPI final : public DeviceAPI {
public:
void SetDevice(Device dev) final {
TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(dev.device_id)); }
@@ -154,22 +337,46 @@ class CUDADeviceAPI final : public DeviceAPI {
}
ffi::Tensor Empty(ffi::ShapeView shape, DLDataType dtype, DLDevice device,
ffi::Optional<runtime::MemoryScope> mem_scope =
std::nullopt) final {
- TVM_FFI_CHECK(!mem_scope.has_value() || mem_scope.value() ==
runtime::MemoryScope::Global(),
- ValueError)
- << "Unsupported memory scope: "
- << mem_scope.value_or(runtime::MemoryScope::Global()).name();
+ // `global` (the default) is a plain device allocation;
+ // `global.locality_domain.<k>` places the tensor in locality domain k.
+ std::optional<int> locality_domain;
+ if (mem_scope.has_value() && !(mem_scope.value() ==
runtime::MemoryScope::Global())) {
+ locality_domain = LocalityDomainOf(mem_scope.value());
+ TVM_FFI_CHECK(locality_domain.has_value(), ValueError)
+ << "Unsupported memory scope: " << mem_scope.value().name();
+ TVM_FFI_CHECK(device.device_type == kDLCUDA, ValueError)
+ << "Memory scope " << mem_scope.value().name() << " requires a CUDA
device";
+ int count = GetNumLocalityDomains(device.device_id);
+ TVM_FFI_CHECK(*locality_domain < count, ValueError)
+ << "Memory scope " << mem_scope.value().name() << " is out of range:
CUDA device "
+ << device.device_id << " has " << count << " locality domain(s)";
+ // Without locality domains, or without CUDA 13.4 support for them,
domain 0
+ // is the whole device: allocate it like `global`.
+ if (count == 1) locality_domain = std::nullopt;
+ }
struct Allocator {
CUDADeviceAPI* api;
+ std::optional<int> locality_domain;
void AllocData(DLTensor* tensor) {
+ size_t nbytes = ffi::GetDataSize(*tensor);
+ if (locality_domain.has_value()) {
+ tensor->data = api->AllocLocalityDomainSpace(tensor->device, nbytes,
*locality_domain);
+ return;
+ }
size_t alignment =
std::max(static_cast<size_t>(kAllocAlignment),
static_cast<size_t>((tensor->dtype.bits / 8) *
tensor->dtype.lanes));
- tensor->data = api->AllocDataSpace(tensor->device,
ffi::GetDataSize(*tensor), alignment,
- tensor->dtype);
+ tensor->data = api->AllocDataSpace(tensor->device, nbytes, alignment,
tensor->dtype);
+ }
+ void FreeData(DLTensor* tensor) {
+ if (locality_domain.has_value()) {
+ api->FreeLocalityDomainSpace(tensor->device, tensor->data);
+ return;
+ }
+ api->FreeDataSpace(tensor->device, tensor->data);
}
- void FreeData(DLTensor* tensor) { api->FreeDataSpace(tensor->device,
tensor->data); }
};
- return ffi::Tensor::FromNDAlloc(Allocator{this}, shape, dtype, device);
+ return ffi::Tensor::FromNDAlloc(Allocator{this, locality_domain}, shape,
dtype, device);
}
private:
@@ -190,6 +397,42 @@ class CUDADeviceAPI final : public DeviceAPI {
return ret;
}
+ /*! \brief Allocate in locality domain `domain` of a device with several
domains. */
+ void* AllocLocalityDomainSpace(Device dev, size_t nbytes, int domain) {
+#if CUDA_VERSION >= 13040
+ ffi::CUDADeviceGuard device_guard(dev.device_id);
+ // Ensure a current context even when the guard did not switch devices: the
+ // driver calls below need one, and this may be the first CUDA call. Since
+ // CUDA 12.0, cudaSetDevice initializes the device's primary context.
+ TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(dev.device_id));
+ CUdeviceptr ret;
+ CUDA_DRIVER_CALL(cuMemAllocFromPoolAsync(
+ &ret, nbytes,
+
LocalityDomainPoolManager::Global()->GetLocalityDomainPool(dev.device_id,
domain),
+ /*hStream=*/nullptr));
+ // Like cudaMalloc, the allocation is usable from any stream once this
returns.
+ CUDA_DRIVER_CALL(cuStreamSynchronize(nullptr));
+ return reinterpret_cast<void*>(ret);
+#else
+ TVM_FFI_THROW(InternalError)
+ << "A device reports several locality domains only with CUDA 13.4+";
+#endif
+ }
+
+ /*! \brief Free memory from AllocLocalityDomainSpace. */
+ void FreeLocalityDomainSpace(Device dev, void* ptr) {
+ // As in FreeDataSpace, skip the free while unwinding from a sticky CUDA
error.
+ if (std::uncaught_exceptions() && cudaPeekAtLastError() ==
cudaErrorIllegalAddress) return;
+ // cudaFree of cudaMalloc memory waits for all work on the device, but
+ // cudaFree of pool memory does not. Synchronize the device first so that,
+ // as for global memory, no kernel on any stream still uses the memory.
+ {
+ ffi::CUDADeviceGuard device_guard(dev.device_id);
+ TVM_FFI_CHECK_CUDA_ERROR(cudaDeviceSynchronize());
+ }
+ FreeDataSpace(dev, ptr);
+ }
+
void FreeDataSpace(Device dev, void* ptr) {
if (std::uncaught_exceptions() && cudaPeekAtLastError() ==
cudaErrorIllegalAddress) {
// For most CUDA calls, an error from an API call will be
@@ -420,6 +663,16 @@ TVM_FFI_STATIC_INIT_BLOCK() {
refl::GlobalDef().def("runtime.GetCudaDeviceCount", GetCudaDeviceCount);
}
+TVM_FFI_STATIC_INIT_BLOCK() {
+ namespace refl = tvm::ffi::reflection;
+ RegisterLocalityDomainScopes();
+ refl::GlobalDef()
+ .def("runtime.cuda.GetNumLocalityDomains", GetNumLocalityDomains)
+ .def("runtime.cuda.IsLocalizationSupported", IsLocalizationSupported)
+ .def("runtime.cuda.GetNumSMsPerLocalityDomain",
GetNumSMsPerLocalityDomain)
+ .def("runtime.cuda.GetLocalityDomainId", GetLocalityDomainId);
+}
+
#if (CUDA_VERSION >= 12000)
/**
* \brief FFI wrapper for cuTensorMapEncodeTiled.
diff --git a/tests/python/backend/cuda/test_locality_domain.py
b/tests/python/backend/cuda/test_locality_domain.py
new file mode 100644
index 000000000..f7639e96b
--- /dev/null
+++ b/tests/python/backend/cuda/test_locality_domain.py
@@ -0,0 +1,378 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Locality-domain memory scopes of the CUDA device API."""
+
+import numpy as np
+import pytest
+import tvm_ffi
+
+import tvm_tirx
+import tvm_tirx.testing
+from tvm_tirx.backend.cuda.runtime import locality_domain
+from tvm_tirx.script import tirx as T
+from tvm_tirx.testing import env
+
+
+def test_locality_domain_scope_names():
+ for k in range(locality_domain.MAX_LOCALITY_DOMAINS):
+ assert locality_domain.locality_domain_scope(k) ==
f"global.locality_domain.{k}"
+ with pytest.raises(ValueError):
+
locality_domain.locality_domain_scope(locality_domain.MAX_LOCALITY_DOMAINS)
+ with pytest.raises(ValueError):
+ locality_domain.locality_domain_scope(-1)
+
+
[email protected](
+ tvm_ffi.get_global_func("runtime.cuda.GetNumLocalityDomains",
allow_missing=True) is None,
+ reason="TVM-TIRx is built without the CUDA runtime",
+)
+def test_locality_domain_scopes_registered():
+ """The CUDA runtime registers the scopes, even on a machine without a
GPU."""
+ for k in range(locality_domain.MAX_LOCALITY_DOMAINS):
+ assert
tvm_tirx.runtime.MemoryScope(locality_domain.locality_domain_scope(k)).name == (
+ f"global.locality_domain.{k}"
+ )
+ with pytest.raises(ValueError):
+ tvm_tirx.runtime.MemoryScope(
+ f"global.locality_domain.{locality_domain.MAX_LOCALITY_DOMAINS}"
+ )
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
+def test_locality_domain_topology():
+ dev = tvm_tirx.cuda(0)
+ count = locality_domain.get_num_locality_domains(dev)
+ per_domain = locality_domain.get_num_sms_per_locality_domain(dev)
+ assert count >= 1
+ assert 0 < per_domain * count <= dev.multi_processor_count
+ assert locality_domain.is_localization_supported(dev) == (count > 1)
+ # A device id works in place of a Device, as in torch.cuda.green_contexts.
+ assert locality_domain.get_num_locality_domains(0) == count
+ assert locality_domain.get_num_sms_per_locality_domain(0) == per_domain
+ assert locality_domain.is_localization_supported(0) == (count > 1)
+ with pytest.raises(ValueError, match="CUDA device"):
+ locality_domain.get_num_locality_domains(tvm_tirx.cpu(0))
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
+def test_locality_domain_allocation_roundtrip():
+ """Each scope places memory in its domain; plain `global` memory is not
localized."""
+ dev = tvm_tirx.cuda(0)
+ count = locality_domain.get_num_locality_domains(dev)
+ plain = tvm_tirx.runtime.empty((1024,), "float32", dev)
+ assert locality_domain.get_locality_domain_id(plain) is None
+ for k in range(count):
+ data = np.arange(4096, dtype=np.float32) + k
+ x = tvm_tirx.runtime.tensor(data, dev,
mem_scope=locality_domain.locality_domain_scope(k))
+ np.testing.assert_array_equal(x.numpy(), data)
+ y = tvm_tirx.runtime.empty((4096,), "float32", dev,
mem_scope=f"global.locality_domain.{k}")
+ y.copyfrom(x)
+ np.testing.assert_array_equal(y.numpy(), data)
+ z = plain.copyto(dev,
mem_scope=locality_domain.locality_domain_scope(k))
+ if count > 1:
+ # On a device without domains, domain 0 is the whole device and
reports None.
+ assert locality_domain.get_locality_domain_id(x) == k
+ assert locality_domain.get_locality_domain_id(y) == k
+ assert locality_domain.get_locality_domain_id(z) == k
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
[email protected]("shape", [(0,), (2, 0, 3)])
+def test_locality_domain_id_empty_tensor(shape):
+ """Empty CUDA tensors have no localized allocation to query."""
+ tensor = tvm_tirx.runtime.empty(shape, "float32", tvm_tirx.cuda(0))
+ assert locality_domain.get_locality_domain_id(tensor) is None
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
+def test_locality_domain_allocation_as_first_cuda_call():
+ """A localized allocation works as the very first CUDA operation of a
process."""
+ import subprocess
+ import sys
+
+ code = (
+ "import tvm_tirx\n"
+ "dev = tvm_tirx.cuda(0)\n"
+ "x = tvm_tirx.runtime.empty((256,), 'float32', dev,
mem_scope='global.locality_domain.0')\n"
+ "print('ok', x.shape)\n"
+ )
+ out = subprocess.run([sys.executable, "-c", code], capture_output=True,
text=True, check=False)
+ assert out.returncode == 0, out.stderr
+ assert out.stdout.startswith("ok")
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
+def test_locality_domain_out_of_range():
+ dev = tvm_tirx.cuda(0)
+ count = locality_domain.get_num_locality_domains(dev)
+ if count >= locality_domain.MAX_LOCALITY_DOMAINS:
+ pytest.skip("every registered scope is a valid domain on this device")
+ with pytest.raises(ValueError, match=f"has {count} locality domain"):
+ tvm_tirx.runtime.empty(
+ (16,), "float32", dev,
mem_scope=locality_domain.locality_domain_scope(count)
+ )
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
+def test_locality_domain_free_returns_memory():
+ """Freed localized tensors go back to the device rather than staying in
the pool."""
+ dev = tvm_tirx.cuda(0)
+ nbytes = 1 << 30
+ scope =
locality_domain.locality_domain_scope(locality_domain.get_num_locality_domains(dev)
- 1)
+
+ def run_and_check():
+ for _ in range(3): # warm up the pool and the allocator
+ tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev,
mem_scope=scope)
+ before = dev.available_global_memory
+ for _ in range(8):
+ x = tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev,
mem_scope=scope)
+ del x
+ dev.sync()
+ assert dev.available_global_memory >= before - nbytes
+
+ tvm_tirx.testing.run_with_gpu_lock(run_and_check)
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
+def test_locality_domain_tensor_freed_while_kernel_runs():
+ """Dropping a tensor right after launching a kernel on it is safe.
+
+ The free waits for the kernel, so the memory is not handed out again,
+ or returned to the device, while the kernel still writes it.
+ """
+ n, steps = 1 << 24, 256
+
+ @T.function
+ def bump(A: T.Tensor((n,), "int32")):
+ T.device_entry(launch=T.cuda.LaunchConfig(grid=n // 256, block=256))
+ i = T.cuda.block_idx("x") * 256 + T.cuda.thread_idx("x")
+ for _ in T.serial(steps):
+ A[i] = A[i] + 1
+
+ mod = tvm_tirx.compile(
+ tvm_tirx.IRModule({"main": bump}),
+ target=tvm_tirx.target.Target({"kind": "cuda", "arch":
env.cuda_arch()}),
+ pipeline="tirx",
+ )
+ dev = tvm_tirx.cuda(0)
+
+ def run_and_check():
+ for k in range(locality_domain.get_num_locality_domains(dev)):
+ scope = locality_domain.locality_domain_scope(k)
+ for _ in range(4):
+ x = tvm_tirx.runtime.empty((n,), "int32", dev, mem_scope=scope)
+ mod(x)
+ del x # no synchronization: the kernel may still be running
+ y = tvm_tirx.runtime.tensor(np.zeros(n, np.int32), dev,
mem_scope=scope)
+ mod(y)
+ np.testing.assert_array_equal(y.numpy(), np.full(n, steps,
np.int32))
+ dev.sync()
+
+ tvm_tirx.testing.run_with_gpu_lock(run_and_check)
+
+
[email protected]
[email protected](not env.has_cuda(), reason="CUDA GPU not available")
+def test_locality_domain_tensor_in_kernel_and_dlpack():
+ """Localized tensors are ordinary CUDA tensors for kernels and DLPack
consumers."""
+ torch = pytest.importorskip("torch")
+
+ @T.function
+ def double(A: T.Tensor((1024,), "float32"), B: T.Tensor((1024,),
"float32")):
+ T.device_entry(launch=T.cuda.LaunchConfig(grid=4, block=256))
+ i = T.cuda.block_idx("x") * 256 + T.cuda.thread_idx("x")
+ B[i] = A[i] * T.float32(2)
+
+ mod = tvm_tirx.compile(
+ tvm_tirx.IRModule({"main": double}),
+ target=tvm_tirx.target.Target({"kind": "cuda", "arch":
env.cuda_arch()}),
+ pipeline="tirx",
+ )
+ dev = tvm_tirx.cuda(0)
+ last = locality_domain.get_num_locality_domains(dev) - 1
+ data = np.random.default_rng(0).standard_normal(1024).astype(np.float32)
+
+ def run_and_check():
+ a = tvm_tirx.runtime.tensor(data, dev,
mem_scope=locality_domain.locality_domain_scope(0))
+ b = tvm_tirx.runtime.empty(
+ (1024,), "float32", dev,
mem_scope=locality_domain.locality_domain_scope(last)
+ )
+ mod(a, b)
+ np.testing.assert_array_equal(b.numpy(), data * 2)
+ t = torch.from_dlpack(b)
+ assert t.is_cuda
+ np.testing.assert_array_equal(t.cpu().numpy(), data * 2)
+ # The query accepts any DLPack tensor, e.g. torch's own (plain)
allocations.
+ assert locality_domain.get_locality_domain_id(torch.zeros(4,
device="cuda")) is None
+
+ tvm_tirx.testing.run_with_gpu_lock(run_and_check)
+
+
[email protected]
[email protected](
+ not env.has_cuda_locality_domain_nvrtc_support(),
+ reason="needs multiple CUDA locality domains and NVRTC 13.5+",
+)
+def test_locality_domain_aware_kernel():
+ """Each CTA writes the half of the output that lives in the domain it runs
on."""
+ dev = tvm_tirx.cuda(0)
+
+ @T.function
+ def fill(
+ O0: T.Tensor((4096,), "int32"),
+ O1: T.Tensor((4096,), "int32"),
+ C: T.Tensor((2,), "int32"),
+ ):
+ T.device_entry(launch=T.cuda.LaunchConfig(grid=64, block=64))
+ dom = T.cuda.mov_sreg(32, "locality_domain_id")
+ tx = T.cuda.thread_idx("x")
+ if tx == 0:
+ if dom == 0:
+ T.cuda.atomic_add(C.ptr_to([0]), T.int32(1))
+ else:
+ T.cuda.atomic_add(C.ptr_to([1]), T.int32(1))
+ for i in T.serial(64):
+ if dom == 0:
+ O0[i * 64 + tx] = dom
+ else:
+ O1[i * 64 + tx] = dom
+
+ mod = tvm_tirx.compile(
+ tvm_tirx.IRModule({"main": fill}),
+ target=tvm_tirx.target.Target({"kind": "cuda", "arch":
env.cuda_arch()}),
+ pipeline="tirx",
+ )
+
+ def run_and_check():
+ o = [
+ tvm_tirx.runtime.tensor(
+ np.full(4096, -1, np.int32), dev,
mem_scope=locality_domain.locality_domain_scope(k)
+ )
+ for k in range(2)
+ ]
+ c = tvm_tirx.runtime.tensor(np.zeros(2, np.int32), dev)
+ mod(o[0], o[1], c)
+ ctas = c.numpy()
+ assert ctas.sum() == 64
+ for k in range(2):
+ got = o[k].numpy()
+ # A domain that received no CTA leaves its output untouched.
+ assert (got == (k if ctas[k] else -1)).all()
+
+ tvm_tirx.testing.run_with_gpu_lock(run_and_check)
+
+
+def _domain_queue_reader(n_words, chunk_words, grid, threads=256):
+ """A persistent kernel: CTAs read the buffer of their own domain, then
help the other.
+
+ B<k> lives in locality domain k and Ctr[k] is its chunk queue. Every word
read
+ is XORed into Sink[0], so the result proves each chunk was read exactly
once.
+ """
+ steps = chunk_words // (threads * 4)
+ n_chunks = n_words // chunk_words
+
+ @T.macro
+ def drain(B, Ctr, k, tx, chunk, c, r, acc):
+ if tx == 0:
+ chunk[0] = T.cuda.atomic_add(Ctr.ptr_to([k]), T.uint32(1))
+ T.cuda.cta_sync()
+ c[0] = chunk[0]
+ T.cuda.cta_sync()
+ while c[0] < T.uint32(n_chunks):
+ for i in T.unroll(steps):
+ idx = T.Cast("int32", c[0]) * chunk_words + (i * threads + tx)
* 4
+ T.ptx.ld.global_.cs.v4.u32(r[0], r[1], r[2], r[3],
B.ptr_to([idx]))
+ acc[0] = acc[0] ^ r[0] ^ r[1] ^ r[2] ^ r[3]
+ if tx == 0:
+ chunk[0] = T.cuda.atomic_add(Ctr.ptr_to([k]), T.uint32(1))
+ T.cuda.cta_sync()
+ c[0] = chunk[0]
+ T.cuda.cta_sync()
+
+ @T.function
+ def read(
+ B0: T.Tensor((n_words,), "uint32"),
+ B1: T.Tensor((n_words,), "uint32"),
+ Ctr: T.Tensor((2,), "uint32"),
+ Sink: T.Tensor((1,), "uint32"),
+ ):
+ T.device_entry(launch=T.cuda.LaunchConfig(grid=grid, block=threads))
+ dom = T.cuda.mov_sreg(32, "locality_domain_id")
+ tx = T.cuda.thread_idx("x")
+ chunk = T.alloc_shared([1], "uint32")
+ c = T.alloc_local([1], "uint32")
+ r = T.alloc_local([4], "uint32")
+ acc = T.alloc_local([1], "uint32")
+ acc[0] = T.uint32(0)
+ # The hardware decides which domain a CTA runs in, so a domain may get
+ # few CTAs or none: drain the own queue first, then the other one.
+ if dom == 0:
+ drain(B0, Ctr, 0, tx, chunk, c, r, acc)
+ drain(B1, Ctr, 1, tx, chunk, c, r, acc)
+ else:
+ drain(B1, Ctr, 1, tx, chunk, c, r, acc)
+ drain(B0, Ctr, 0, tx, chunk, c, r, acc)
+ T.ptx.red.global_.xor.b32(Sink.ptr_to([0]), acc[0])
+
+ return read
+
+
[email protected]
[email protected](
+ not env.has_cuda_locality_domain_nvrtc_support(),
+ reason="needs multiple CUDA locality domains and NVRTC 13.5+",
+)
[email protected]("grid", ["full", 1])
+def test_locality_domain_work_queue_kernel(grid):
+ """A domain-aware persistent kernel reads every chunk exactly once,
however CTAs land."""
+ dev = tvm_tirx.cuda(0)
+ if locality_domain.get_num_locality_domains(dev) != 2:
+ pytest.skip("needs a GPU with exactly two locality domains")
+ n_words, chunk_words = 1 << 22, 1 << 14
+ if grid == "full":
+ grid = dev.multi_processor_count * 2
+ mod = tvm_tirx.compile(
+ tvm_tirx.IRModule({"main": _domain_queue_reader(n_words, chunk_words,
grid)}),
+ target=tvm_tirx.target.Target({"kind": "cuda", "arch":
env.cuda_arch()}),
+ pipeline="tirx",
+ )
+ rng = np.random.default_rng(0)
+ data = [rng.integers(0, 1 << 32, n_words, dtype=np.uint32) for _ in
range(2)]
+
+ def run_and_check():
+ b = [
+ tvm_tirx.runtime.tensor(
+ data[k], dev,
mem_scope=locality_domain.locality_domain_scope(k)
+ )
+ for k in range(2)
+ ]
+ ctr = tvm_tirx.runtime.tensor(np.zeros(2, np.uint32), dev)
+ sink = tvm_tirx.runtime.tensor(np.zeros(1, np.uint32), dev)
+ mod(b[0], b[1], ctr, sink)
+ want = np.bitwise_xor.reduce(data[0]) ^ np.bitwise_xor.reduce(data[1])
+ assert sink.numpy()[0] == want
+ # Each queue hands out all its chunks, then one end marker to every
CTA.
+ assert (ctr.numpy() == n_words // chunk_words + grid).all()
+
+ tvm_tirx.testing.run_with_gpu_lock(run_and_check)