[EM] Check whether memory policy is set. (#11556)

Author: trivialfis

doc/tutorials/external_memory.rst

@@ -274,6 +274,33 @@ with version ``>=565.47`` is required, it should come with CTK 12.7 and later
 versions. Lastly, there's a known issue with Linux 6.11 that can lead to CUDA host memory
 allocation failure with an ``invalid argument`` error.
 
+.. _extmem-adaptive-cache:
+
+==============
+Adaptive Cache
+==============
+
+Starting with 3.1, XGBoost introduces an adaptive cache for GPU-based external memory
+training. The feature helps split the data cache into a host cache and a device cache. By
+keeping a portion of the cache on the GPU, we can reduce the amount of data transfer
+during training when there's sufficient amount of GPU memory. The feature can be
+controlled by the ``cache_host_ratio`` parameter in the
+:py:class:`xgboost.ExtMemQuantileDMatrix`. It is disabled when the device has full C2C
+bandwidth since it's not needed there. On devices that with reduced bandwidth or devices
+with PCIe connections, unless explicitly specified, the ratio is automatically estimated
+based on device memory size and the size of the dataset.
+
+However, this parameter increases memory fragmentation as XGBoost needs large memory pages
+with irregular sizes. As a result, you might see out of memory error after the
+construction of the ``DMatrix`` but before the actual training begins.
+
+For reference, we tested the adaptive cache with a 128GB (512 features) dense 32bit
+floating dataset using a NVIDIA A6000 GPU, which comes with 48GB device memory. The
+``cache_host_ratio`` was estimated to be about 0.3, meaning about 30 percent of the
+quantized cache was on the host and rest of 70 percent was actually in-core. Given this
+ratio, the overhead is minimal. However, the estimated ratio increases as the data size
+grows.
+
 ================================
 Non-Uniform Memory Access (NUMA)
 ================================
@@ -314,6 +341,13 @@ shown below, the `GPU0` is associated with the `0` node ID::
     NIC2    NODE    SYS     NODE    NODE     X      SYS
     NIC3    SYS     NODE    SYS     SYS     SYS      X
 
+Alternatively, one can also use the ``hwloc`` command line interface, please make sure the
+strict flag is used:
+
+.. code-block:: sh
+
+    hwloc-bind --strict --membind node:${NODEID} --cpubind node:${NODEID} ./myapp
+
 Another approach is to use the CPU affinity. The `dask-cuda
 <https://github.com/rapidsai/dask-cuda>`__ project configures optimal CPU affinity for the
 Dask interface through using the `nvml` library in addition to the Linux sched

python-package/xgboost/core.py

@@ -1841,6 +1841,8 @@ def __init__(  # pylint: disable=super-init-not-called
             parameter specifies the size of host cache compared to the size of the
             entire cache: :math:`host / (host + device)`.
 
+            See :ref:`extmem-adaptive-cache` for more info.
+
         """
         self.max_bin = max_bin
         self.missing = missing if missing is not None else np.nan

src/common/error_msg.cc

@@ -1,12 +1,14 @@
 /**
- * Copyright 2023 by XGBoost contributors
+ * Copyright 2023-2025, XGBoost contributors
  */
 #include "error_msg.h"
 
-#include <mutex>    // for call_once, once_flag
-#include <sstream>  // for stringstream
+#include <mutex>         // for call_once, once_flag
+#include <sstream>       // for stringstream
+#include <system_error>  // for error_code, system_category
 
 #include "../collective/communicator-inl.h"  // for GetRank
+#include "xgboost/collective/socket.h"       // for LastError
 #include "xgboost/context.h"                 // for Context
 #include "xgboost/logging.h"
 
@@ -76,4 +78,10 @@ void CheckOldNccl(std::int32_t major, std::int32_t minor, std::int32_t patch) {
     LOG(WARNING) << msg();
   }
 }
+
+[[nodiscard]] std::error_code SystemError() {
+  std::int32_t errsv = system::LastError();
+  auto err = std::error_code{errsv, std::system_category()};
+  return err;
+}
 }  // namespace xgboost::error

src/common/error_msg.h

@@ -6,9 +6,10 @@
 #ifndef XGBOOST_COMMON_ERROR_MSG_H_
 #define XGBOOST_COMMON_ERROR_MSG_H_
 
-#include <cstdint>    // for uint64_t
-#include <limits>     // for numeric_limits
-#include <string>     // for string
+#include <cstdint>       // for uint64_t
+#include <limits>        // for numeric_limits
+#include <string>        // for string
+#include <system_error>  // for error_code
 
 #include "xgboost/base.h"     // for bst_feature_t
 #include "xgboost/context.h"  // for Context
@@ -140,5 +141,7 @@ constexpr StringView CacheHostRatioNotImpl() {
 constexpr StringView CacheHostRatioInvalid() {
   return "`cache_host_ratio` must be in range [0, 1].";
 }
+
+[[nodiscard]] std::error_code SystemError();
 }  // namespace xgboost::error
 #endif  // XGBOOST_COMMON_ERROR_MSG_H_

src/common/io.cc

@@ -1,6 +1,7 @@
 /**
  * Copyright 2019-2025, by XGBoost Contributors
  */
+#include "error_msg.h"
 #if defined(__unix__) || defined(__APPLE__)
 
 #include <fcntl.h>     // for open, O_RDONLY, posix_fadvise
@@ -29,12 +30,10 @@
 #include <iterator>      // for distance
 #include <memory>        // for unique_ptr, make_unique
 #include <string>        // for string
-#include <system_error>  // for error_code, system_category
 #include <utility>       // for move
 #include <vector>        // for vector
 
 #include "io.h"
-#include "xgboost/collective/socket.h"  // for LastError
 #include "xgboost/logging.h"            // for CHECK_LE
 #include "xgboost/string_view.h"        // for StringView
 
@@ -134,19 +133,13 @@ std::size_t GetMmapAlignment() {
   return getpagesize();
 #endif
 }
-
-auto SystemErrorMsg() {
-  std::int32_t errsv = system::LastError();
-  auto err = std::error_code{errsv, std::system_category()};
-  return err.message();
-}
 }  // anonymous namespace
 
 std::vector<char> LoadSequentialFile(std::string uri) {
   auto OpenErr = [&uri]() {
     std::string msg;
     msg = "Opening " + uri + " failed: ";
-    msg += SystemErrorMsg();
+    msg += error::SystemError().message();
     LOG(FATAL) << msg;
   };
 
@@ -193,10 +186,11 @@ MMAPFile* detail::OpenMmap(std::string path, std::size_t offset, std::size_t len
 #if defined(xgboost_IS_WIN)
   HANDLE fd = CreateFile(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
                          FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, nullptr);
-  CHECK_NE(fd, INVALID_HANDLE_VALUE) << "Failed to open:" << path << ". " << SystemErrorMsg();
+  CHECK_NE(fd, INVALID_HANDLE_VALUE)
+      << "Failed to open:" << path << ". " << error::SystemError().message();
 #else
   auto fd = open(path.c_str(), O_RDONLY);
-  CHECK_GE(fd, 0) << "Failed to open:" << path << ". " << SystemErrorMsg();
+  CHECK_GE(fd, 0) << "Failed to open:" << path << ". " << error::SystemError().message();
 #endif
 
   std::byte* ptr{nullptr};
@@ -207,7 +201,7 @@ MMAPFile* detail::OpenMmap(std::string path, std::size_t offset, std::size_t len
 #if defined(__linux__) || defined(__GLIBC__)
   int prot{PROT_READ};
   ptr = reinterpret_cast<std::byte*>(mmap(nullptr, view_size, prot, MAP_PRIVATE, fd, view_start));
-  CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << path << ". " << SystemErrorMsg();
+  CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << path << ". " << error::SystemError().message();
   auto handle = new MMAPFile{fd, ptr, view_size, offset - view_start, std::move(path)};
 #elif defined(xgboost_IS_WIN)
   auto file_size = GetFileSize(fd, nullptr);
@@ -216,16 +210,16 @@ MMAPFile* detail::OpenMmap(std::string path, std::size_t offset, std::size_t len
   access = FILE_MAP_READ;
   std::uint32_t loff = static_cast<std::uint32_t>(view_start);
   std::uint32_t hoff = view_start >> 32;
-  CHECK(map_file) << "Failed to map: " << path << ". " << SystemErrorMsg();
+  CHECK(map_file) << "Failed to map: " << path << ". " << error::SystemError().message();
   ptr = reinterpret_cast<std::byte*>(MapViewOfFile(map_file, access, hoff, loff, view_size));
-  CHECK_NE(ptr, nullptr) << "Failed to map: " << path << ". " << SystemErrorMsg();
+  CHECK_NE(ptr, nullptr) << "Failed to map: " << path << ". " << error::SystemError().message();
   auto handle = new MMAPFile{fd, map_file, ptr, view_size, offset - view_start, std::move(path)};
 #else
   CHECK_LE(offset, std::numeric_limits<off_t>::max())
       << "File size has exceeded the limit on the current system.";
   int prot{PROT_READ};
   ptr = reinterpret_cast<std::byte*>(mmap(nullptr, view_size, prot, MAP_PRIVATE, fd, view_start));
-  CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << path << ". " << SystemErrorMsg();
+  CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << path << ". " << error::SystemError().message();
   auto handle = new MMAPFile{fd, ptr, view_size, offset - view_start, std::move(path)};
 #endif  // defined(__linux__) || defined(__GLIBC__)
 
@@ -238,22 +232,24 @@ void detail::CloseMmap(MMAPFile* handle) {
   }
 #if defined(xgboost_IS_WIN)
   if (handle->base_ptr) {
-    CHECK(UnmapViewOfFile(handle->base_ptr)) << "Failed to call munmap: " << SystemErrorMsg();
+    CHECK(UnmapViewOfFile(handle->base_ptr))
+        << "Failed to call munmap: " << error::SystemError().message();
   }
   if (handle->fd != INVALID_HANDLE_VALUE) {
-    CHECK(CloseHandle(handle->fd)) << "Failed to close handle: " << SystemErrorMsg();
+    CHECK(CloseHandle(handle->fd)) << "Failed to close handle: " << error::SystemError().message();
   }
   if (handle->file_map != INVALID_HANDLE_VALUE) {
-    CHECK(CloseHandle(handle->file_map)) << "Failed to close mapping object: " << SystemErrorMsg();
+    CHECK(CloseHandle(handle->file_map))
+        << "Failed to close mapping object: " << error::SystemError().message();
   }
 #else
   if (handle->base_ptr) {
     CHECK_NE(munmap(handle->base_ptr, handle->base_size), -1)
-        << "Failed to call munmap: `" << handle->path << "`. " << SystemErrorMsg();
+        << "Failed to call munmap: `" << handle->path << "`. " << error::SystemError().message();
   }
   if (handle->fd != 0) {
     CHECK_NE(close(handle->fd), -1)
-        << "Failed to close: `" << handle->path << "`. " << SystemErrorMsg();
+        << "Failed to close: `" << handle->path << "`. " << error::SystemError().message();
   }
 #endif
   delete handle;
@@ -293,7 +289,7 @@ std::shared_ptr<MallocResource> MemBufFileReadStream::ReadFileIntoBuffer(StringV
   std::unique_ptr<FILE, std::function<int(FILE*)>> fp{fopen(path.c_str(), "rb"), fclose};
 
   auto err = [&] {
-    auto e = SystemErrorMsg();
+    auto e = error::SystemError().message();
     LOG(FATAL) << "Failed to read file `" << path << "`. System error message: " << e;
   };
 #if defined(__linux__)
@@ -302,7 +298,7 @@ std::shared_ptr<MallocResource> MemBufFileReadStream::ReadFileIntoBuffer(StringV
     err();
   }
   if (posix_fadvise(fd, offset, length, POSIX_FADV_SEQUENTIAL) != 0) {
-    LOG(FATAL) << SystemErrorMsg();
+    LOG(FATAL) << error::SystemError().message();
   }
 #endif  // defined(__linux__)
 
@@ -358,12 +354,12 @@ AlignedMemWriteStream::~AlignedMemWriteStream() = default;
 [[nodiscard]] std::size_t TotalMemory() {
 #if defined(__linux__)
   struct sysinfo info;
-  CHECK_EQ(sysinfo(&info), 0) << SystemErrorMsg();
+  CHECK_EQ(sysinfo(&info), 0) << error::SystemError().message();
   return info.totalram * info.mem_unit;
 #elif defined(xgboost_IS_WIN)
   MEMORYSTATUSEX status;
   status.dwLength = sizeof(status);
-  CHECK(GlobalMemoryStatusEx(&status)) << SystemErrorMsg();
+  CHECK(GlobalMemoryStatusEx(&status)) << error::SystemError().message();
   return static_cast<std::size_t>(status.ullTotalPhys);
 #else
   LOG(FATAL) << "Not implemented";