From a15e8e4af62ce34814988931280e45a5e9e1fa85 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Fri, 1 May 2026 17:20:26 -0500 Subject: [PATCH 1/4] Updating podring to run on single node without need to force single pod --- Makefile | 21 ++++++++++----------- src/client/Presets/PodRing.hpp | 18 +++++++++--------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 05c4ddfe..71562cc2 100644 --- a/Makefile +++ b/Makefile @@ -6,18 +6,18 @@ ROCM_PATH ?= /opt/rocm CUDA_PATH ?= /usr/local/cuda MPI_PATH ?= /usr/local/openmpi +HIPCC ?= $(ROCM_PATH)/bin/amdclang++ +NVCC ?= $(CUDA_PATH)/bin/nvcc +DEBUG ?= 0 # Optional features (set to 0 to disable, 1 to enable) -# DISABLE_NIC_EXEC: Disable RDMA/NIC executor support (default: 0) -# DISABLE_MPI_COMM: Disable MPI communicator support (default: 0) -# DISABLE_DMA_BUF: Disable DMA-BUF support for GPU Direct RDMA (default: 1) -# DISABLE_AMD_SMI: Disable AMD-SMI pod membership checking support (default: 0) -# DISABLE_NVML: Disable NVML pod membership detection for CUDA builds (default: 0) -# DISABLE_POD_COMM: Disable pod communication support (default: 0) -# DISABLE_CUMEM: Disable CUDA driver API (default: 0). On CUDA, POD_COMM_ENABLED requires CUMEM_ENABLED. - -HIPCC ?= $(ROCM_PATH)/bin/amdclang++ -NVCC ?= $(CUDA_PATH)/bin/nvcc +# DISABLE_NIC_EXEC: Disable RDMA/NIC executor support (default: 0) +# DISABLE_MPI_COMM: Disable MPI communicator support (default: 0) +# DISABLE_DMA_BUF: Disable DMA-BUF support for GPU Direct RDMA (default: 1) +# DISABLE_AMD_SMI: Disable AMD-SMI pod membership checking support (default: 0) +# DISABLE_NVML: Disable NVML pod membership detection for CUDA builds (default: 0) +# DISABLE_POD_COMM: Disable pod communication support (default: 0) +# DISABLE_CUMEM: Disable CUDA driver API (also disables pod on CUDA) (default: 0) # ROCm device libraries can live in different locations depending on packaging. # hipcc/clang needs to find the amdgcn bitcode directory at link time. @@ -36,7 +36,6 @@ SINGLE_KERNEL ?= 0 GPU_TARGETS ?= native EXE=TransferBench -DEBUG ?= 0 # Only perform this check if 'make clean' is not the target ifeq ($(filter clean,$(MAKECMDGOALS)),) diff --git a/src/client/Presets/PodRing.hpp b/src/client/Presets/PodRing.hpp index 5b449e58..6d0d84b6 100644 --- a/src/client/Presets/PodRing.hpp +++ b/src/client/Presets/PodRing.hpp @@ -25,21 +25,21 @@ int PodRingPreset(EnvVars& ev, std::string const presetName, bool const bytesSpecified) { - // Assuming single pod, for now + // Check for homogenous ranks if (Utils::GetNumRankGroups() > 1) { Utils::Print("[ERROR] PodRing preset can only be run across ranks that are homogenous\n"); Utils::Print("[ERROR] Run ./TransferBench without any args to display topology information\n"); Utils::Print("[ERROR] TB_NIC_FILTER may also be used to limit NIC visibility\n"); return 1; } - if (Utils::GetRankPerPodMap().empty()) { + + // Check for pod support (if multi-node) + int numRanks = TransferBench::GetNumRanks(); + if (numRanks > 1 && Utils::GetRankPerPodMap().empty()) { Utils::Print("[ERROR] No pods detected. Set TB_FORCE_SINGLE_POD=1 to treat all ranks as a single pod.\n"); return 1; } - ev.gfxUnroll = EnvVars::GetEnvVar("GFX_UNROLL", 2); - - int numRanks = TransferBench::GetNumRanks(); int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX); int memTypeIdx = EnvVars::GetEnvVar("MEM_TYPE" , 0); @@ -56,10 +56,10 @@ int PodRingPreset(EnvVars& ev, Utils::Print("[ERROR] Cannot use %d GPUs. Detected %d GPUs\n", numGpus, numDetectedGpus); return 1; } - if (groupSize < 2) { - Utils::Print("[ERROR] Group size must be at least 2 to form a ring\n"); - return 1; - } +// if (groupSize < 2) { +// Utils::Print("[ERROR] Group size must be at least 2 to form a ring\n"); +// return 1; +// } if (numRanks * numGpus % groupSize) { Utils::Print("[ERROR] Group size %d cannot evenly divide %d total devices from %d ranks.\n", groupSize, numRanks * numGpus, numRanks); From afb6beb5cfe653d233bd73b98239a76ecb7c9b09 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Fri, 1 May 2026 21:35:49 -0600 Subject: [PATCH 2/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/client/Presets/PodRing.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/Presets/PodRing.hpp b/src/client/Presets/PodRing.hpp index 6d0d84b6..13e5a210 100644 --- a/src/client/Presets/PodRing.hpp +++ b/src/client/Presets/PodRing.hpp @@ -56,10 +56,10 @@ int PodRingPreset(EnvVars& ev, Utils::Print("[ERROR] Cannot use %d GPUs. Detected %d GPUs\n", numGpus, numDetectedGpus); return 1; } -// if (groupSize < 2) { -// Utils::Print("[ERROR] Group size must be at least 2 to form a ring\n"); -// return 1; -// } + if (groupSize <= 0) { + Utils::Print("[ERROR] Group size must be greater than 0\n"); + return 1; + } if (numRanks * numGpus % groupSize) { Utils::Print("[ERROR] Group size %d cannot evenly divide %d total devices from %d ranks.\n", groupSize, numRanks * numGpus, numRanks); From ce77c0ef75d7b20c4f023ad24b35e242bfb62b3c Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Fri, 1 May 2026 22:39:21 -0500 Subject: [PATCH 3/4] Fixing typo --- src/client/Presets/PodRing.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/Presets/PodRing.hpp b/src/client/Presets/PodRing.hpp index 13e5a210..78cacd56 100644 --- a/src/client/Presets/PodRing.hpp +++ b/src/client/Presets/PodRing.hpp @@ -25,9 +25,9 @@ int PodRingPreset(EnvVars& ev, std::string const presetName, bool const bytesSpecified) { - // Check for homogenous ranks + // Check for homogeneous ranks if (Utils::GetNumRankGroups() > 1) { - Utils::Print("[ERROR] PodRing preset can only be run across ranks that are homogenous\n"); + Utils::Print("[ERROR] PodRing preset can only be run across ranks that are homogeneous\n"); Utils::Print("[ERROR] Run ./TransferBench without any args to display topology information\n"); Utils::Print("[ERROR] TB_NIC_FILTER may also be used to limit NIC visibility\n"); return 1; From 18f35ed497ee4722c84bb8adcac62e3d3569bb97 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Fri, 1 May 2026 23:54:50 -0500 Subject: [PATCH 4/4] Renaming podring preset to rings --- CHANGELOG.md | 1 + src/client/Presets/Presets.hpp | 4 ++-- src/client/Presets/{PodRing.hpp => Rings.hpp} | 16 ++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) rename src/client/Presets/{PodRing.hpp => Rings.hpp} (95%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6115dbe4..0babb6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Documentation for TransferBench is available at - Added new "smoketest" preset that runs a variety of DMA/GFX tests for simple correctness tests - Added new "help" preset to show config file examples - Added new "presets" preset to show available presets and their descriptions +- Added new "rings" preset that runs parallel rings of transfers (pod-capable) - Added new "envvars" preset to show environment variables that can change TransferBench behavior - Adding information on how to run multi-rank with TransferBench, when run with no args - Added new "nica2a" preset (NIC all-to-all over GPUs via NIC executors, multi-node) diff --git a/src/client/Presets/Presets.hpp b/src/client/Presets/Presets.hpp index 07f0f7dc..43631a45 100644 --- a/src/client/Presets/Presets.hpp +++ b/src/client/Presets/Presets.hpp @@ -44,7 +44,7 @@ THE SOFTWARE. #include "PeerToPeer.hpp" #include "PodAllToAll.hpp" #include "PodPeerToPeer.hpp" -#include "PodRing.hpp" +#include "Rings.hpp" #include "Scaling.hpp" #include "Schmoo.hpp" #include "SmokeTest.hpp" @@ -80,7 +80,7 @@ std::map presetFuncMap = {"p2p" , {PeerToPeerPreset, "Peer-to-peer device memory bandwidth test"}}, {"poda2a", {PodAllToAllPreset, "All-to-all transfers between subgroups of ranks within a pod"}}, {"podp2p", {PodPeerToPeerPreset, "Peer-to-peer transfers test among ranks within a pod"}}, - {"podring", {PodRingPreset, "Ring transfers within subgroups of ranks in a pod"}}, + {"rings", {RingsPreset, "Ring transfers within subgroups of ranks in a pod"}}, {"rsweep", {SweepPreset, "Randomly sweep through sets of Transfers"}}, {"scaling", {ScalingPreset, "Run scaling test from one GPU to other devices"}}, {"schmoo", {SchmooPreset, "Scaling tests for local/remote read/write/copy"}}, diff --git a/src/client/Presets/PodRing.hpp b/src/client/Presets/Rings.hpp similarity index 95% rename from src/client/Presets/PodRing.hpp rename to src/client/Presets/Rings.hpp index 78cacd56..bee03055 100644 --- a/src/client/Presets/PodRing.hpp +++ b/src/client/Presets/Rings.hpp @@ -20,14 +20,14 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -int PodRingPreset(EnvVars& ev, - size_t const numBytesPerTransfer, - std::string const presetName, - bool const bytesSpecified) +int RingsPreset(EnvVars& ev, + size_t const numBytesPerTransfer, + std::string const presetName, + bool const bytesSpecified) { // Check for homogeneous ranks if (Utils::GetNumRankGroups() > 1) { - Utils::Print("[ERROR] PodRing preset can only be run across ranks that are homogeneous\n"); + Utils::Print("[ERROR] rings preset can only be run across ranks that are homogeneous\n"); Utils::Print("[ERROR] Run ./TransferBench without any args to display topology information\n"); Utils::Print("[ERROR] TB_NIC_FILTER may also be used to limit NIC visibility\n"); return 1; @@ -70,7 +70,7 @@ int PodRingPreset(EnvVars& ev, bool nicDifference = false; for (int rank = 0; rank < numRanks; rank++) { if (numGpus > TransferBench::GetNumExecutors(EXE_GPU_GFX, rank)) { - Utils::Print("[ERROR] PodRing preset requires each rank to have the same number of GPUs\n"); + Utils::Print("[ERROR] rings preset requires each rank to have the same number of GPUs\n"); return 1; } if (numQueuePairs > 0 && numNics != TransferBench::GetNumExecutors(EXE_NIC, rank)) @@ -85,7 +85,7 @@ int PodRingPreset(EnvVars& ev, if (Utils::RankDoesOutput()) { ev.DisplayEnvVars(); if (!ev.hideEnv) { - if (!ev.outputToCsv) printf("[PodRing Related]\n"); + if (!ev.outputToCsv) printf("[Rings Related]\n"); ev.Print("MEM_TYPE" , memTypeIdx , "Using %s GPU memory (%s)", devMemTypeStr.c_str(), Utils::GetAllGpuMemTypeStr().c_str()); ev.Print("NUM_GPU_DEVICES", numGpus , "Using %d GPUs", numGpus); ev.Print("NUM_QUEUE_PAIRS", numQueuePairs, "Using %d queue pairs for NIC transfers", numQueuePairs); @@ -177,7 +177,7 @@ int PodRingPreset(EnvVars& ev, } if (Utils::RankDoesOutput()) { - Utils::Print("\n--- Pod Ring Group %d ---\n", group); + Utils::Print("\n--- Ring Group %d ---\n", group); int const numHops = groupSize; int const numRows = 2 + numHops + 3;