Updating podring to run on single node without need to force single pod

CHANGELOG.md

@@ -20,6 +20,7 @@ Documentation for TransferBench is available at
 - Added new "smoketest" preset that runs a variety of DMA/GFX tests for simple correctness tests
 - Added new "help" preset to show config file examples
 - Added new "presets" preset to show available presets and their descriptions
+- Added new "rings" preset that runs parallel rings of transfers (pod-capable)
 - Added new "envvars" preset to show environment variables that can change TransferBench behavior
 - Adding information on how to run multi-rank with TransferBench, when run with no args
 - Added new "nica2a" preset (NIC all-to-all over GPUs via NIC executors, multi-node)

Makefile

@@ -6,18 +6,18 @@
 ROCM_PATH ?= /opt/rocm
 CUDA_PATH ?= /usr/local/cuda
 MPI_PATH  ?= /usr/local/openmpi
+HIPCC     ?= $(ROCM_PATH)/bin/amdclang++
+NVCC      ?= $(CUDA_PATH)/bin/nvcc
+DEBUG     ?= 0
 
 # Optional features (set to 0 to disable, 1 to enable)
-# DISABLE_NIC_EXEC: Disable RDMA/NIC executor support (default: 0)
-# DISABLE_MPI_COMM: Disable MPI communicator support (default: 0)
-# DISABLE_DMA_BUF: Disable DMA-BUF support for GPU Direct RDMA (default: 1)
-# DISABLE_AMD_SMI: Disable AMD-SMI pod membership checking support (default: 0)
-# DISABLE_NVML: Disable NVML pod membership detection for CUDA builds (default: 0)
-# DISABLE_POD_COMM: Disable pod communication support (default: 0)
-# DISABLE_CUMEM: Disable CUDA driver API (default: 0). On CUDA, POD_COMM_ENABLED requires CUMEM_ENABLED.
-
-HIPCC ?= $(ROCM_PATH)/bin/amdclang++
-NVCC ?= $(CUDA_PATH)/bin/nvcc
+# DISABLE_NIC_EXEC: Disable RDMA/NIC executor support                     (default: 0)
+# DISABLE_MPI_COMM: Disable MPI communicator support                      (default: 0)
+# DISABLE_DMA_BUF:  Disable DMA-BUF support for GPU Direct RDMA           (default: 1)
+# DISABLE_AMD_SMI:  Disable AMD-SMI pod membership checking support       (default: 0)
+# DISABLE_NVML:     Disable NVML pod membership detection for CUDA builds (default: 0)
+# DISABLE_POD_COMM: Disable pod communication support                     (default: 0)
+# DISABLE_CUMEM:    Disable CUDA driver API (also disables pod on CUDA)   (default: 0)
 
 # ROCm device libraries can live in different locations depending on packaging.
 # hipcc/clang needs to find the amdgcn bitcode directory at link time.
@@ -36,7 +36,6 @@ SINGLE_KERNEL ?= 0
 GPU_TARGETS ?= native
 
 EXE=TransferBench
-DEBUG ?= 0
 
 # Only perform this check if 'make clean' is not the target
 ifeq ($(filter clean,$(MAKECMDGOALS)),)

src/client/Presets/Presets.hpp

@@ -44,7 +44,7 @@ THE SOFTWARE.
 #include "PeerToPeer.hpp"
 #include "PodAllToAll.hpp"
 #include "PodPeerToPeer.hpp"
-#include "PodRing.hpp"
+#include "Rings.hpp"
 #include "Scaling.hpp"
 #include "Schmoo.hpp"
 #include "SmokeTest.hpp"
@@ -80,7 +80,7 @@ std::map<std::string, PresetInfo> presetFuncMap =
   {"p2p"   ,      {PeerToPeerPreset,    "Peer-to-peer device memory bandwidth test"}},
   {"poda2a",      {PodAllToAllPreset,   "All-to-all transfers between subgroups of ranks within a pod"}},
   {"podp2p",      {PodPeerToPeerPreset, "Peer-to-peer transfers test among ranks within a pod"}},
-  {"podring",     {PodRingPreset,       "Ring transfers within subgroups of ranks in a pod"}},
+  {"rings",       {RingsPreset,         "Ring transfers within subgroups of ranks in a pod"}},
   {"rsweep",      {SweepPreset,         "Randomly sweep through sets of Transfers"}},
   {"scaling",     {ScalingPreset,       "Run scaling test from one GPU to other devices"}},
   {"schmoo",      {SchmooPreset,        "Scaling tests for local/remote read/write/copy"}},

src/client/Presets/PodRing.hpp → src/client/Presets/Rings.hpp

@@ -20,26 +20,26 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-int PodRingPreset(EnvVars&          ev,
-                  size_t      const numBytesPerTransfer,
-                  std::string const presetName,
-                  bool        const bytesSpecified)
+int RingsPreset(EnvVars&          ev,
+                size_t      const numBytesPerTransfer,
+                std::string const presetName,
+                bool        const bytesSpecified)
 {
-  // Assuming single pod, for now
+  // Check for homogeneous ranks
   if (Utils::GetNumRankGroups() > 1) {
-    Utils::Print("[ERROR] PodRing preset can only be run across ranks that are homogenous\n");
+    Utils::Print("[ERROR] rings preset can only be run across ranks that are homogeneous\n");
     Utils::Print("[ERROR] Run ./TransferBench without any args to display topology information\n");
     Utils::Print("[ERROR] TB_NIC_FILTER may also be used to limit NIC visibility\n");
     return 1;
   }
-  if (Utils::GetRankPerPodMap().empty()) {
+
+  // Check for pod support (if multi-node)
+  int numRanks = TransferBench::GetNumRanks();
+  if (numRanks > 1 && Utils::GetRankPerPodMap().empty()) {
     Utils::Print("[ERROR] No pods detected. Set TB_FORCE_SINGLE_POD=1 to treat all ranks as a single pod.\n");
     return 1;
   }
 
-  ev.gfxUnroll       = EnvVars::GetEnvVar("GFX_UNROLL", 2);
-
-  int numRanks       = TransferBench::GetNumRanks();
   int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX);
 
   int memTypeIdx    = EnvVars::GetEnvVar("MEM_TYPE"       , 0);
@@ -56,8 +56,8 @@ int PodRingPreset(EnvVars&          ev,
     Utils::Print("[ERROR] Cannot use %d GPUs.  Detected %d GPUs\n", numGpus, numDetectedGpus);
     return 1;
   }
-  if (groupSize < 2) {
-    Utils::Print("[ERROR] Group size must be at least 2 to form a ring\n");
+  if (groupSize <= 0) {
+    Utils::Print("[ERROR] Group size must be greater than 0\n");
     return 1;
   }
   if (numRanks * numGpus % groupSize) {
@@ -70,7 +70,7 @@ int PodRingPreset(EnvVars&          ev,
   bool nicDifference = false;
   for (int rank = 0; rank < numRanks; rank++) {
     if (numGpus > TransferBench::GetNumExecutors(EXE_GPU_GFX, rank)) {
-      Utils::Print("[ERROR] PodRing preset requires each rank to have the same number of GPUs\n");
+      Utils::Print("[ERROR] rings preset requires each rank to have the same number of GPUs\n");
       return 1;
     }
     if (numQueuePairs > 0 && numNics != TransferBench::GetNumExecutors(EXE_NIC, rank))
@@ -85,7 +85,7 @@ int PodRingPreset(EnvVars&          ev,
   if (Utils::RankDoesOutput()) {
     ev.DisplayEnvVars();
     if (!ev.hideEnv) {
-      if (!ev.outputToCsv) printf("[PodRing Related]\n");
+      if (!ev.outputToCsv) printf("[Rings Related]\n");
       ev.Print("MEM_TYPE"       , memTypeIdx   , "Using %s GPU memory (%s)", devMemTypeStr.c_str(), Utils::GetAllGpuMemTypeStr().c_str());
       ev.Print("NUM_GPU_DEVICES", numGpus      , "Using %d GPUs", numGpus);
       ev.Print("NUM_QUEUE_PAIRS", numQueuePairs, "Using %d queue pairs for NIC transfers", numQueuePairs);
@@ -177,7 +177,7 @@ int PodRingPreset(EnvVars&          ev,
     }
 
     if (Utils::RankDoesOutput()) {
-      Utils::Print("\n--- Pod Ring Group %d ---\n", group);
+      Utils::Print("\n--- Ring Group %d ---\n", group);
 
       int const numHops   = groupSize;
       int const numRows   = 2 + numHops + 3;