diff --git a/CHANGELOG.md b/CHANGELOG.md index 6115dbe4..0babb6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Documentation for TransferBench is available at - Added new "smoketest" preset that runs a variety of DMA/GFX tests for simple correctness tests - Added new "help" preset to show config file examples - Added new "presets" preset to show available presets and their descriptions +- Added new "rings" preset that runs parallel rings of transfers (pod-capable) - Added new "envvars" preset to show environment variables that can change TransferBench behavior - Adding information on how to run multi-rank with TransferBench, when run with no args - Added new "nica2a" preset (NIC all-to-all over GPUs via NIC executors, multi-node) diff --git a/Makefile b/Makefile index 05c4ddfe..71562cc2 100644 --- a/Makefile +++ b/Makefile @@ -6,18 +6,18 @@ ROCM_PATH ?= /opt/rocm CUDA_PATH ?= /usr/local/cuda MPI_PATH ?= /usr/local/openmpi +HIPCC ?= $(ROCM_PATH)/bin/amdclang++ +NVCC ?= $(CUDA_PATH)/bin/nvcc +DEBUG ?= 0 # Optional features (set to 0 to disable, 1 to enable) -# DISABLE_NIC_EXEC: Disable RDMA/NIC executor support (default: 0) -# DISABLE_MPI_COMM: Disable MPI communicator support (default: 0) -# DISABLE_DMA_BUF: Disable DMA-BUF support for GPU Direct RDMA (default: 1) -# DISABLE_AMD_SMI: Disable AMD-SMI pod membership checking support (default: 0) -# DISABLE_NVML: Disable NVML pod membership detection for CUDA builds (default: 0) -# DISABLE_POD_COMM: Disable pod communication support (default: 0) -# DISABLE_CUMEM: Disable CUDA driver API (default: 0). On CUDA, POD_COMM_ENABLED requires CUMEM_ENABLED. - -HIPCC ?= $(ROCM_PATH)/bin/amdclang++ -NVCC ?= $(CUDA_PATH)/bin/nvcc +# DISABLE_NIC_EXEC: Disable RDMA/NIC executor support (default: 0) +# DISABLE_MPI_COMM: Disable MPI communicator support (default: 0) +# DISABLE_DMA_BUF: Disable DMA-BUF support for GPU Direct RDMA (default: 1) +# DISABLE_AMD_SMI: Disable AMD-SMI pod membership checking support (default: 0) +# DISABLE_NVML: Disable NVML pod membership detection for CUDA builds (default: 0) +# DISABLE_POD_COMM: Disable pod communication support (default: 0) +# DISABLE_CUMEM: Disable CUDA driver API (also disables pod on CUDA) (default: 0) # ROCm device libraries can live in different locations depending on packaging. # hipcc/clang needs to find the amdgcn bitcode directory at link time. @@ -36,7 +36,6 @@ SINGLE_KERNEL ?= 0 GPU_TARGETS ?= native EXE=TransferBench -DEBUG ?= 0 # Only perform this check if 'make clean' is not the target ifeq ($(filter clean,$(MAKECMDGOALS)),) diff --git a/src/client/Presets/Presets.hpp b/src/client/Presets/Presets.hpp index 07f0f7dc..43631a45 100644 --- a/src/client/Presets/Presets.hpp +++ b/src/client/Presets/Presets.hpp @@ -44,7 +44,7 @@ THE SOFTWARE. #include "PeerToPeer.hpp" #include "PodAllToAll.hpp" #include "PodPeerToPeer.hpp" -#include "PodRing.hpp" +#include "Rings.hpp" #include "Scaling.hpp" #include "Schmoo.hpp" #include "SmokeTest.hpp" @@ -80,7 +80,7 @@ std::map presetFuncMap = {"p2p" , {PeerToPeerPreset, "Peer-to-peer device memory bandwidth test"}}, {"poda2a", {PodAllToAllPreset, "All-to-all transfers between subgroups of ranks within a pod"}}, {"podp2p", {PodPeerToPeerPreset, "Peer-to-peer transfers test among ranks within a pod"}}, - {"podring", {PodRingPreset, "Ring transfers within subgroups of ranks in a pod"}}, + {"rings", {RingsPreset, "Ring transfers within subgroups of ranks in a pod"}}, {"rsweep", {SweepPreset, "Randomly sweep through sets of Transfers"}}, {"scaling", {ScalingPreset, "Run scaling test from one GPU to other devices"}}, {"schmoo", {SchmooPreset, "Scaling tests for local/remote read/write/copy"}}, diff --git a/src/client/Presets/PodRing.hpp b/src/client/Presets/Rings.hpp similarity index 92% rename from src/client/Presets/PodRing.hpp rename to src/client/Presets/Rings.hpp index 5b449e58..bee03055 100644 --- a/src/client/Presets/PodRing.hpp +++ b/src/client/Presets/Rings.hpp @@ -20,26 +20,26 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -int PodRingPreset(EnvVars& ev, - size_t const numBytesPerTransfer, - std::string const presetName, - bool const bytesSpecified) +int RingsPreset(EnvVars& ev, + size_t const numBytesPerTransfer, + std::string const presetName, + bool const bytesSpecified) { - // Assuming single pod, for now + // Check for homogeneous ranks if (Utils::GetNumRankGroups() > 1) { - Utils::Print("[ERROR] PodRing preset can only be run across ranks that are homogenous\n"); + Utils::Print("[ERROR] rings preset can only be run across ranks that are homogeneous\n"); Utils::Print("[ERROR] Run ./TransferBench without any args to display topology information\n"); Utils::Print("[ERROR] TB_NIC_FILTER may also be used to limit NIC visibility\n"); return 1; } - if (Utils::GetRankPerPodMap().empty()) { + + // Check for pod support (if multi-node) + int numRanks = TransferBench::GetNumRanks(); + if (numRanks > 1 && Utils::GetRankPerPodMap().empty()) { Utils::Print("[ERROR] No pods detected. Set TB_FORCE_SINGLE_POD=1 to treat all ranks as a single pod.\n"); return 1; } - ev.gfxUnroll = EnvVars::GetEnvVar("GFX_UNROLL", 2); - - int numRanks = TransferBench::GetNumRanks(); int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX); int memTypeIdx = EnvVars::GetEnvVar("MEM_TYPE" , 0); @@ -56,8 +56,8 @@ int PodRingPreset(EnvVars& ev, Utils::Print("[ERROR] Cannot use %d GPUs. Detected %d GPUs\n", numGpus, numDetectedGpus); return 1; } - if (groupSize < 2) { - Utils::Print("[ERROR] Group size must be at least 2 to form a ring\n"); + if (groupSize <= 0) { + Utils::Print("[ERROR] Group size must be greater than 0\n"); return 1; } if (numRanks * numGpus % groupSize) { @@ -70,7 +70,7 @@ int PodRingPreset(EnvVars& ev, bool nicDifference = false; for (int rank = 0; rank < numRanks; rank++) { if (numGpus > TransferBench::GetNumExecutors(EXE_GPU_GFX, rank)) { - Utils::Print("[ERROR] PodRing preset requires each rank to have the same number of GPUs\n"); + Utils::Print("[ERROR] rings preset requires each rank to have the same number of GPUs\n"); return 1; } if (numQueuePairs > 0 && numNics != TransferBench::GetNumExecutors(EXE_NIC, rank)) @@ -85,7 +85,7 @@ int PodRingPreset(EnvVars& ev, if (Utils::RankDoesOutput()) { ev.DisplayEnvVars(); if (!ev.hideEnv) { - if (!ev.outputToCsv) printf("[PodRing Related]\n"); + if (!ev.outputToCsv) printf("[Rings Related]\n"); ev.Print("MEM_TYPE" , memTypeIdx , "Using %s GPU memory (%s)", devMemTypeStr.c_str(), Utils::GetAllGpuMemTypeStr().c_str()); ev.Print("NUM_GPU_DEVICES", numGpus , "Using %d GPUs", numGpus); ev.Print("NUM_QUEUE_PAIRS", numQueuePairs, "Using %d queue pairs for NIC transfers", numQueuePairs); @@ -177,7 +177,7 @@ int PodRingPreset(EnvVars& ev, } if (Utils::RankDoesOutput()) { - Utils::Print("\n--- Pod Ring Group %d ---\n", group); + Utils::Print("\n--- Ring Group %d ---\n", group); int const numHops = groupSize; int const numRows = 2 + numHops + 3;