From 4b85f28cd825377f47116c5e82ae8ace6680f336 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sat, 25 Apr 2026 04:45:46 -0500 Subject: [PATCH 1/7] Adding smoketest preset for simple correctness tests --- CHANGELOG.md | 1 + src/client/EnvVars.hpp | 24 +++ src/client/Presets/Presets.hpp | 2 + src/client/Presets/SmokeTest.hpp | 321 +++++++++++++++++++++++++++++++ src/client/Utilities.hpp | 1 - src/header/TransferBench.hpp | 1 - 6 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 src/client/Presets/SmokeTest.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 3525886c..e051e312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Documentation for TransferBench is available at - Adding new batched-DMA executor "B", which utilizes the hipMemcpyBatchAsync API introduced in HIP 7.1 / CUDA 12.8 - Added new bmasweep preset that compares DMA to batched DMA execution for parallel transfers to other GPUs - Added new wallclock preset that compares wallclock counters across XCCs within a GPU +- Added new smoketest preset that runs a variety of DMA/GFX tests for simple correctness tests ### Modified - DMA-BUF support enablement in CMake changed to ENABLE_DMA_BUF to be more similar to other compile-time options diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index 78304f61..c3ec3af4 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -538,6 +538,21 @@ class EnvVars return defaultValue; } + static std::vector GetEnvVarStrArray(std::string const& varname, std::vector const& defaultValue) + { + if (getenv(varname.c_str())) { + std::vector values; + char* arrayStr = getenv(varname.c_str()); + char* token = strtok(arrayStr, ","); + while (token) { + values.push_back(token); + token = strtok(NULL, ","); + } + return values; + } + return defaultValue; + } + static std::vector GetEnvVarRangeArray(std::string const& varname, std::vector const& defaultValue) { if (getenv(varname.c_str())) { @@ -579,6 +594,15 @@ class EnvVars return result; } + std::string GetStr(std::vector const& varnameList) const { + std::string result = ""; + for (int i = 0; i < varnameList.size(); i++) { + if (i) result += ","; + result += varnameList[i]; + } + return result; + } + std::string GetCuMaskDesc() const { std::vector> runs; diff --git a/src/client/Presets/Presets.hpp b/src/client/Presets/Presets.hpp index c64a79d4..5e3869bd 100644 --- a/src/client/Presets/Presets.hpp +++ b/src/client/Presets/Presets.hpp @@ -42,6 +42,7 @@ THE SOFTWARE. #include "PodPeerToPeer.hpp" #include "Scaling.hpp" #include "Schmoo.hpp" +#include "SmokeTest.hpp" #include "Sweep.hpp" #include "WallClock.hpp" @@ -68,6 +69,7 @@ std::map> presetFuncMap = {"rsweep", {SweepPreset, "Randomly sweep through sets of Transfers"}}, {"scaling", {ScalingPreset, "Run scaling test from one GPU to other devices"}}, {"schmoo", {SchmooPreset, "Scaling tests for local/remote read/write/copy"}}, + {"smoketest", {SmokeTestPreset, "Simple correctness smoke-test"}}, {"sweep", {SweepPreset, "Ordered sweep through sets of Transfers"}}, {"wallclock", {WallClockPreset, "Tests wallclock consistency across XCCs within a GPU"}}, }; diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp new file mode 100644 index 00000000..2d7eb9bc --- /dev/null +++ b/src/client/Presets/SmokeTest.hpp @@ -0,0 +1,321 @@ +/* +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include + +namespace { + +#define NUM_SMOKE_TESTS 14 + +int RunTest(int testNum, + std::set const& testsToRun, + std::vector const& sizeList, + int numSubExecPerGpu, + ConfigOptions& cfg, + MemType cpuMemType, + MemType gpuMemType, + size_t maxBytesPerSubExec, + int totalGpus) +{ + int numFail = 0; + + // Collect some topology information + int numRanks = TransferBench::GetNumRanks(); + + std::vector transfers; + std::vector allTransfers; + TestResults results; + char transferStr[128] = {}; + + // What to print on pass/fail/skip + std::string pass = "\xE2\x9C\x93"; + std::string fail = "X"; + std::string skip = "."; + + // Different test categories + bool isH2D = (testNum == 1 || testNum == 8); + bool isD2H = (testNum == 2 || testNum == 9); + bool isD2D_RW = (testNum == 3 || testNum == 10); + bool isD2D_RR = (testNum == 4 || testNum == 11); + bool isBroadcast = (testNum == 5 || testNum == 12); + bool isGather = (testNum == 6 || testNum == 13); + bool isAllToAll = (testNum == 7 || testNum == 14); + + // Determine executor type + ExeType exeType; + if (1 <= testNum && testNum <= 7) exeType = EXE_GPU_DMA; + else if (8 <= testNum && testNum <= 14) exeType = EXE_GPU_GFX; + else { + Utils::Print("[ERROR] Unsupported test number %d\n", testNum); + exit(1); + } + + // Adjust number of subexecutors per transfer if performing multiple transfers + int numSubExec = exeType == EXE_GPU_DMA ? 1 : numSubExecPerGpu; + if (exeType == EXE_GPU_GFX && (isBroadcast || isGather || isAllToAll)) + numSubExec = std::max(1, numSubExecPerGpu / totalGpus); + + for (size_t numBytes : sizeList) { + + // Print skip symbol for skipped tests + if (!testsToRun.count(testNum)) { + Utils::Print("%s", skip.c_str()); fflush(stdout); + continue; + } + if (exeType == EXE_GPU_GFX && + (numSubExec * cfg.data.blockBytes > numBytes || + numSubExec * maxBytesPerSubExec < numBytes)) { + Utils::Print("%s", skip.c_str()); fflush(stdout); + continue; + } + // Skip test that require pod + if (numRanks > 1 && Utils::GetRankPerPodMap().size() != 1 && !(isH2D || isD2H)) { + Utils::Print("%s", skip.c_str()); fflush(stdout); + continue; + } + + bool allPass = true; + allTransfers.clear(); + + // Combine transfers from each GPU and run them all in parallel + for (int rank = 0; allPass && rank < numRanks; rank++) { + int numGpus = GetNumExecutors(exeType, rank); + for (int gpuIdx = 0; allPass && gpuIdx < numGpus; gpuIdx++) { + if (isH2D | isD2H) { + // Copy to/from closest CPU NUMA node for this GPU + int cpuIdx = GetClosestCpuNumaToGpu(gpuIdx, rank); + sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R%d%c%d %d %lu)", + rank, MemTypeStr[isH2D ? cpuMemType : gpuMemType], isH2D ? cpuIdx : gpuIdx, + rank, ExeTypeStr[exeType], gpuIdx, + rank, MemTypeStr[isH2D ? gpuMemType : cpuMemType], isH2D ? gpuIdx : cpuIdx, + numSubExec, numBytes); + } else if (isD2D_RW || isD2D_RR) { + // Copy from this GPU to "next" GPU + int dstRank = rank, dstGpuIdx = gpuIdx + 1; + if (dstGpuIdx >= GetNumExecutors(exeType, dstRank)) { + dstGpuIdx = 0; + dstRank = (rank+1) % numRanks; + } + sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R%d%c%d %d %lu)", + rank, MemTypeStr[gpuMemType], gpuIdx, + isD2D_RW ? rank : dstRank, ExeTypeStr[exeType], isD2D_RW ? gpuIdx : dstGpuIdx, + dstRank, MemTypeStr[gpuMemType], dstGpuIdx, + numSubExec, numBytes); + } else if (isBroadcast) { + // Split up the number of CUs across all Transfers + sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R*%c* %d %lu)", + rank, MemTypeStr[gpuMemType], gpuIdx, + rank, ExeTypeStr[exeType], gpuIdx, + MemTypeStr[gpuMemType], + numSubExec, numBytes); + } else if (isGather) { + // Split up the number of CUs across all Transfers + sprintf(transferStr, "-1 (R*%c* R%d%c%d R%d%c%d %d %lu)", + MemTypeStr[gpuMemType], + rank, ExeTypeStr[exeType], gpuIdx, + rank, MemTypeStr[gpuMemType], gpuIdx, + numSubExec, numBytes); + } else if (isAllToAll) { + // Split up the number of CUs across all Transfers + sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R*%c* %d %lu)", + rank, MemTypeStr[gpuMemType], gpuIdx, + rank, ExeTypeStr[exeType], gpuIdx, + MemTypeStr[gpuMemType], + numSubExec, numBytes); + } + + ErrResult err = ParseTransfers(transferStr, transfers); + if (err.errType != ERR_NONE) { + Utils::Print("[ERROR] Unexpected parsing error - %s. This is a coding error\n", err.errMsg.c_str()); + exit(1); + } + + allTransfers.insert(allTransfers.end(), transfers.begin(), transfers.end()); + } + } + if (!RunTransfers(cfg, allTransfers, results)) { + allPass = false; + } + Utils::Print("%s", allPass ? pass.c_str() : fail.c_str()); fflush(stdout); + numFail += (allPass ? 0 : 1); + } + return numFail; +} + +int SmokeTestPreset(EnvVars& ev, + size_t const numBytesPerTransfer, + std::string const presetName, + bool const bytesSpecified) +{ + // Check for single pod + if (Utils::GetRankPerPodMap().size() > 1) { + Utils::Print("[ERROR] %s preset can only be run within a single pod\n", presetName.c_str()); + Utils::Print("[ERROR] Pod membership may be forced by setting TB_FORCE_SINGLE_POD=1\n"); + return 1; + } + + // Collect topology and check that all GPUs have the same number of subExecutors + int numRanks = TransferBench::GetNumRanks(); + int totalGpus = 0; + int numSubExec = TransferBench::GetNumSubExecutors({EXE_GPU_GFX, 0, 0}); + for (int rank = 0; rank < numRanks; rank++) { + int numGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX, rank); + totalGpus += numGpus; + for (int gpu = 0; gpu < numGpus; gpu++) { + if (numSubExec != TransferBench::GetNumSubExecutors({EXE_GPU_GFX, gpu, rank})) { + Utils::Print("[ERROR] %s preset can only be run on GPUs with the same number of subexecutors\n", presetName.c_str()); + return 1; + } + } + } + + // Modify defaults unless they were set + if (!getenv("ALWAYS_VALIDATE")) ev.alwaysValidate = 1; + if (!getenv("NUM_ITERATIONS" )) ev.numIterations = 2; + if (!getenv("NUM_WARMUPS" )) ev.numWarmups = 0; + + // Collect env vars + int cpuMemTypeIdx = EnvVars::GetEnvVar ("CPU_MEM_TYPE", 0); + int gpuMemTypeIdx = EnvVars::GetEnvVar ("GPU_MEM_TYPE", 0); + vector gfxSesList = EnvVars::GetEnvVarArray ("GFX_SE_LIST", {1,numSubExec}); + std::string seMaxBytesStr = EnvVars::GetEnvVar ("SE_MAX_BYTES", "128M"); + vector sizeStrList = EnvVars::GetEnvVarStrArray ("SIZE_LIST", {"1K","16M","256M"}); + vector testList = EnvVars::GetEnvVarRangeArray("TEST_LIST", {}); + + MemType cpuMemType = Utils::GetCpuMemType(cpuMemTypeIdx); + MemType gpuMemType = Utils::GetGpuMemType(gpuMemTypeIdx); + std::set testsToRun(testList.begin(), testList.end()); + if (testList.empty()) { + for (int testIdx = 1; testIdx <= NUM_SMOKE_TESTS; testIdx++) + testsToRun.insert(testIdx); + } + + vector sizeList; + if (bytesSpecified) { + sizeList = {numBytesPerTransfer}; + } else { + for (auto s : sizeStrList) { + size_t val; + if (sscanf(s.c_str(), "%lu", &val) == 1) { + switch (s[s.size()-1]) { + case 'G': case 'g': val *= 1024; + case 'M': case 'm': val *= 1024; + case 'K': case 'k': val *= 1024; + } + sizeList.push_back(val); + } + } + } + size_t seMaxBytes = 128 * 1024 * 1024; + if (sscanf(seMaxBytesStr.c_str(), " %lu", &seMaxBytes) == 1) { + switch (seMaxBytesStr[seMaxBytesStr.size()-1]) { + case 'G': case 'g': seMaxBytes *= 1024; + case 'M': case 'm': seMaxBytes *= 1024; + case 'K': case 'k': seMaxBytes *= 1024; + } + } + + TransferBench::ConfigOptions cfg = ev.ToConfigOptions(); + + // Print off environment variables + if (Utils::RankDoesOutput()) { + ev.DisplayEnvVars(); + if (!ev.hideEnv) { + if (!ev.outputToCsv) printf("[%s-preset Related]\n", presetName.c_str()); + ev.Print("CPU_MEM_TYPE", cpuMemTypeIdx, "Using %s (%s)", Utils::GetCpuMemTypeStr(cpuMemTypeIdx).c_str(), Utils::GetAllCpuMemTypeStr().c_str()); + ev.Print("GFX_SE_LIST" , gfxSesList.size(), "Testing GFX with subexecutor counts: %s", EnvVars::ToStr(gfxSesList).c_str()); + ev.Print("GPU_MEM_TYPE", gpuMemTypeIdx, "Using %s (%s)", Utils::GetGpuMemTypeStr(gpuMemTypeIdx).c_str(), Utils::GetAllGpuMemTypeStr().c_str()); + ev.Print("SIZE_LIST" , sizeStrList.size(), "Transfer sizes tested: %s", ev.GetStr(sizeStrList).c_str()); + ev.Print("SE_MAX_BYTES", seMaxBytesStr, "Each SubExecutor can work on at most %lu bytes", seMaxBytes); + ev.Print("TEST_LIST" , testsToRun.size(), testList.empty() ? "Running all tests" : "Running Tests: %s", ev.GetStr(testList).c_str()); + printf("\n"); + } + } + + // Calculate cell-spacing / padding + int numSizes = sizeList.size(); + int colSize = std::max(5, 2 + numSizes); + int lPad1Size = (colSize - 3) / 2, rPad1Size = colSize - lPad1Size - 3; + int lPad2Size = (colSize - numSizes) / 2, rPad2Size = colSize - lPad2Size - numSizes; + + std::string l1(lPad1Size, ' '), r1(rPad1Size, ' '); + std::string l2(lPad2Size, ' '), r2(rPad2Size, ' '); + + int testsFailed = 0; + auto test = [&](int x, int y) { + Utils::Print(" %02d |%s", x, l2.c_str()); + testsFailed += RunTest(x, testsToRun, sizeList, 1, cfg, cpuMemType, gpuMemType, seMaxBytes, totalGpus); + Utils::Print("%s| %02d |", r2.c_str(), y); + for (auto numSubExec : gfxSesList) { + Utils::Print("%s", l2.c_str()); + testsFailed += RunTest(y, testsToRun, sizeList, numSubExec, cfg, cpuMemType, gpuMemType, seMaxBytes, totalGpus); + Utils::Print("%s|", r2.c_str()); + } + Utils::Print("\n"); + }; + + Utils::Print("Running tests on %d GPUs total across %d rank(s)\n\n", totalGpus, numRanks); + + // Print headers + Utils::Print(" %s %s |", l1.c_str(), r1.c_str()); + for ([[maybe_unused]] auto numSubExec : gfxSesList) + Utils::Print("%sGFX%s|", l1.c_str(), r1.c_str()); + Utils::Print("\n"); + Utils::Print("| Name | Test |%sDMA%s| Test |", l1.c_str(), r1.c_str()); + for (auto numSubExec : gfxSesList) + Utils::Print("%s%03d%s|", l1.c_str(), numSubExec, r1.c_str()); + Utils::Print("\n"); + Utils::Print("|---------------------------|------|%s|------|", std::string(colSize, '-').c_str()); + for ([[maybe_unused]] auto numSubExec : gfxSesList) + Utils::Print("%s|", std::string(colSize, '-').c_str()); + Utils::Print("\n"); + + // Print table / Run Tests + Utils::Print("| Copy (H2D) |"); test(1, 8); + Utils::Print("| Copy (D2H) |"); test(2, 9); + Utils::Print("| Copy (D2D) (Remote Write) |"); test(3,10); + Utils::Print("| Copy (D2D) (Remote Read ) |"); test(4,11); + Utils::Print("| Broadcast (One to All) |"); test(5,12); + Utils::Print("| Gather (All to One) |"); test(6,13); + Utils::Print("| All To All |"); test(7,14); + + Utils::Print("|---------------------------|------|%s|------|", std::string(colSize, '-').c_str()); + for ([[maybe_unused]] auto numSubExec : gfxSesList) + Utils::Print("%s|", std::string(colSize, '-').c_str()); + Utils::Print("\n\n"); + + // Show summary + if (testsFailed) { + Utils::Print("[WARN] %d Tests FAILED\n", testsFailed); + } else { + Utils::Print("All tests passed\n"); + } + if (numRanks > 1 && Utils::GetRankPerPodMap().size() != 1) { + Utils::Print("[WARN] Copy (D2D) / Broadcast / Gather / AllToAll tests are skipped if ranks are not in same pod\n"); + } + if (Utils::HasDuplicateHostname()) { + Utils::Print("[WARN] It is recommended to run TransferBench with one rank per host to avoid potential aliasing of executors\n"); + } + return 0; +} + +} diff --git a/src/client/Utilities.hpp b/src/client/Utilities.hpp index 381d97c4..e5797690 100644 --- a/src/client/Utilities.hpp +++ b/src/client/Utilities.hpp @@ -769,5 +769,4 @@ namespace TransferBench::Utils { return (TransferBench::DeallocateMemory(memType, memPtr, bytes).errType != TransferBench::ERR_NONE); } - }; diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index de177866..6abe80f9 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3805,7 +3805,6 @@ static bool IsConfiguredGid(union ibv_gid const& gid) transferIdx, i, dstIdx, t.dsts[dstIdx].memRank, expected[i], output[i]}; } } - return {ERR_FATAL, "Transfer %d: Unexpected output mismatch for destination %d", transferIdx, dstIdx}; } } } From 6d94b95988784bcab567fc9e7ed0ddb6ce4805e7 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sat, 25 Apr 2026 05:02:43 -0500 Subject: [PATCH 2/7] Adjusting broadcast/allgather --- src/client/Presets/SmokeTest.hpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index 2d7eb9bc..a3ddfd28 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -149,11 +149,20 @@ int RunTest(int testNum, exit(1); } - allTransfers.insert(allTransfers.end(), transfers.begin(), transfers.end()); + if (isBroadcast || isGather) { + if (!RunTransfers(cfg, transfers, results)) { + allPass = false; + break; + } + } else { + allTransfers.insert(allTransfers.end(), transfers.begin(), transfers.end()); + } } } - if (!RunTransfers(cfg, allTransfers, results)) { - allPass = false; + if (!(isBroadcast || isGather)) { + if (!RunTransfers(cfg, allTransfers, results)) { + allPass = false; + } } Utils::Print("%s", allPass ? pass.c_str() : fail.c_str()); fflush(stdout); numFail += (allPass ? 0 : 1); From 2a41b367dd6f002d63c2375854fada67cdc02c8e Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sat, 25 Apr 2026 05:09:00 -0500 Subject: [PATCH 3/7] Switching to using P to denote pass to make it easier to copy/paste --- src/client/Presets/SmokeTest.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index a3ddfd28..97abd5dc 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -47,7 +47,7 @@ int RunTest(int testNum, char transferStr[128] = {}; // What to print on pass/fail/skip - std::string pass = "\xE2\x9C\x93"; + std::string pass = "P"; std::string fail = "X"; std::string skip = "."; From 14d65daf22b48c108555f7fab7d99ebf1fda3eeb Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sat, 25 Apr 2026 05:19:51 -0500 Subject: [PATCH 4/7] Changing display again --- src/client/Presets/SmokeTest.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index 97abd5dc..e713f71b 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -47,8 +47,8 @@ int RunTest(int testNum, char transferStr[128] = {}; // What to print on pass/fail/skip - std::string pass = "P"; - std::string fail = "X"; + std::string pass = "*"; + std::string fail = "F"; std::string skip = "."; // Different test categories From 75aefae15fef740c31da642d8741987f9b13379f Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sat, 25 Apr 2026 14:45:00 -0500 Subject: [PATCH 5/7] Minor fixes --- src/client/EnvVars.hpp | 4 +- src/client/Presets/SmokeTest.hpp | 67 +++++++++++++++++--------------- src/header/TransferBench.hpp | 1 + 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index c3ec3af4..4f76ea23 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -587,7 +587,7 @@ class EnvVars std::string GetStr(std::vector const& varnameList) const { std::string result = ""; - for (int i = 0; i < varnameList.size(); i++) { + for (auto i = 0; i < varnameList.size(); i++) { if (i) result += ","; result += std::to_string(varnameList[i]); } @@ -596,7 +596,7 @@ class EnvVars std::string GetStr(std::vector const& varnameList) const { std::string result = ""; - for (int i = 0; i < varnameList.size(); i++) { + for (auto i = 0; i < varnameList.size(); i++) { if (i) result += ","; result += varnameList[i]; } diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index e713f71b..3aa9fd9f 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -25,6 +25,12 @@ THE SOFTWARE. namespace { #define NUM_SMOKE_TESTS 14 +#define MAX_TRANSFER_STRLEN 128 + +// What to print on pass/fail/skip +const std::string pass = "*"; +const std::string fail = "F"; +const std::string skip = "."; int RunTest(int testNum, std::set const& testsToRun, @@ -44,12 +50,8 @@ int RunTest(int testNum, std::vector transfers; std::vector allTransfers; TestResults results; - char transferStr[128] = {}; + char transferStr[MAX_TRANSFER_STRLEN] = {}; - // What to print on pass/fail/skip - std::string pass = "*"; - std::string fail = "F"; - std::string skip = "."; // Different test categories bool isH2D = (testNum == 1 || testNum == 8); @@ -100,14 +102,14 @@ int RunTest(int testNum, for (int rank = 0; allPass && rank < numRanks; rank++) { int numGpus = GetNumExecutors(exeType, rank); for (int gpuIdx = 0; allPass && gpuIdx < numGpus; gpuIdx++) { - if (isH2D | isD2H) { + if (isH2D || isD2H) { // Copy to/from closest CPU NUMA node for this GPU int cpuIdx = GetClosestCpuNumaToGpu(gpuIdx, rank); - sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R%d%c%d %d %lu)", - rank, MemTypeStr[isH2D ? cpuMemType : gpuMemType], isH2D ? cpuIdx : gpuIdx, - rank, ExeTypeStr[exeType], gpuIdx, - rank, MemTypeStr[isH2D ? gpuMemType : cpuMemType], isH2D ? gpuIdx : cpuIdx, - numSubExec, numBytes); + snprintf(transferStr, MAX_TRANSFER_STRLEN, "-1 (R%d%c%d R%d%c%d R%d%c%d %d %lu)", + rank, MemTypeStr[isH2D ? cpuMemType : gpuMemType], isH2D ? cpuIdx : gpuIdx, + rank, ExeTypeStr[exeType], gpuIdx, + rank, MemTypeStr[isH2D ? gpuMemType : cpuMemType], isH2D ? gpuIdx : cpuIdx, + numSubExec, numBytes); } else if (isD2D_RW || isD2D_RR) { // Copy from this GPU to "next" GPU int dstRank = rank, dstGpuIdx = gpuIdx + 1; @@ -115,32 +117,32 @@ int RunTest(int testNum, dstGpuIdx = 0; dstRank = (rank+1) % numRanks; } - sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R%d%c%d %d %lu)", - rank, MemTypeStr[gpuMemType], gpuIdx, - isD2D_RW ? rank : dstRank, ExeTypeStr[exeType], isD2D_RW ? gpuIdx : dstGpuIdx, - dstRank, MemTypeStr[gpuMemType], dstGpuIdx, - numSubExec, numBytes); + snprintf(transferStr, MAX_TRANSFER_STRLEN, "-1 (R%d%c%d R%d%c%d R%d%c%d %d %lu)", + rank, MemTypeStr[gpuMemType], gpuIdx, + isD2D_RW ? rank : dstRank, ExeTypeStr[exeType], isD2D_RW ? gpuIdx : dstGpuIdx, + dstRank, MemTypeStr[gpuMemType], dstGpuIdx, + numSubExec, numBytes); } else if (isBroadcast) { // Split up the number of CUs across all Transfers - sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R*%c* %d %lu)", - rank, MemTypeStr[gpuMemType], gpuIdx, - rank, ExeTypeStr[exeType], gpuIdx, - MemTypeStr[gpuMemType], - numSubExec, numBytes); + snprintf(transferStr, MAX_TRANSFER_STRLEN, "-1 (R%d%c%d R%d%c%d R*%c* %d %lu)", + rank, MemTypeStr[gpuMemType], gpuIdx, + rank, ExeTypeStr[exeType], gpuIdx, + MemTypeStr[gpuMemType], + numSubExec, numBytes); } else if (isGather) { // Split up the number of CUs across all Transfers - sprintf(transferStr, "-1 (R*%c* R%d%c%d R%d%c%d %d %lu)", - MemTypeStr[gpuMemType], - rank, ExeTypeStr[exeType], gpuIdx, - rank, MemTypeStr[gpuMemType], gpuIdx, - numSubExec, numBytes); + snprintf(transferStr, MAX_TRANSFER_STRLEN, "-1 (R*%c* R%d%c%d R%d%c%d %d %lu)", + MemTypeStr[gpuMemType], + rank, ExeTypeStr[exeType], gpuIdx, + rank, MemTypeStr[gpuMemType], gpuIdx, + numSubExec, numBytes); } else if (isAllToAll) { // Split up the number of CUs across all Transfers - sprintf(transferStr, "-1 (R%d%c%d R%d%c%d R*%c* %d %lu)", - rank, MemTypeStr[gpuMemType], gpuIdx, - rank, ExeTypeStr[exeType], gpuIdx, - MemTypeStr[gpuMemType], - numSubExec, numBytes); + snprintf(transferStr, MAX_TRANSFER_STRLEN, "-1 (R%d%c%d R%d%c%d R*%c* %d %lu)", + rank, MemTypeStr[gpuMemType], gpuIdx, + rank, ExeTypeStr[exeType], gpuIdx, + MemTypeStr[gpuMemType], + numSubExec, numBytes); } ErrResult err = ParseTransfers(transferStr, transfers); @@ -282,7 +284,8 @@ int SmokeTestPreset(EnvVars& ev, Utils::Print("\n"); }; - Utils::Print("Running tests on %d GPUs total across %d rank(s)\n\n", totalGpus, numRanks); + Utils::Print("Running tests on %d GPUs total across %d rank(s)\n", totalGpus, numRanks); + Utils::Print("Legend: %s=Pass %s=Skip %s=Fail\n", pass.c_str(), skip.c_str(), fail.c_str()); // Print headers Utils::Print(" %s %s |", l1.c_str(), r1.c_str()); diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 6abe80f9..de177866 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3805,6 +3805,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) transferIdx, i, dstIdx, t.dsts[dstIdx].memRank, expected[i], output[i]}; } } + return {ERR_FATAL, "Transfer %d: Unexpected output mismatch for destination %d", transferIdx, dstIdx}; } } } From cf630118878f31b144d6b4f03cfa24ef0cc6e73a Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sat, 25 Apr 2026 14:52:13 -0500 Subject: [PATCH 6/7] Modifying return code --- src/client/Presets/SmokeTest.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index 3aa9fd9f..e63b59bb 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -327,7 +327,7 @@ int SmokeTestPreset(EnvVars& ev, if (Utils::HasDuplicateHostname()) { Utils::Print("[WARN] It is recommended to run TransferBench with one rank per host to avoid potential aliasing of executors\n"); } - return 0; + return testFailed ? 1 : 0; } } From 3b4012994ca8c9fbdf820c6bc2c6352a3248a235 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sat, 25 Apr 2026 18:30:02 -0500 Subject: [PATCH 7/7] Fixing typo --- src/client/Presets/SmokeTest.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/Presets/SmokeTest.hpp b/src/client/Presets/SmokeTest.hpp index e63b59bb..42984644 100644 --- a/src/client/Presets/SmokeTest.hpp +++ b/src/client/Presets/SmokeTest.hpp @@ -327,7 +327,7 @@ int SmokeTestPreset(EnvVars& ev, if (Utils::HasDuplicateHostname()) { Utils::Print("[WARN] It is recommended to run TransferBench with one rank per host to avoid potential aliasing of executors\n"); } - return testFailed ? 1 : 0; + return testsFailed ? 1 : 0; } }