From 02851a964a5f7affaaa282c35ce856b9c7fd741d Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Sun, 26 Apr 2026 23:13:01 -0500 Subject: [PATCH 1/2] a2asweep: reformat output to match gfxsweep - Default BLOCKSIZES changed from {256} to {256,512,768,1024} - Single unified table: header printed once, blockSize is a row column - USE_HIP_EVENTS controls timing mode (preset default: 1 = GPU-event timed) USE_HIP_EVENTS=1: GPU-event-timed per-executor minBw; SHOW_MIN_ONLY=0 adds a maxBw column per SE USE_HIP_EVENTS=0: CPU wall-clock avgTotalBandwidthGbPerSec - Banner and best-result summary reflect the active timing mode - Results map keyed by (blockSize, numSes, unroll) for verbose output - Increase column width for output values - Fixed [WARN} typo Co-authored-by: Claude --- src/client/Presets/AllToAllSweep.hpp | 104 +++++++++++++++++---------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/src/client/Presets/AllToAllSweep.hpp b/src/client/Presets/AllToAllSweep.hpp index dc0314e7..3e9c0afe 100644 --- a/src/client/Presets/AllToAllSweep.hpp +++ b/src/client/Presets/AllToAllSweep.hpp @@ -43,6 +43,8 @@ int AllToAllSweepPreset(EnvVars& ev, // Force single-stream mode for all-to-all benchmark ev.useSingleStream = 1; + // Default to GPU-event timing for a2asweep (overridable via USE_HIP_EVENTS=0 for CPU wall-clock) + ev.useHipEvents = EnvVars::GetEnvVar("USE_HIP_EVENTS", 1); int numDetectedGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX); @@ -57,7 +59,7 @@ int AllToAllSweepPreset(EnvVars& ev, int useSpray = EnvVars::GetEnvVar("USE_SPRAY", 0); int verbose = EnvVars::GetEnvVar("VERBOSE", 0); - std::vector blockList = EnvVars::GetEnvVarArray("BLOCKSIZES", {256}); + std::vector blockList = EnvVars::GetEnvVarArray("BLOCKSIZES", {256,512,768,1024}); std::vector unrollList = EnvVars::GetEnvVarArray("UNROLLS", {1,2,3,4,6,8}); std::vector numSesList = EnvVars::GetEnvVarArray("NUM_SUB_EXECS", {4,8,12,16,24,32}); @@ -182,72 +184,100 @@ int AllToAllSweepPreset(EnvVars& ev, } } - printf("GPU-GFX All-To-All Sweep benchmark:\n"); - printf("==========================\n"); - printf("- Copying %lu bytes between %s pairs of GPUs\n", numBytesPerTransfer, a2aDirect ? "directly connected" : "all"); + Utils::Print("GPU-GFX All-To-All Sweep benchmark (%lu bytes, local=%s). All values are %s GB/s\n", + numBytesPerTransfer, + a2aLocal ? "yes" : "no", + ev.useHipEvents ? "GPU-Event-Timed (min over GPUs)": "CPU-Timed"); + Utils::Print("=======================================================================================\n"); if (transfers.size() == 0) { - printf("[WARN} No transfers requested. Try adjusting A2A_DIRECT or A2A_LOCAL\n"); + Utils::Print("[WARN] No transfers requested. Try adjusting A2A_DIRECT or A2A_LOCAL\n"); return 0; } // Execute Transfers TransferBench::ConfigOptions cfg = ev.ToConfigOptions(); - // Run tests - std::map, TransferBench::TestResults> results; + char sep = ev.outputToCsv ? ',' : ' '; + + double bestMinBw = 0.0; + int bestBlock = -1, bestUnroll = -1, bestNumSes = -1; + + // Print header once + Utils::Print(" BlkS %c UnR ", sep); + for (int c : numSesList) { + Utils::Print("%c SE %03d", sep, c); + if (ev.useHipEvents && !showMinOnly) Utils::Print("%c SE%03dMx", sep, c); + } + Utils::Print("\n"); + + // Results keyed by (blockSize, numSes, unroll) for verbose output + std::map, TransferBench::TestResults> results; - // Display summary for (int blockSize : blockList) { - printf("Blocksize: %d\n", blockSize); - ev.gfxBlockSize = cfg.gfx.blockSize = blockSize; + cfg.gfx.blockSize = blockSize; - printf("#CUs\\Unroll"); for (int u : unrollList) { - printf(" %d(Min) ", u); - if (!showMinOnly) printf(" %d(Max) ", u); - } - printf("\n"); - for (int c : numSesList) { - printf(" %5d ", c); fflush(stdout); - for (int u : unrollList) { - ev.gfxUnroll = cfg.gfx.unrollFactor = u; + cfg.gfx.unrollFactor = u; + Utils::Print("%5d %c %3d ", blockSize, sep, u); + fflush(stdout); + + for (int c : numSesList) { for (auto& transfer : transfers) transfer.numSubExecs = useSpray ? (c * targetCount) : c; - double minBandwidth = std::numeric_limits::max(); - double maxBandwidth = std::numeric_limits::lowest(); TransferBench::TestResults result; + double minBw = 0.0, maxBw = 0.0; if (TransferBench::RunTransfers(cfg, transfers, result)) { - for (auto const& exeResult : result.exeResults) { - minBandwidth = std::min(minBandwidth, exeResult.second.avgBandwidthGbPerSec); - maxBandwidth = std::max(maxBandwidth, exeResult.second.avgBandwidthGbPerSec); + if (!ev.useHipEvents) { + minBw = result.avgTotalBandwidthGbPerSec; + if (useSpray) minBw *= targetCount; + } else { + minBw = std::numeric_limits::max(); + maxBw = std::numeric_limits::lowest(); + for (auto const& exeResult : result.exeResults) { + minBw = std::min(minBw, exeResult.second.avgBandwidthGbPerSec); + maxBw = std::max(maxBw, exeResult.second.avgBandwidthGbPerSec); + } + if (useSpray) { minBw *= targetCount; maxBw *= targetCount; } } - if (useSpray) { - minBandwidth *= targetCount; - maxBandwidth *= targetCount; + if (minBw > bestMinBw) { + bestMinBw = minBw; + bestBlock = blockSize; + bestUnroll = u; + bestNumSes = c; } - results[std::make_pair(c,u)] = result; - } else { - minBandwidth = 0.0; + results[std::make_tuple(blockSize, c, u)] = result; } - printf(" %7.2f ", minBandwidth); - if (!showMinOnly) printf(" %7.2f ", maxBandwidth); + Utils::Print("%c%8.2f", sep, minBw); + if (ev.useHipEvents && !showMinOnly) + Utils::Print("%c%8.2f", sep, maxBw); fflush(stdout); } - printf("\n"); fflush(stdout); + Utils::Print("\n"); + fflush(stdout); } + } + Utils::Print("=======================================================================================\n"); - if (verbose) { - int testNum = 0; + if (verbose) { + int testNum = 0; + for (int blockSize : blockList) { for (int c : numSesList) { for (int u : unrollList) { - printf("SubExecs: %d Unroll %d\n", c, u); - Utils::PrintResults(ev, ++testNum, transfers, results[std::make_pair(c,u)]); + Utils::Print("BlockSize: %d SubExecs: %d Unroll: %d\n", blockSize, c, u); + Utils::PrintResults(ev, ++testNum, transfers, results[std::make_tuple(blockSize, c, u)]); } } } } + // Print combination that produced highest bandwidth + Utils::Print("Highest %s bandwidth found: %7.2f GB/s\n", + ev.useHipEvents ? "GPU-event-timed (min)" : "CPU-timed", bestMinBw); + Utils::Print(" BlockSize : %7d\n", bestBlock); + Utils::Print(" Unroll : %7d\n", bestUnroll); + Utils::Print(" NumSubExec : %7d\n", bestNumSes); + if (useFineGrain != -999) { Utils::Print("[WARN] USE_FINE_GRAIN has been deprecated and replaced by MEM_TYPE\n"); Utils::Print("[WARN] MEM_TYPE has been set to %d to correspond to previous use of USE_FINE_GRAIN=%d\n", memTypeIdx, useFineGrain); From 31660618714be9735e3105ddb3e634426f0779de Mon Sep 17 00:00:00 2001 From: nileshnegi Date: Sun, 26 Apr 2026 23:48:55 -0500 Subject: [PATCH 2/2] a2asweep: address PR #272 review feedback - Fix verbose block using stale numSubExecs: copy transfers per (blockSize,c,unroll) combination before calling PrintResults so subexec count matches stored result - Gate results map insertion on verbose flag to avoid storing all TestResults when VERBOSE=0 - Guard best-result summary block on bestBlock != -1 to suppress misleading -1 output if all RunTransfers calls fail - Widen value columns from %7.2f to %8.2f to accommodate 4-digit GB/s values - Add note on spray targetCount asymmetry for non-uniform A2A_DIRECT topologies Co-authored-by: Claude --- src/client/Presets/AllToAllSweep.hpp | 46 ++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/client/Presets/AllToAllSweep.hpp b/src/client/Presets/AllToAllSweep.hpp index 3e9c0afe..2e4b647d 100644 --- a/src/client/Presets/AllToAllSweep.hpp +++ b/src/client/Presets/AllToAllSweep.hpp @@ -156,7 +156,10 @@ int AllToAllSweepPreset(EnvVars& ev, } } } else { - // Each CU will work on all targets + // Each CU will work on all targets. + // NOTE: targetCount ends up reflecting the last GPU's target count. This is correct for + // symmetric topologies (all GPUs have equal peer counts), but may be inaccurate with + // A2A_DIRECT on asymmetric hardware where different GPUs have different hop-1 peer counts. for (int i = 0; i < numGpus; i++) { TransferBench::Transfer transfer; transfer.numBytes = numBytesPerTransfer; @@ -206,7 +209,9 @@ int AllToAllSweepPreset(EnvVars& ev, Utils::Print(" BlkS %c UnR ", sep); for (int c : numSesList) { Utils::Print("%c SE %03d", sep, c); - if (ev.useHipEvents && !showMinOnly) Utils::Print("%c SE%03dMx", sep, c); + if (ev.useHipEvents && !showMinOnly) { + Utils::Print("%c SE%03dMx", sep, c); + } } Utils::Print("\n"); @@ -222,15 +227,18 @@ int AllToAllSweepPreset(EnvVars& ev, fflush(stdout); for (int c : numSesList) { - for (auto& transfer : transfers) + for (auto& transfer : transfers) { transfer.numSubExecs = useSpray ? (c * targetCount) : c; + } TransferBench::TestResults result; double minBw = 0.0, maxBw = 0.0; if (TransferBench::RunTransfers(cfg, transfers, result)) { if (!ev.useHipEvents) { minBw = result.avgTotalBandwidthGbPerSec; - if (useSpray) minBw *= targetCount; + if (useSpray) { + minBw *= targetCount; + } } else { minBw = std::numeric_limits::max(); maxBw = std::numeric_limits::lowest(); @@ -238,7 +246,10 @@ int AllToAllSweepPreset(EnvVars& ev, minBw = std::min(minBw, exeResult.second.avgBandwidthGbPerSec); maxBw = std::max(maxBw, exeResult.second.avgBandwidthGbPerSec); } - if (useSpray) { minBw *= targetCount; maxBw *= targetCount; } + if (useSpray) { + minBw *= targetCount; + maxBw *= targetCount; + } } if (minBw > bestMinBw) { bestMinBw = minBw; @@ -246,11 +257,14 @@ int AllToAllSweepPreset(EnvVars& ev, bestUnroll = u; bestNumSes = c; } - results[std::make_tuple(blockSize, c, u)] = result; + if (verbose) { + results[std::make_tuple(blockSize, c, u)] = result; + } } Utils::Print("%c%8.2f", sep, minBw); - if (ev.useHipEvents && !showMinOnly) + if (ev.useHipEvents && !showMinOnly) { Utils::Print("%c%8.2f", sep, maxBw); + } fflush(stdout); } Utils::Print("\n"); @@ -264,19 +278,25 @@ int AllToAllSweepPreset(EnvVars& ev, for (int blockSize : blockList) { for (int c : numSesList) { for (int u : unrollList) { + auto verboseTransfers = transfers; + for (auto& t : verboseTransfers) { + t.numSubExecs = useSpray ? (c * targetCount) : c; + } Utils::Print("BlockSize: %d SubExecs: %d Unroll: %d\n", blockSize, c, u); - Utils::PrintResults(ev, ++testNum, transfers, results[std::make_tuple(blockSize, c, u)]); + Utils::PrintResults(ev, ++testNum, verboseTransfers, results[std::make_tuple(blockSize, c, u)]); } } } } // Print combination that produced highest bandwidth - Utils::Print("Highest %s bandwidth found: %7.2f GB/s\n", - ev.useHipEvents ? "GPU-event-timed (min)" : "CPU-timed", bestMinBw); - Utils::Print(" BlockSize : %7d\n", bestBlock); - Utils::Print(" Unroll : %7d\n", bestUnroll); - Utils::Print(" NumSubExec : %7d\n", bestNumSes); + if (bestBlock != -1) { + Utils::Print("Highest %s bandwidth found: %7.2f GB/s\n", + ev.useHipEvents ? "GPU-event-timed (min)" : "CPU-timed", bestMinBw); + Utils::Print(" BlockSize : %7d\n", bestBlock); + Utils::Print(" Unroll : %7d\n", bestUnroll); + Utils::Print(" NumSubExec : %7d\n", bestNumSes); + } if (useFineGrain != -999) { Utils::Print("[WARN] USE_FINE_GRAIN has been deprecated and replaced by MEM_TYPE\n");