From 4db94bd0baf52ae81f9409d55b168be604c2ad7e Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sun, 10 May 2026 02:11:45 -0500 Subject: [PATCH 1/5] Adding LaunchTransferBench helper script, and fixing multirank wallclock detection bug --- CHANGELOG.md | 1 + LaunchTransferBench.sh | 182 +++++++++++++++++++++++++++++++++++ src/header/TransferBench.hpp | 2 +- 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100755 LaunchTransferBench.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 443f667d..2e3e72c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Documentation for TransferBench is available at - Added new "nica2a" preset (NIC all-to-all over GPUs via NIC executors, multi-node) - Added new GFX_KERNEL to allow experimenting with copy-only GFX kernel. Currently this is opt-in only - Added `SHOW_PERCENTILES` (e.g. `50,75,90,95,99`) to show empirical percentiles of per-iteration duration +- Adding new LaunchTransferBench.sh script to simplify launching TransferBench across multiple nodes (via socket) ### Modified - DMA-BUF support enablement in CMake changed to ENABLE_DMA_BUF to be more similar to other compile-time options diff --git a/LaunchTransferBench.sh b/LaunchTransferBench.sh new file mode 100755 index 00000000..02ca6c93 --- /dev/null +++ b/LaunchTransferBench.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +# +# LaunchTransferBench - Multi-rank TransferBench Socket Execution Script +# +# This script simplifies the execution of socket-based multi-rank TransferBench +# by automatically setting up SSH connections to specified hosts and setting +# the appropriate environment variables. +# +# Usage: +# ./LaunchTransferBench.sh [env_vars...] [-- ] +# +# Arguments: +# hosts: Comma-separated list of hostnames/IPs to run on +# env_vars: Optional environment variables (e.g., NUM_ITERATIONS=10 NUM_SUBITERATIONS=100) +# transferbench_args: Arguments to pass to TransferBench (after --) +# +# Examples: +# ./LaunchTransferBench.sh node1,node2,node3,node4 NUM_ITERATIONS=10 NUM_SUBITERATIONS=100 -- a2a +# ./LaunchTransferBench.sh host1,host2 -- cmdline 1G "1 1 R0G0 R0D0 R1G0" +# ./LaunchTransferBench.sh server1,server2,server3 TB_MASTER_PORT=30000 -- example.cfg +# + +set -e + +# Function to display usage information +show_usage() { + cat << EOF +Usage: $0 [env_vars...] [-- ] + +Arguments: + hosts Comma-separated list of hostnames/IPs to run on + env_vars Optional environment variables (KEY=VALUE format) + transferbench_args Arguments to pass to TransferBench (after --) + +Environment Variables for TransferBench: + NUM_ITERATIONS Number of timed iterations to perform (default: 10) + NUM_SUBITERATIONS Number of subiterations to perform (default: 1) + NUM_WARMUPS Number of warmup iterations (default: 3) + TB_MASTER_PORT Port for rank 0 communication (default: 29500) + ... and many others (see TransferBench documentation) + +Examples: + $0 node1,node2,node3,node4 NUM_ITERATIONS=10 NUM_SUBITERATIONS=100 -- a2a + $0 host1,host2 -- cmdline 1G "1 1 R0G0 R0D0 R1G0" + $0 server1,server2,server3 TB_MASTER_PORT=30000 -- example.cfg + +Notes: + - The first host in the list becomes rank 0 (master) + - TransferBench must be built in the same directory as this script on all hosts + - SSH access must be configured for all hosts +EOF +} + + +# Parse command line arguments +if [[ $# -lt 1 ]]; then + echo + show_usage + exit 1 +fi + +# Parse hosts +hosts_input="$1" +shift + +if [[ -z "$hosts_input" ]]; then + echo "ERROR: No hosts specified" + show_usage + exit 1 +fi + +# Convert comma-separated hosts to array +IFS=',' read -ra hosts <<< "$hosts_input" +num_ranks=${#hosts[@]} + +if [[ $num_ranks -lt 2 ]]; then + echo "ERROR: At least 2 hosts are required for multi-rank execution" + echo "For single-node execution, run TransferBench directly without this script" + exit 1 +fi + +echo "Hosts : ${hosts[*]}" +echo "Ranks : $num_ranks" + +# Parse environment variables and TransferBench arguments +env_vars=() +tb_args=() +parsing_tb_args=false + +while [[ $# -gt 0 ]]; do + if [[ "$1" == "--" ]]; then + parsing_tb_args=true + shift + continue + fi + + if [[ $parsing_tb_args == true ]]; then + tb_args+=("$1") + elif [[ "$1" =~ ^[A-Za-z_][A-Za-z0-9_]*=.*$ ]]; then + env_vars+=("$1") + else + echo "ERROR: Invalid environment variable format: $1" + echo "Environment variables should be in KEY=VALUE format" + exit 1 + fi + shift +done + +echo "EnvVars : ${env_vars[*]:-none}" +if [[ ${#tb_args[@]} -eq 0 ]]; then + echo "Args : none (will show topology)" +else + echo "Args : ${tb_args[*]}" +fi + +# Get the absolute directory where this script is located +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +transferbench_path="$script_dir/TransferBench" + +echo + +# Build environment variable string +env_string="" +for env_var in "${env_vars[@]}"; do + env_string="$env_string $env_var" +done + +# Cleanup function for interruption +cleanup() { + echo + echo "Interrupted! Cleaning up worker processes..." >&2 + for pid in "${worker_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + fi + done + exit 130 +} + +# Set up signal handlers for Ctrl-C and termination +trap cleanup INT TERM + +# Start worker ranks in the background +master_host="${hosts[0]}" +worker_pids=() +worker_hosts=() + +# Build properly escaped arguments string +tb_args_escaped="" +for arg in "${tb_args[@]}"; do + tb_args_escaped+=" $(printf '%q' "$arg")" +done + +for ((rank=1; rank/dev/null 2>&1 & + worker_pids+=($!) + worker_hosts+=("$worker_host") +done + +# Start master rank (TransferBench will wait for all workers to connect) +master_cmd="TB_NUM_RANKS=$num_ranks TB_RANK=0 TB_SINGLE_LOG=1 $env_string '$transferbench_path'$tb_args_escaped" +if ! ssh -q -o LogLevel=ERROR "$master_host" "$master_cmd"; then + echo "ERROR: Master rank failed on $master_host" >&2 + exit 1 +fi + +# Check worker exit codes +any_worker_failed=false +for ((i=0; i<${#worker_pids[@]}; i++)); do + if ! wait "${worker_pids[$i]}"; then + rank=$((i+1)) + echo "ERROR: Worker rank $rank failed on ${worker_hosts[$i]}" >&2 + any_worker_failed=true + fi +done + +if [[ "$any_worker_failed" == "true" ]]; then + exit 1 +fi diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 6844a94c..9f4589c5 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -4408,7 +4408,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } // Check that GPU wallclock rate is non-zero - if (exeDevice.exeType == EXE_GPU_GFX && exeInfo.wallClockRate == 0) { + if (exeDevice.exeType == EXE_GPU_GFX && exeInfo.wallClockRate == 0 && exeDevice.exeRank == localRank) { if (getenv("TB_WALLCLOCK_RATE")) { exeInfo.wallClockRate = atoi(getenv("TB_WALLCLOCK_RATE")); return {ERR_WARN, From 732b8a9bd73976bcdd44106b002caa2e885ce759 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sun, 10 May 2026 02:25:50 -0500 Subject: [PATCH 2/5] Adding copyright header --- LaunchTransferBench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LaunchTransferBench.sh b/LaunchTransferBench.sh index 02ca6c93..2d74bc9c 100755 --- a/LaunchTransferBench.sh +++ b/LaunchTransferBench.sh @@ -1,6 +1,6 @@ #!/bin/bash +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# # LaunchTransferBench - Multi-rank TransferBench Socket Execution Script # # This script simplifies the execution of socket-based multi-rank TransferBench From b9547675806464b08e5ee7936223a03805f3d9f2 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sun, 10 May 2026 02:33:18 -0500 Subject: [PATCH 3/5] Cleaning up workers on error, hostname whitespace trim --- LaunchTransferBench.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/LaunchTransferBench.sh b/LaunchTransferBench.sh index 2d74bc9c..4c4c77f5 100755 --- a/LaunchTransferBench.sh +++ b/LaunchTransferBench.sh @@ -1,6 +1,8 @@ #!/bin/bash + # Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# # LaunchTransferBench - Multi-rank TransferBench Socket Execution Script # # This script simplifies the execution of socket-based multi-rank TransferBench @@ -55,7 +57,6 @@ EOF # Parse command line arguments if [[ $# -lt 1 ]]; then - echo show_usage exit 1 fi @@ -70,8 +71,23 @@ if [[ -z "$hosts_input" ]]; then exit 1 fi -# Convert comma-separated hosts to array -IFS=',' read -ra hosts <<< "$hosts_input" +# Convert comma-separated hosts to array and trim whitespace +IFS=',' read -ra hosts_raw <<< "$hosts_input" +hosts=() +for host in "${hosts_raw[@]}"; do + # Trim leading and trailing whitespace + host=$(echo "$host" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + if [[ -z "$host" ]]; then + echo "ERROR: Empty hostname found in host list" + exit 1 + fi + # Check for remaining whitespace in hostname + if [[ "$host" =~ [[:space:]] ]]; then + echo "ERROR: Hostname '$host' contains whitespace" + exit 1 + fi + hosts+=("$host") +done num_ranks=${#hosts[@]} if [[ $num_ranks -lt 2 ]]; then @@ -164,6 +180,8 @@ done master_cmd="TB_NUM_RANKS=$num_ranks TB_RANK=0 TB_SINGLE_LOG=1 $env_string '$transferbench_path'$tb_args_escaped" if ! ssh -q -o LogLevel=ERROR "$master_host" "$master_cmd"; then echo "ERROR: Master rank failed on $master_host" >&2 + # Clean up worker processes before exiting + cleanup exit 1 fi From 6fed0e888944f15d86b240555be2ce5c9dd401ac Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sun, 10 May 2026 02:50:40 -0500 Subject: [PATCH 4/5] Adding more env var checks, better cleanup --- LaunchTransferBench.sh | 49 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/LaunchTransferBench.sh b/LaunchTransferBench.sh index 4c4c77f5..4432e264 100755 --- a/LaunchTransferBench.sh +++ b/LaunchTransferBench.sh @@ -136,21 +136,64 @@ transferbench_path="$script_dir/TransferBench" echo -# Build environment variable string +# Build properly escaped environment variable string env_string="" for env_var in "${env_vars[@]}"; do - env_string="$env_string $env_var" + # Validate that env_var is in KEY=VALUE format + if [[ ! "$env_var" =~ ^[A-Za-z_][A-Za-z0-9_]*=.*$ ]]; then + echo "ERROR: Invalid environment variable format: $env_var" >&2 + exit 1 + fi + + # Split into key and value + key="${env_var%%=*}" + value="${env_var#*=}" + + # Escape the value and rebuild the env var + escaped_value=$(printf '%q' "$value") + env_string="$env_string $key=$escaped_value" done # Cleanup function for interruption cleanup() { echo echo "Interrupted! Cleaning up worker processes..." >&2 + + # Kill remote TransferBench processes first (only our own processes) + for ((i=0; i<${#worker_hosts[@]}; i++)); do + host="${worker_hosts[$i]}" + echo "Killing TransferBench on $host..." >&2 + # Kill only TransferBench processes owned by the current user + ssh -q -o LogLevel=ERROR -o ConnectTimeout=2 "$host" "pkill -u \$(whoami) -f TransferBench 2>/dev/null || true" 2>/dev/null & + done + + # Wait a moment for remote kills to take effect + sleep 2 + + # Now kill local SSH processes + for pid in "${worker_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + fi + done + + # Give local processes a moment to terminate + sleep 1 + + # Force kill any remaining local SSH processes for pid in "${worker_pids[@]}"; do if kill -0 "$pid" 2>/dev/null; then - kill "$pid" 2>/dev/null || true + echo "Force killing local SSH PID $pid..." >&2 + kill -KILL "$pid" 2>/dev/null || true fi done + + # Wait for all local processes to actually terminate + for pid in "${worker_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + + echo "Cleanup complete" >&2 exit 130 } From fba6e1eeb34e583123aa78ca8cd46920a969e4db Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Sun, 10 May 2026 02:59:02 -0500 Subject: [PATCH 5/5] Minor modificatons to process cleanup --- LaunchTransferBench.sh | 90 ++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 48 deletions(-) diff --git a/LaunchTransferBench.sh b/LaunchTransferBench.sh index 4432e264..604a4201 100755 --- a/LaunchTransferBench.sh +++ b/LaunchTransferBench.sh @@ -66,7 +66,7 @@ hosts_input="$1" shift if [[ -z "$hosts_input" ]]; then - echo "ERROR: No hosts specified" + echo "ERROR: No hosts specified" >&2 show_usage exit 1 fi @@ -78,12 +78,12 @@ for host in "${hosts_raw[@]}"; do # Trim leading and trailing whitespace host=$(echo "$host" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') if [[ -z "$host" ]]; then - echo "ERROR: Empty hostname found in host list" + echo "ERROR: Empty hostname found in host list" >&2 exit 1 fi # Check for remaining whitespace in hostname if [[ "$host" =~ [[:space:]] ]]; then - echo "ERROR: Hostname '$host' contains whitespace" + echo "ERROR: Hostname '$host' contains whitespace" >&2 exit 1 fi hosts+=("$host") @@ -91,8 +91,8 @@ done num_ranks=${#hosts[@]} if [[ $num_ranks -lt 2 ]]; then - echo "ERROR: At least 2 hosts are required for multi-rank execution" - echo "For single-node execution, run TransferBench directly without this script" + echo "ERROR: At least 2 hosts are required for multi-rank execution" >&2 + echo "For single-node execution, run TransferBench directly without this script" >&2 exit 1 fi @@ -116,8 +116,8 @@ while [[ $# -gt 0 ]]; do elif [[ "$1" =~ ^[A-Za-z_][A-Za-z0-9_]*=.*$ ]]; then env_vars+=("$1") else - echo "ERROR: Invalid environment variable format: $1" - echo "Environment variables should be in KEY=VALUE format" + echo "ERROR: Invalid environment variable format: $1" >&2 + echo "Environment variables should be in KEY=VALUE format" >&2 exit 1 fi shift @@ -139,13 +139,7 @@ echo # Build properly escaped environment variable string env_string="" for env_var in "${env_vars[@]}"; do - # Validate that env_var is in KEY=VALUE format - if [[ ! "$env_var" =~ ^[A-Za-z_][A-Za-z0-9_]*=.*$ ]]; then - echo "ERROR: Invalid environment variable format: $env_var" >&2 - exit 1 - fi - - # Split into key and value + # Split into key and value (validation already done during parsing) key="${env_var%%=*}" value="${env_var#*=}" @@ -156,42 +150,42 @@ done # Cleanup function for interruption cleanup() { - echo + echo >&2 echo "Interrupted! Cleaning up worker processes..." >&2 - # Kill remote TransferBench processes first (only our own processes) - for ((i=0; i<${#worker_hosts[@]}; i++)); do - host="${worker_hosts[$i]}" - echo "Killing TransferBench on $host..." >&2 - # Kill only TransferBench processes owned by the current user - ssh -q -o LogLevel=ERROR -o ConnectTimeout=2 "$host" "pkill -u \$(whoami) -f TransferBench 2>/dev/null || true" 2>/dev/null & - done - - # Wait a moment for remote kills to take effect - sleep 2 - - # Now kill local SSH processes - for pid in "${worker_pids[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - kill -TERM "$pid" 2>/dev/null || true - fi - done - - # Give local processes a moment to terminate - sleep 1 - - # Force kill any remaining local SSH processes - for pid in "${worker_pids[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - echo "Force killing local SSH PID $pid..." >&2 - kill -KILL "$pid" 2>/dev/null || true - fi - done - - # Wait for all local processes to actually terminate - for pid in "${worker_pids[@]}"; do - wait "$pid" 2>/dev/null || true - done + # First kill local SSH processes to stop remote TransferBench + if [[ ${#worker_pids[@]} -gt 0 ]]; then + for pid in "${worker_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + fi + done + + # Give SSH processes a moment to terminate and clean up remote processes + sleep 2 + + # Force kill any remaining local SSH processes + for pid in "${worker_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "Force killing SSH PID $pid..." >&2 + kill -KILL "$pid" 2>/dev/null || true + fi + done + + # Brief wait for cleanup, but don't hang + for pid in "${worker_pids[@]}"; do + # Wait with timeout - if process doesn't exit in 1 second, move on + timeout 1 bash -c "wait $pid" 2>/dev/null || true + done + fi + + # Final cleanup: kill any remaining TransferBench processes on all hosts + if [[ ${#worker_hosts[@]} -gt 0 ]]; then + for host in "${worker_hosts[@]}"; do + ssh -q -o LogLevel=ERROR -o ConnectTimeout=1 "$host" "pkill -u \$(whoami) -f TransferBench 2>/dev/null || true" 2>/dev/null || true & + done + # Don't wait for these - let them complete in background + fi echo "Cleanup complete" >&2 exit 130