Add Logic for Using TDM and Async Loads and Stores

Makefile

@@ -6,7 +6,15 @@
 ROCM_PATH ?= /opt/rocm
 CUDA_PATH ?= /usr/local/cuda
 MPI_PATH  ?= /usr/local/openmpi
+# pip ROCm wheels ship bin/amdclang++ but often omit bin/amdllvm (which that stub execs).
+# Default to llvm/bin when bin/amdllvm is absent so HIP builds work without extra flags.
+ifeq ("$(shell test -e $(ROCM_PATH)/bin/amdllvm && echo found)", "found")
 HIPCC     ?= $(ROCM_PATH)/bin/amdclang++
+else ifeq ("$(shell test -e $(ROCM_PATH)/llvm/bin/amdclang++ && echo found)", "found")
+HIPCC     ?= $(ROCM_PATH)/llvm/bin/amdclang++
+else
+HIPCC     ?= $(ROCM_PATH)/bin/amdclang++
+endif
 NVCC      ?= $(CUDA_PATH)/bin/nvcc
 DEBUG     ?= 0
 

src/client/EnvVars.hpp

@@ -124,6 +124,11 @@ class EnvVars
   int nicTrafficClass;               // DSCP/traffic class byte for RoCE GRH
   int roceVersion;                   // RoCE version number
 
+  // TDM (async tensor) options
+  int tdmBlockSize;                  // Size of each threadblock for TDM (async tensor) kernels (must be multiple of 32)
+  int tdmMaxLdsBytes;                // Max LDS (shared memory) bytes per workgroup for TDM kernels (INT_MAX = use device max)
+  int tdmPipelined;                  // Use the pipelined (depth=2) TDM tensor kernel (0=non-pipelined, 1=pipelined)
+
   // Developer features
   int gpuMaxHwQueues;                // Tracks GPU_MAX_HW_QUEUES environment variable
 
@@ -173,6 +178,9 @@ class EnvVars
     showBorders       = GetEnvVar("SHOW_BORDERS"        , 1);
     showIterations    = GetEnvVar("SHOW_ITERATIONS"     , 0);
     showPercentiles   = GetEnvVarArray("SHOW_PERCENTILES", {});
+    tdmBlockSize      = GetEnvVar("TDM_BLOCK_SIZE"      , 256);
+    tdmMaxLdsBytes    = GetEnvVar("TDM_MAX_LDS_BYTES"   , INT_MAX);
+    tdmPipelined      = GetEnvVar("TDM_PIPELINED"       , 0);
     useHipEvents      = GetEnvVar("USE_HIP_EVENTS"      , 1);
     useHsaDma         = GetEnvVar("USE_HSA_DMA"         , 0);
     useInteractive    = GetEnvVar("USE_INTERACTIVE"     , 0);
@@ -394,6 +402,9 @@ class EnvVars
     printf(" SHOW_BORDERS        - Show ASCII box-drawing characters in tables\n");
     printf(" SHOW_ITERATIONS     - Show per-iteration timing info\n");
     printf(" SHOW_PERCENTILES    - Comma-separated percentiles iteration duration\n");
+    printf(" TDM_BLOCK_SIZE      - # of threads per threadblock for TDM (async tensor) kernels (Must be multiple of 32)\n");
+    printf(" TDM_MAX_LDS_BYTES   - Max LDS bytes per workgroup for TDM kernels (defaults to device max; K/M/G suffixes accepted)\n");
+    printf(" TDM_PIPELINED       - 1=use pipelined (depth=2) TDM kernel, 0=use non-pipelined kernel\n");
     printf(" USE_HIP_EVENTS      - Use HIP events for GFX executor timing\n");
     printf(" USE_HSA_DMA         - Use hsa_amd_async_copy instead of hipMemcpy for non-targeted DMA execution\n");
     printf(" USE_INTERACTIVE     - Pause for user-input before starting transfer loop\n");
@@ -541,6 +552,14 @@ class EnvVars
           "%s per-iteration timing", showIterations ? "Showing" : "Hiding");
     Print("SHOW_PERCENTILES", showPercentiles.empty() ? 0 : 1, "%s",
           showPercentiles.empty() ? "Disabled" : GetStr(showPercentiles).c_str());
+    Print("TDM_BLOCK_SIZE", tdmBlockSize,
+          "TDM threadblock size of %d", tdmBlockSize);
+    Print("TDM_MAX_LDS_BYTES", tdmMaxLdsBytes,
+          "%s", tdmMaxLdsBytes == INT_MAX
+                  ? "Using device max LDS bytes per workgroup"
+                  : (std::string("Capping LDS to ") + std::to_string(tdmMaxLdsBytes) + " bytes per workgroup").c_str());
+    Print("TDM_PIPELINED", tdmPipelined,
+          "Using %s TDM tensor kernel", tdmPipelined ? "pipelined (depth=2)" : "non-pipelined");
     Print("USE_HIP_EVENTS", useHipEvents,
           "Using %s for GFX/DMA Executor timing", useHipEvents ? "HIP events" : "CPU wall time");
     Print("USE_HSA_DMA", useHsaDma,
@@ -749,6 +768,10 @@ class EnvVars
     cfg.nic.trafficClass           = nicTrafficClass;
     cfg.nic.roceVersion            = roceVersion;
 
+    cfg.tdm.blockSize              = tdmBlockSize;
+    cfg.tdm.maxLDSBytes            = tdmMaxLdsBytes;
+    cfg.tdm.pipelined              = tdmPipelined;
+
     return cfg;
   }
 };

src/client/Presets/Help.hpp

@@ -37,13 +37,15 @@ int HelpPreset([[maybe_unused]] EnvVars&          ev,
   printf("#                SRC 1 -> Executor -> DST 1\n");
   printf("#                SRC X                DST Y\n");
   printf("\n");
-  printf("# Five Executors are supported by TransferBench\n");
-  printf("#   Executor:        SubExecutor:\n");
-  printf("#   1) CPU           CPU thread\n");
-  printf("#   2) GPU           GPU threadblock/Compute Unit (CU)\n");
-  printf("#   3) DMA           N/A.                                 (Must have single SRC, at least one DST)\n");
-  printf("#   4) NIC           Queue Pair\n");
-  printf("#   5) Batched-DMA   Batch item                           (Must have single SRC, at least one DST)\n");
+  printf("# Seven Executors are supported by TransferBench\n");
+  printf("#   Executor:              SubExecutor:\n");
+  printf("#   1) CPU                 CPU thread\n");
+  printf("#   2) GPU                 GPU threadblock/Compute Unit (CU)\n");
+  printf("#   3) DMA                 N/A.                                 (Must have single SRC, at least one DST)\n");
+  printf("#   4) NIC                 Queue Pair\n");
+  printf("#   5) Batched-DMA         Batch item                           (Must have single SRC, at least one DST)\n");
+  printf("#   6) TDM                 GPU threadblock/Compute Unit (CU)\n");
+  printf("#   7) Async Load/Store    GPU threadblock/Compute Unit (CU)\n");
   printf("\n");
   printf("# Each single line in the configuration file defines a set of Transfers (a Test) to run in parallel\n");
   printf("\n");
@@ -71,6 +73,8 @@ int HelpPreset([[maybe_unused]] EnvVars&          ev,
   printf("#                 - B:    Batched-DMA-executor  (Indexed from 0 to # GPUs - 1)\n");
   printf("#                 - I#.#: NIC executor          (Indexed from 0 to # NICs - 1)\n");
   printf("#                 - N#.#: Nearest NIC executor  (Indexed from 0 to # GPUs - 1)\n");
+  printf("#                 - T:    GPU TDM kernel kernel (Indexed from 0 to # GPUs - 1)\n");
+  printf("#                 - L:    GPU async load/store  (Indexed from 0 to # GPUs - 1)\n");
   printf("#   dstMemL   :   Destination memory locations (Where the data is to be written to)\n");
   printf("#   bytesL    :   Number of bytes to copy (0 means use command-line specified size)\n");
   printf("#                 Must be a multiple of 4 and may be suffixed with ('K','M', or 'G')\n");
@@ -108,6 +112,12 @@ int HelpPreset([[maybe_unused]] EnvVars&          ev,
   printf("## Single DMA executed Transfer between GPUs 0 and 1\n");
   printf("1 1 (G0->D0->G1)\n");
   printf("\n");
+  printf("## Single GPU-executed Transfer between GPUs 0 and 1 using 1 CU for tensor-op path\n");
+  printf("1 1 (G0->T0->G1)\n");
+  printf("\n");
+  printf("## Single GPU-executed Transfer between GPUs 0 and 1 using 1 CU for async load/store path\n");
+  printf("1 1 (G0->L0->G1)\n");
+  printf("\n");
   printf("## Copy 1Mb from GPU0 to GPU1 with 4 CUs, and 2Mb from GPU1 to GPU0 with 8 CUs\n");
   printf("-2 (G0->G0->G1 4 1M) (G1->G1->G0 8 2M)\n");
   printf("\n");

src/client/Utilities.hpp

@@ -439,6 +439,8 @@ namespace TransferBench::Utils
     case EXE_NIC:           return "NIC";
     case EXE_NIC_NEAREST:   return "NIC";
     case EXE_GPU_BDMA:      return "BMA";
+    case EXE_GPU_ASYNC_TENSOR: return "AT"; // async tensor kernel path
+    case EXE_GPU_ASYNC_MEMOPS: return "AL"; // async load/store kernel path
     default:                return "N/A";
     }
   }