Arm64: opt-in low-power spin-wait using FEAT_WFxT (WFET)

src/coreclr/inc/clrconfigvalues.h

@@ -400,6 +400,9 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_SpinLimitProcCap, W("SpinLimitProcCap"), 0xFFF
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_SpinLimitProcFactor, W("SpinLimitProcFactor"), 0x4E20, "Hex value specifying the multiplier on NumProcs to use when calculating the maximum spin duration")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_SpinLimitConstant, W("SpinLimitConstant"), 0x0, "Hex value specifying the constant to add when calculating the maximum spin duration")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_SpinRetryCount, W("SpinRetryCount"), 0xA, "Hex value specifying the number of times the entire spin process is repeated (when applicable)")
+// Opt-in (disabled by default). On Arm64 hardware with FEAT_WFxT (WFET), issue a low-power timed wait
+// during normalized spin-waits instead of a busy YieldProcessor loop. Reduces energy usage while spinning.
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_ThreadWfetSpinWait, W("ThreadWfetSpinWait"), 0, "When set to 1 on Arm64 with FEAT_WFxT, use WFET for low-power spin-waits")
 
 ///
 /// Profiling API / ETW

src/coreclr/inc/yieldprocessornormalized.h

@@ -3,6 +3,8 @@
 
 #pragma once
 
+#include "minipal/cpufeatures.h"
+
 #ifdef FEATURE_NATIVEAOT
 FORCEINLINE void System_YieldProcessor() { PalYieldProcessor(); }
 #else
@@ -87,6 +89,24 @@ class YieldProcessorNormalizationInfo
     friend void YieldProcessorWithBackOffNormalized(const YieldProcessorNormalizationInfo &, unsigned int);
 };
 
+// When enabled at startup (Arm64 with FEAT_WFxT and the ThreadWfetSpinWait knob), a normalized spin-wait
+// issues a single low-power WFET wait for the equivalent duration instead of a busy YieldProcessor loop.
+// This trades a small amount of extra wake-up latency for substantially lower energy usage while spinning.
+// 'normalizedYields' is the number of normalized yields the caller intended to perform. It is 'unsigned int'
+// (not size_t) so that 'normalizedYields * TargetNsPerNormalizedYield' cannot overflow the uint64_t result.
+FORCEINLINE bool TryYieldProcessorWithWfet(unsigned int normalizedYields)
+{
+#if defined(HOST_ARM64) && !defined(HOST_WINDOWS)
+    if (g_minipalWfetSpinWaitEnabled)
+    {
+        minipal_wfet_wait_ns((uint64_t)normalizedYields * YieldProcessorNormalization::TargetNsPerNormalizedYield);
+        return true;
+    }
+#endif
+    (void)normalizedYields;
+    return false;
+}
+
 // See YieldProcessorNormalized() for preliminary info. Typical usage:
 //     if (!condition)
 //     {
@@ -98,6 +118,11 @@ class YieldProcessorNormalizationInfo
 //     }
 FORCEINLINE void YieldProcessorNormalized(const YieldProcessorNormalizationInfo &normalizationInfo)
 {
+    if (TryYieldProcessorWithWfet(1))
+    {
+        return;
+    }
+
     unsigned int n = normalizationInfo.yieldsPerNormalizedYield;
     _ASSERTE(n != 0);
     do
@@ -141,6 +166,11 @@ FORCEINLINE void YieldProcessorNormalized(const YieldProcessorNormalizationInfo
 {
     _ASSERTE(count != 0);
 
+    if (TryYieldProcessorWithWfet(count))
+    {
+        return;
+    }
+
     if (sizeof(size_t) <= sizeof(unsigned int))
     {
         // On platforms with a small size_t, prevent overflow on the multiply below
@@ -271,16 +301,23 @@ FORCEINLINE void YieldProcessorWithBackOffNormalized(
     static_assert(
         ((unsigned int)1 << (MaxShift + 1)) > YieldProcessorNormalization::MaxOptimalMaxNormalizedYieldsPerSpinIteration, "");
 
-    unsigned int n;
+    unsigned int normalizedYields;
     if (spinIteration <= MaxShift &&
         ((unsigned int)1 << spinIteration) < normalizationInfo.optimalMaxNormalizedYieldsPerSpinIteration)
     {
-        n = ((unsigned int)1 << spinIteration) * normalizationInfo.yieldsPerNormalizedYield;
+        normalizedYields = (unsigned int)1 << spinIteration;
     }
     else
     {
-        n = normalizationInfo.optimalMaxYieldsPerSpinIteration;
+        normalizedYields = normalizationInfo.optimalMaxNormalizedYieldsPerSpinIteration;
     }
+
+    if (TryYieldProcessorWithWfet(normalizedYields))
+    {
+        return;
+    }
+
+    unsigned int n = normalizedYields * normalizationInfo.yieldsPerNormalizedYield;
     _ASSERTE(n != 0);
     do
     {

src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs

@@ -232,6 +232,9 @@ private static class Arm64IntrinsicConstants
             public const int SveAes = (1 << 13);
             public const int SveSha3 = (1 << 14);
             public const int SveSm4 = (1 << 15);
+            // Runtime-only feature (FEAT_WFxT), not a JIT hardware intrinsic ISA. Reserved here to keep the
+            // bit layout in sync with cpufeatures.h; it is intentionally not mapped to an InstructionSet.
+            public const int Wfxt = (1 << 16);
 
             public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags)
             {

src/coreclr/vm/codeman.cpp

@@ -1784,6 +1784,17 @@ void EEJitManager::SetCpuInfo()
     {
         g_arm64_atomics_present = true;
     }
+
+    // Opt into low-power WFET-based spin-waits when the hardware supports FEAT_WFxT (+ FEAT_ECV) and the
+    // ThreadWfetSpinWait knob is set. This is consumed on the spin-wait hot path via YieldProcessorNormalized.
+    // WFET is emitted via GNU inline asm in the minipal, so it is only available on non-Windows Arm64.
+#if !defined(TARGET_WINDOWS)
+    if (((cpuFeatures & ARM64IntrinsicConstants_Wfxt) != 0) &&
+        (CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_ThreadWfetSpinWait) != 0))
+    {
+        g_minipalWfetSpinWaitEnabled = 1;
+    }
+#endif // !TARGET_WINDOWS
 #elif defined(TARGET_RISCV64)
     if (g_pConfig->EnableHWIntrinsic())
     {

src/native/minipal/cpufeatures.c

@@ -83,6 +83,12 @@
 #ifndef HWCAP2_SVESM4
 #define HWCAP2_SVESM4   (1 << 6)
 #endif
+#ifndef HWCAP2_ECV
+#define HWCAP2_ECV   (1 << 19)
+#endif
+#ifndef HWCAP2_WFXT
+#define HWCAP2_WFXT   (1UL << 31)
+#endif
 
 #endif
 
@@ -566,6 +572,11 @@ int minipal_getcpufeatures(void)
     if (hwCap2 & HWCAP2_SVESM4)
         result |= ARM64IntrinsicConstants_SveSm4;
 
+    // WFET is only used together with the self-synchronized CNTVCTSS_EL0 counter read (FEAT_ECV),
+    // so require both FEAT_WFxT and FEAT_ECV.
+    if ((hwCap2 & HWCAP2_WFXT) && (hwCap2 & HWCAP2_ECV))
+        result |= ARM64IntrinsicConstants_Wfxt;
+
 #else // !HAVE_AUXV_HWCAP_H
 
 #if HAVE_SYSCTLBYNAME
@@ -636,6 +647,20 @@ int minipal_getcpufeatures(void)
 
     if ((sysctlbyname("hw.optional.arm.FEAT_SVE_SM4", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0))
         result |= ARM64IntrinsicConstants_SveSm4;
+
+    // WFET is only used together with the self-synchronized CNTVCTSS_EL0 counter read (FEAT_ECV),
+    // so require both FEAT_WFxT and FEAT_ECV.
+    {
+        int64_t hasWfxt = 0;
+        int64_t hasEcv = 0;
+        size_t szWfxt = sizeof(hasWfxt);
+        size_t szEcv = sizeof(hasEcv);
+        if ((sysctlbyname("hw.optional.arm.FEAT_WFxT", &hasWfxt, &szWfxt, NULL, 0) == 0) && (hasWfxt != 0) &&
+            (sysctlbyname("hw.optional.arm.FEAT_ECV", &hasEcv, &szEcv, NULL, 0) == 0) && (hasEcv != 0))
+        {
+            result |= ARM64IntrinsicConstants_Wfxt;
+        }
+    }
 #endif // HAVE_SYSCTLBYNAME
 #endif // HAVE_AUXV_HWCAP_H
 #endif // HOST_UNIX
@@ -793,3 +818,57 @@ bool minipal_detect_rosetta(void)
 
     return false;
 }
+
+#if defined(HOST_ARM64) && !defined(HOST_WINDOWS)
+// Opt-in flag set by the runtime once it has verified FEAT_WFxT support and read the configuration knob.
+int g_minipalWfetSpinWaitEnabled = 0;
+
+// Instruction encodings, used via .inst so that the sources build without requiring an assembler that
+// understands the Armv8.7 mnemonics:
+//   WFET Xn               : 0xD5031000 | n
+//   MRS  Xn, CNTVCTSS_EL0 : 0xD53BE0C0 | n   (self-synchronized virtual counter, FEAT_ECV)
+//   MRS  Xn, CNTFRQ_EL0   : 0xD53BE000 | n
+//   SB                    : 0xD50330FF
+static inline uint64_t minipal_read_cntvctss_el0(void)
+{
+    uint64_t value;
+    __asm__ __volatile__(".inst 0xd53be0c0\n\tmov %0, x0" : "=r"(value) : : "x0", "memory"); // mrs x0, cntvctss_el0
+    return value;
+}
+
+static inline uint64_t minipal_read_cntfrq_el0(void)
+{
+    uint64_t value;
+    __asm__ __volatile__(".inst 0xd53be000\n\tmov %0, x0" : "=r"(value) : : "x0"); // mrs x0, cntfrq_el0
+    return value;
+}
+
+void minipal_wfet_wait_ns(uint64_t ns)
+{
+    // CNTFRQ_EL0 is a constant system register, so reading it per call is a cheap non-trapping mrs and
+    // avoids any shared cache state (and the associated data race).
+    uint64_t cntfrq = minipal_read_cntfrq_el0();
+
+    // Convert nanoseconds to counter ticks. From Armv8.6 onwards the counter runs at a fixed 1GHz
+    // (1 tick == 1 ns), which is the common case; scale by CNTFRQ_EL0 otherwise to stay correct. The
+    // scaling uses a 128-bit intermediate so the multiply cannot overflow for large 'ns'.
+    const uint64_t NsPerSecond = 1000000000ULL;
+    uint64_t ticks = (cntfrq == NsPerSecond) ? ns : (uint64_t)(((unsigned __int128)ns * cntfrq) / NsPerSecond);
+    if (ticks == 0)
+    {
+        ticks = 1;
+    }
+
+    uint64_t start = minipal_read_cntvctss_el0();
+    uint64_t target = start + ticks;
+    do
+    {
+        // WFET enters a low-power state until an event arrives or CNTVCT_EL0 reaches the target in x9.
+        // It may also wake spuriously, so re-read the counter and loop until the deadline is reached.
+        __asm__ __volatile__("mov x9, %0\n\t.inst 0xd5031009" : : "r"(target) : "x9", "memory"); // wfet x9
+        // Subtract-and-compare against the elapsed ticks so the loop is robust to counter roll-over.
+    } while ((minipal_read_cntvctss_el0() - start) < ticks);
+
+    __asm__ __volatile__(".inst 0xd50330ff" : : : "memory"); // sb - prevent speculation past the wait
+}
+#endif // HOST_ARM64 && !HOST_WINDOWS

src/native/minipal/cpufeatures.h

@@ -4,6 +4,8 @@
 #ifndef HAVE_MINIPAL_CPUFEATURES_H
 #define HAVE_MINIPAL_CPUFEATURES_H
 
+#include <stdint.h>
+
 //
 // Should match the constants defined in the compiler in HardwareIntrinsicHelpers.cs
 //
@@ -50,6 +52,9 @@
 #define ARM64IntrinsicConstants_SveAes (1 << 13)
 #define ARM64IntrinsicConstants_SveSha3 (1 << 14)
 #define ARM64IntrinsicConstants_SveSm4 (1 << 15)
+// Runtime-only feature (not a JIT hardware intrinsic ISA). Indicates FEAT_WFxT (WFET/WFIT) is
+// available together with FEAT_ECV (required for the self-synchronized CNTVCTSS_EL0 counter read).
+#define ARM64IntrinsicConstants_Wfxt (1 << 16)
 
 #include <assert.h>
 
@@ -72,6 +77,16 @@ extern "C"
 int minipal_getcpufeatures(void);
 bool minipal_detect_rosetta(void);
 
+#if defined(HOST_ARM64) && !defined(HOST_WINDOWS)
+// Non-zero when the runtime has opted into using FEAT_WFxT (WFET) for low-power spin-waits.
+// Set once during startup; read on the spin-wait hot path.
+extern int g_minipalWfetSpinWaitEnabled;
+
+// Low-power wait for approximately 'ns' nanoseconds using the WFET instruction. The caller must
+// ensure ARM64IntrinsicConstants_Wfxt is present (FEAT_WFxT + FEAT_ECV).
+void minipal_wfet_wait_ns(uint64_t ns);
+#endif // HOST_ARM64 && !HOST_WINDOWS
+
 #ifdef __cplusplus
 }
 #endif // __cplusplus