Interpreter block-count PGO for WebAssembly CoreCLR

src/coreclr/clrfeatures.cmake

@@ -13,6 +13,10 @@ if (FEATURE_DYNAMIC_CODE_COMPILED)
   set(FEATURE_PGO 1)
 endif()
 
+if (NOT DEFINED FEATURE_PGO AND CLR_CMAKE_TARGET_ARCH_WASM)
+  set(FEATURE_PGO 1)
+endif()
+
 # On desktop, if dynamic code compiled is false, we still enable static linking so we don't have to add platform manifest entries
 # for interpreter library, which is required for the packs build
 if (CLR_CMAKE_TARGET_ARCH_WASM OR CLR_CMAKE_TARGET_APPLE_MOBILE OR NOT FEATURE_DYNAMIC_CODE_COMPILED)

src/coreclr/inc/clrconfigvalues.h

@@ -515,6 +515,7 @@ CONFIG_DWORD_INFO(INTERNAL_OSR_HighId, W("OSR_HighId"), 10000000, "High end of e
 RETAIL_CONFIG_STRING_INFO(INTERNAL_PGODataPath, W("PGODataPath"), "Read/Write PGO data from/to the indicated file.")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_ReadPGOData, W("ReadPGOData"), 0, "Read PGO data")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_WritePGOData, W("WritePGOData"), 0, "Write PGO data")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_InterpPGO, W("InterpPGO"), 0, "Instrument interpreted methods with block counters and make the profile available for offline PGO (e.g. dotnet-pgo).")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_TieredPGO, W("TieredPGO"), 1, "Instrument Tier0 code and make counts available to Tier1")
 
 // TieredPGO_InstrumentOnlyHotCode values:

src/coreclr/interpreter/compiler.cpp

@@ -51,6 +51,7 @@ bool InterpCompiler::s_samplingProfilerEnabled = false;
 bool InterpCompiler::s_browserProfilerEnabled = false;
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
+bool InterpCompiler::s_interpPgoEnabled = false;
 
 #if MEASURE_MEM_ALLOC
 #include <minipal/mutex.h>
@@ -2246,6 +2247,10 @@ InterpCompiler::InterpCompiler(COMP_HANDLE compHnd,
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
 
+    m_emitInterpPGO = s_interpPgoEnabled
+        && (InterpConfig.InterpPGOMethods().isEmpty()
+            || InterpConfig.InterpPGOMethods().contains(compHnd, m_methodHnd, m_classHnd, &m_methodInfo->args));
+
 #ifdef DEBUG
     m_methodName = ::PrintMethodName(compHnd, m_classHnd, m_methodHnd, &m_methodInfo->args,
                             /* includeAssembly */ false,
@@ -2348,6 +2353,9 @@ bool InterpCompiler::CompileMethod()
     }
 #endif
 
+    if (m_emitInterpPGO)
+        InstrumentBlockCounts();
+
     AllocOffsets();
     PatchInitLocals(m_methodInfo);
 
@@ -3002,6 +3010,9 @@ void InterpCompiler::EmitBranch(InterpOpcode opcode, int32_t ilOffset)
     if (pTargetBB == NULL)
         BADCODE("code jumps to invalid offset");
 
+    if (ilOffset < 0)
+        pTargetBB->isBackwardBranchTarget = true;
+
     EmitBranchToBB(opcode, pTargetBB);
 }
 
@@ -3164,6 +3175,11 @@ void InterpCompiler::EmitLeave(int32_t ilOffset, int32_t target)
     }
     InterpBasicBlock *pTargetBB = m_ppOffsetToBB[target];
 
+    // Mark a backward leave target as a loop head here, while pTargetBB is still the real IL block: below it
+    // may be redirected to a finally call island, which shares its IL offset with another block.
+    if (target < ilOffset && pTargetBB != NULL)
+        pTargetBB->isBackwardBranchTarget = true;
+
     m_pStackPointer = m_pStackBase;
 
     // The leave will jump:
@@ -8668,6 +8684,64 @@ void InterpCompiler::CreateSynchronizedRetValVar()
     INTERP_DUMP("Created ret val var V%d\n", m_synchronizedOrAsyncRetValVarIndex);
 }
 
+// Instrument the method entry and loop heads with block-count PGO probes. The counters are allocated by
+// allocPgoInstrumentationBySchema (native PgoManager memory), so they persist independently of
+// EventPipe session lifetime; the accumulated profile is flushed to the trace as
+// JitInstrumentationDataVerbose events, which dotnet-pgo consumes to build an .mibc.
+void InterpCompiler::InstrumentBlockCounts()
+{
+    // Probe the same points as the sampling profiler - method entry and loop heads (targets of backward
+    // branches) - but with an exact counter rather than a sample. This yields exact method invocation and
+    // loop trip counts; acyclic branch structure is deliberately not profiled here, since mapping the
+    // interpreter's blocks onto the JIT's is approximate at best (#130517).
+    // Clones (funclet / leave-chain islands) and blocks removed by optimization are skipped, so every schema
+    // entry carries a unique IL offset, matching what getPgoInstrumentationResults and dotnet-pgo expect.
+    TArray<InterpBasicBlock*, MemPoolAllocator> blocks(GetMemPoolAllocator(IMK_DataItem));
+    for (InterpBasicBlock *bb = m_pEntryBB; bb != NULL; bb = bb->pNextBB)
+    {
+        if (bb->ilOffset < 0 || bb->ilOffset >= m_ILCodeSizeFromILHeader || m_ppOffsetToBB[bb->ilOffset] != bb)
+            continue;
+        if (bb->ilOffset == 0 || bb->isBackwardBranchTarget)
+            blocks.Add(bb);
+    }
+
+    int32_t numBlocks = blocks.GetSize();
+    if (numBlocks == 0)
+        return;
+
+    // One 4-byte block counter per block.
+    TArray<ICorJitInfo::PgoInstrumentationSchema, MemPoolAllocator> schema(GetMemPoolAllocator(IMK_DataItem));
+    for (int32_t i = 0; i < numBlocks; i++)
+    {
+        ICorJitInfo::PgoInstrumentationSchema schemaElem;
+        schemaElem.Offset = 0;
+        schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::BasicBlockIntCount;
+        schemaElem.ILOffset = blocks.Get(i)->ilOffset;
+        schemaElem.Count = 1;
+        schemaElem.Other = 0;
+        schema.Add(schemaElem);
+    }
+
+    ICorJitInfo::PgoInstrumentationSchema *pSchema = schema.GetUnderlyingArray();
+    uint8_t *pInstrumentationData = NULL;
+    HRESULT hr = m_compHnd->allocPgoInstrumentationBySchema(m_methodHnd, pSchema, (uint32_t)numBlocks, &pInstrumentationData);
+    if (FAILED(hr) || pInstrumentationData == NULL)
+    {
+        INTERP_DUMP("InstrumentBlockCounts: allocPgoInstrumentationBySchema failed (hr=0x%08x)\n", hr);
+        return;
+    }
+
+    // Insert an INTOP_PGO_COUNT probe at the start of each block, pointing at its counter.
+    for (int32_t i = 0; i < numBlocks; i++)
+    {
+        uint32_t *pCounter = (uint32_t*)(pInstrumentationData + pSchema[i].Offset);
+        InterpInst *ins = InsertInsBB(blocks.Get(i), NULL, INTOP_PGO_COUNT);
+        // Probe is a pure counter increment with no IL mapping; keep it out of the debug maps.
+        ins->ilOffset = -1;
+        ins->data[0] = GetDataItemIndex((void*)pCounter);
+    }
+}
+
 void InterpCompiler::GenerateCode(CORINFO_METHOD_INFO* methodInfo)
 {
     bool readonly = false;
@@ -10234,6 +10308,10 @@ void InterpCompiler::GenerateCode(CORINFO_METHOD_INFO* methodInfo)
                     uint32_t target = (uint32_t)(nextIp - m_pILCode + offset);
                     InterpBasicBlock *targetBB = m_ppOffsetToBB[target];
                     assert(targetBB);
+                    // Offsets are relative to the instruction after the switch, so a negative one is a
+                    // backward branch, i.e. a loop head.
+                    if (offset < 0)
+                        targetBB->isBackwardBranchTarget = true;
                     targetOffsets[i] = target;
                     targetBBTable[i] = targetBB;
                     m_ip += 4;

src/coreclr/interpreter/compiler.h

@@ -355,6 +355,9 @@ struct InterpBasicBlock
     // Is a leave chain island basic block
     bool isLeaveChainIsland;
 
+    // True if this block is the target of a backward branch, i.e. a loop head.
+    bool isBackwardBranchTarget;
+
     // If this basic block is a catch or filter funclet entry, this is the index of the variable
     // that holds the exception object.
     int clauseVarIndex;
@@ -387,6 +390,7 @@ struct InterpBasicBlock
         isFilterOrCatchFuncletEntry = false;
         isFinallyCallIsland = false;
         isLeaveChainIsland = false;
+        isBackwardBranchTarget = false;
         clauseVarIndex = -1;
         overlappingEHClauseCount = 0;
         enclosingTryBlockCount = -1;
@@ -641,6 +645,8 @@ class InterpCompiler
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
 
+    bool m_emitInterpPGO;
+
     void DeclarePointerIsClass(CORINFO_CLASS_HANDLE clsHnd)
     {
 #ifdef DEBUG
@@ -767,6 +773,8 @@ class InterpCompiler
     void CreateSynchronizedRetValVar();
 
     void GenerateCode(CORINFO_METHOD_INFO* methodInfo);
+
+    void InstrumentBlockCounts();
     InterpBasicBlock* GenerateCodeForLeaveChainIslands(InterpBasicBlock *pNewBB, InterpBasicBlock *pPrevBB);
     void PatchInitLocals(CORINFO_METHOD_INFO* methodInfo);
 
@@ -1150,6 +1158,7 @@ class InterpCompiler
     static bool s_browserProfilerEnabled;
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
+    static bool s_interpPgoEnabled;
 
 #if MEASURE_MEM_ALLOC
     // Memory statistics for profiling.

src/coreclr/interpreter/eeinterp.cpp

@@ -41,6 +41,11 @@ extern "C" INTERP_API void jitStartup(ICorJitHost* jitHost)
 #endif // PERFTRACING_DISABLE_THREADS
     }
 
+    if (InterpConfig.InterpPGO() != 0)
+    {
+        InterpCompiler::s_interpPgoEnabled = true;
+    }
+
     g_interpInitialized = true;
 }
 /*****************************************************************************/

src/coreclr/interpreter/inc/intops.def

@@ -90,6 +90,8 @@ OPDEF(INTOP_PROF_ENTER, "prof.enter", 2, 0, 0, InterpOpMethodHandle)
 OPDEF(INTOP_PROF_LEAVE, "prof.leave", 1, 0, 0, InterpOpNoArgs)
 #endif
 
+OPDEF(INTOP_PGO_COUNT, "pgo.count", 2, 0, 0, InterpOpLdPtr)
+
 OPDEF(INTOP_BR, "br", 2, 0, 0, InterpOpBranch)
 
 OPDEF(INTOP_BRFALSE_I4, "brfalse.i4", 3, 0, 1, InterpOpBranch)

src/coreclr/interpreter/interpconfigvalues.h

@@ -37,6 +37,8 @@ RELEASE_CONFIG_INTEGER(InterpMode, "InterpMode", 0); // Interpreter mode, one of
 
 RELEASE_CONFIG_INTEGER(DisplayMemStats, "JitMemStats", 0); // Display interpreter memory usage statistics (0=off, 1=summary, 2=detailed per-method)
 RELEASE_CONFIG_METHODSET(WasmPerformanceInstrumentation, "WasmPerformanceInstrumentation") // Method filter for WASM performance instrumentation profiler. Uses standard MethodSet pattern format.
+RELEASE_CONFIG_INTEGER(InterpPGO, "InterpPGO", 0); // Instrument interpreted methods with block counters and make the profile available for offline PGO (e.g. dotnet-pgo).
+RELEASE_CONFIG_METHODSET(InterpPGOMethods, "InterpPGOMethods") // Optional method filter scoping InterpPGO instrumentation (empty = all methods). Uses standard MethodSet pattern format.
 
 #undef CONFIG_STRING
 #undef RELEASE_CONFIG_STRING

src/coreclr/nativeaot/Runtime/eventpipe/ep-rt-aot.h

@@ -594,6 +594,14 @@ ep_rt_notify_profiler_provider_created (EventPipeProvider *provider)
     // Following mono's path of no-op
 }
 
+static
+inline
+void
+ep_rt_session_stopping (void)
+{
+    // Following mono's path of no-op
+}
+
 /*
  * Arrays.
  */

src/coreclr/vm/eventing/eventpipe/ep-rt-coreclr.cpp

@@ -6,9 +6,14 @@
 #ifdef ENABLE_PERFTRACING
 #include <eventpipe/ep-types.h>
 #include <eventpipe/ep.h>
+#include <eventpipe/ep-event.h>
+#include <eventpipe/ep-session.h>
 #include <eventpipe/ep-stack-contents.h>
 #include <eventpipe/ep-rt.h>
 #include "threadsuspend.h"
+#ifdef FEATURE_PGO
+#include "pgo.h"
+#endif
 
 ep_rt_lock_handle_t _ep_rt_coreclr_config_lock_handle;
 CrstStatic _ep_rt_coreclr_config_lock;
@@ -169,4 +174,27 @@ ep_rt_coreclr_sample_profiler_write_sampling_event_for_threads (
 	return;
 }
 
+void
+ep_rt_coreclr_session_stopping (void)
+{
+	STATIC_CONTRACT_NOTHROW;
+#if defined(FEATURE_PGO) && defined(PERFTRACING_DISABLE_THREADS)
+	// The EventPipe session_stopping helper has bound this thread to the stopping session as its rundown
+	// session, so ep_event_is_enabled_for_current_thread tests that session's mask and the events emitted by
+	// the flush route only to it (dotnet-pgo drops a method once data arrives after its final chunk, so a
+	// single destination is required).
+	extern EventPipeEvent *EventPipeEventJitInstrumentationDataVerbose;
+	if (EventPipeEventJitInstrumentationDataVerbose != NULL &&
+		ep_event_is_enabled_for_current_thread (EventPipeEventJitInstrumentationDataVerbose))
+	{
+		EX_TRY
+		{
+			PgoManager::LogInstrumentationData ();
+		}
+		EX_CATCH { }
+		EX_END_CATCH
+	}
+#endif // FEATURE_PGO && PERFTRACING_DISABLE_THREADS
+}
+
 #endif /* ENABLE_PERFTRACING */

src/coreclr/vm/eventing/eventpipe/ep-rt-coreclr.h

@@ -659,6 +659,16 @@ ep_rt_notify_profiler_provider_created (EventPipeProvider *provider)
 #endif // !DACCESS_COMPILE && PROFILING_SUPPORTED
 }
 
+static
+inline
+void
+ep_rt_session_stopping (void)
+{
+    STATIC_CONTRACT_NOTHROW;
+    extern void ep_rt_coreclr_session_stopping (void);
+    ep_rt_coreclr_session_stopping ();
+}
+
 /*
  * Arrays.
  */
@@ -1093,8 +1103,13 @@ ep_rt_queue_job (
 	void *params)
 {
 #ifdef HOST_BROWSER
-	// In single-threaded mode the job runs on the browser event loop
-	SystemJS_DiagnosticServerQueueJob ((ep_rt_job_cb_t)job_func, params);
+	// In single-threaded mode the job runs on the browser event loop. Run the callback inline the
+	// first time so the diagnostic server makes progress synchronously (e.g. it can connect and
+	// resume during startup suspension) and only defer a re-schedule if it isn't done yet. Mirrors
+	// the Mono ep_rt_queue_job in ep-rt-mono.h.
+	ep_rt_job_cb_t cb = (ep_rt_job_cb_t)job_func;
+	if (!cb (params))
+		SystemJS_DiagnosticServerQueueJob (cb, params);
 	return true;
 #else
 	EP_UNREACHABLE ("Not implemented on this platform");

src/coreclr/vm/interpexec.cpp

@@ -2088,6 +2088,12 @@ void InterpExecMethod(InterpreterFrame *pInterpreterFrame, InterpMethodContextFr
                     INTOP_NEXT;
 #endif // TARGET_BROWSER && PERFTRACING_DISABLE_THREADS
 
+                INTOP_CASE(INTOP_PGO_COUNT)
+                    // Interlocked so concurrent executions of an instrumented method don't lose counts.
+                    InterlockedIncrement((LONG*)pMethod->pDataItems[ip[1]]);
+                    ip += 2;
+                    INTOP_NEXT;
+
                 INTOP_CASE(INTOP_BR)
                     ip += ip[1];
                     INTOP_NEXT;

src/coreclr/vm/jitinterface.cpp

@@ -13062,7 +13062,15 @@ CORJIT_FLAGS* CEECodeGenInfo::getJitFlagsInternal()
 }
 
 /*********************************************************************/
-HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
+#ifdef FEATURE_PGO
+static bool InterpreterPgoInstrumentationEnabled()
+{
+    static ConfigDWORD s_interpPgo;
+    return s_interpPgo.val(CLRConfig::INTERNAL_InterpPGO) != 0;
+}
+#endif // FEATURE_PGO
+
+HRESULT CEECodeGenInfo::allocPgoInstrumentationBySchema(
             CORINFO_METHOD_HANDLE ftnHnd, /* IN */
             PgoInstrumentationSchema* pSchema, /* IN/OUT */
             uint32_t countSchemaItems, /* IN */
@@ -13081,9 +13089,10 @@ HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
 
 #ifdef FEATURE_PGO
 
-    // Only try instrumenting tiering-eligible methods
+    // Only try instrumenting tiering-eligible methods, unless interpreter PGO is enabled, in
+    // which case we instrument every method for offline profile collection.
     MethodDesc* pMD = (MethodDesc*)ftnHnd;
-    if (pMD->IsEligibleForTieredCompilation())
+    if (pMD->IsEligibleForTieredCompilation() || InterpreterPgoInstrumentationEnabled())
     {
         hr = PgoManager::allocPgoInstrumentationBySchema(pMD, m_ILHeader, pSchema, countSchemaItems, pInstrumentationData);
     }
@@ -13092,7 +13101,7 @@ HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
         hr = E_NOTIMPL;
     }
 #else
-    _ASSERTE(!"allocMethodBlockCounts not implemented on CEEJitInfo!");
+    _ASSERTE(!"allocMethodBlockCounts not implemented on CEECodeGenInfo!");
     hr = E_NOTIMPL;
 #endif // !FEATURE_PGO
 
@@ -13101,9 +13110,9 @@ HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
     return hr;
 }
 
-// Consider implementing getBBProfileData on CEEJitInfo.  This will allow us
+// Consider implementing getBBProfileData on CEECodeGenInfo.  This will allow us
 // to use profile info in codegen for non zapped images.
-HRESULT CEEJitInfo::getPgoInstrumentationResults(
+HRESULT CEECodeGenInfo::getPgoInstrumentationResults(
             CORINFO_METHOD_HANDLE      ftnHnd,
             PgoInstrumentationSchema **pSchema,                    // pointer to the schema table which describes the instrumentation results (pointer will not remain valid after jit completes)
             uint32_t *                 pCountSchemaItems,          // pointer to the count schema items
@@ -13164,7 +13173,7 @@ HRESULT CEEJitInfo::getPgoInstrumentationResults(
     *pPgoSource = pDataCur->m_pgoSource;
     hr = pDataCur->m_hr;
 #else
-    _ASSERTE(!"getPgoInstrumentationResults not implemented on CEEJitInfo!");
+    _ASSERTE(!"getPgoInstrumentationResults not implemented on CEECodeGenInfo!");
     hr = E_NOTIMPL;
 #endif