Add NativeRuntimeEventSource metadata to NativeAot CoreLib (#106641)

* Add NativeRuntimeEventSource metadata to NativeAot CoreLib This is necessary to allow consumers to parse the payload. Delete unnecessary runtimeflavor argument of genRuntimeEventSources.py script. Fixes #105556 * Exclude unused keywords and events
dotnet · Aug 20, 2024 · 094649e · 094649e
1 parent 9876ed1
commit 094649e
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 52 deletions.
diff --git a/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj
@@ -297,11 +297,6 @@
     <ProjectReference Include="$(LibrariesProjectRoot)\System.Private.CoreLib\gen\System.Private.CoreLib.Generators.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
   </ItemGroup>
 
-  <ItemGroup>
-    <!-- This is the T4 template service and is added by VS anytime you modify a T4 template. Required for .tt files. -->
-    <Service Include="{508349b6-6b84-4df5-91f0-309beebad82d}" />
-  </ItemGroup>
-
   <Import Project="CreateRuntimeRootILLinkDescriptorFile.targets" />
 
   <Target Name="CreateRuntimeRootIlLinkDescFile" BeforeTargets="CoreCompile" DependsOnTargets="_CreateILLinkRuntimeRootDescriptorFile" />
@@ -312,15 +307,15 @@
   <ItemGroup>
     <EventingGenerationScript Include="$(CoreClrProjectRoot)scripts/genRuntimeEventSources.py" />
     <EventManifestFile Include="$(CoreClrProjectRoot)vm/ClrEtwAll.man" />
-    <EventingSourceFile Include="$(IntermediateOutputPath)..\..\..\Eventing\$(TargetArchitecture)\$(Configuration)\NativeRuntimeEventSource.CoreCLR.cs" Condition="'$(FeaturePerfTracing)' == 'true' ">
-      <Link>src\System\Diagnostics\Eventing\Generated\NativeRuntimeEventSource.CoreCLR.cs</Link>
+    <EventingSourceFile Include="$(IntermediateOutputPath)NativeRuntimeEventSource.Generated.cs" Condition="'$(FeaturePerfTracing)' == 'true' ">
+      <Link>src\System\Diagnostics\Eventing\NativeRuntimeEventSource.Generated.cs</Link>
     </EventingSourceFile>
     <Compile Include="@(EventingSourceFile)" />
   </ItemGroup>
 
   <Target Name="GenerateEventingFiles" Inputs="@(EventingGenerationScript);@(EventManifestFile)" Outputs="@(EventingSourceFile)" DependsOnTargets="FindPython" BeforeTargets="BeforeCompile">
 
-    <Error Condition="'$(PYTHON)' == ''" Text="Unable to locate Python. NativeRuntimeEventSource.CoreCLR.cs cannot be generated without Python installed on the machine. Either install Python in your path or point to it with the PYTHON environment variable." />
+    <Error Condition="'$(PYTHON)' == ''" Text="Unable to locate Python. NativeRuntimeEventSource.Generated.cs cannot be generated without Python installed on the machine. Either install Python in your path or point to it with the PYTHON environment variable." />
     <PropertyGroup>
       <_PythonWarningParameter>-Wall</_PythonWarningParameter>
       <_PythonWarningParameter Condition="'$(MSBuildTreatWarningsAsErrors)' == 'true'">$(_PythonWarningParameter) -Werror</_PythonWarningParameter>

diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj
@@ -571,8 +571,35 @@
     <ProjectReference Include="$(LibrariesProjectRoot)\System.Private.CoreLib\gen\System.Private.CoreLib.Generators.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
   </ItemGroup>
 
+  <!-- Setup eventing file generation -->
   <ItemGroup>
-    <!-- This is the T4 template service and is added by VS anytime you modify a T4 template. Required for .tt files. -->
-    <Service Include="{508349b6-6b84-4df5-91f0-309beebad82d}" />
+    <EventingGenerationScript Include="$(CoreClrProjectRoot)scripts/genRuntimeEventSources.py" />
+    <EventManifestFile Include="$(CoreClrProjectRoot)vm/ClrEtwAll.man" />
+    <EventingInclusionList Include="$(CoreClrProjectRoot)nativeaot/Runtime/eventpipe/gen-eventing-event-inc.lst" />
+    <EventingSourceFile Include="$(IntermediateOutputPath)NativeRuntimeEventSource.Generated.cs" Condition="'$(FeaturePerfTracing)' == 'true' ">
+      <Link>src\System\Diagnostics\Eventing\NativeRuntimeEventSource.Generated.cs</Link>
+    </EventingSourceFile>
+    <Compile Include="@(EventingSourceFile)" />
   </ItemGroup>
+
+  <Target Name="GenerateEventingFiles"
+          Inputs="@(EventingGenerationScript);@(EventManifestFile);@(EventingInclusionList)"
+          Outputs="@(EventingSourceFile)"
+          DependsOnTargets="FindPython"
+          BeforeTargets="BeforeCompile">
+
+    <Error Condition="'$(PYTHON)' == ''" Text="Unable to locate Python. NativeRuntimeEventSource.Generared.cs cannot be generated without Python installed on the machine. Either install Python in your path or point to it with the PYTHON environment variable." />
+    <PropertyGroup>
+      <_PythonWarningParameter>-Wall</_PythonWarningParameter>
+      <_PythonWarningParameter Condition="'$(MSBuildTreatWarningsAsErrors)' == 'true'">$(_PythonWarningParameter) -Werror</_PythonWarningParameter>
+      <_EventingSourceFileDirectory>%(EventingSourceFile.RootDir)%(EventingSourceFile.Directory)</_EventingSourceFileDirectory>
+      <_EventingSourceFileDirectory Condition="HasTrailingSlash('$(_EventingSourceFileDirectory)')">$(_EventingSourceFileDirectory.TrimEnd('\'))</_EventingSourceFileDirectory>
+    </PropertyGroup>
+
+    <Exec Command="&quot;$(PYTHON)&quot; -B $(_PythonWarningParameter) &quot;@(EventingGenerationScript)&quot; --man &quot;@(EventManifestFile)&quot; --intermediate &quot;$(_EventingSourceFileDirectory)&quot; --inc &quot;@(EventingInclusionList)&quot;" />
+
+    <ItemGroup>
+      <FileWrites Include="@(EventingSourceFile)" />
+    </ItemGroup>
+  </Target>
 </Project>
diff --git a/src/coreclr/scripts/genRuntimeEventSources.py b/src/coreclr/scripts/genRuntimeEventSources.py
@@ -5,8 +5,7 @@
 
 import os
 import xml.dom.minidom as DOM
-from utilities import open_for_update
-from genEventing import RuntimeFlavor
+from utilities import open_for_update, parseInclusionList
 import argparse
 import sys
 
@@ -24,12 +23,8 @@
 ########################################################################
 # START CONFIGURATION
 ########################################################################
-coreCLRManifestsToGenerate = {
-    "Microsoft-Windows-DotNETRuntime" : "NativeRuntimeEventSource.CoreCLR.cs"
-}
-
-monoManifestsToGenerate = {
-    "Microsoft-Windows-DotNETRuntime" : "NativeRuntimeEventSource.Mono.cs"
+manifestsToGenerate = {
+    "Microsoft-Windows-DotNETRuntime" : "NativeRuntimeEventSource.Generated.cs"
 }
 
 providerNameToClassNameMap = {
@@ -77,11 +72,22 @@ def writeOutput(outputFile, str):
 def getCSharpTypeFromManifestType(manifestType):
     return manifestTypeToCSharpTypeMap[manifestType]
 
-def getManifestsToGenerate(runtimeFlavor):
-    if runtimeFlavor.coreclr:
-        return coreCLRManifestsToGenerate
-    elif runtimeFlavor.mono:
-        return monoManifestsToGenerate
+def getManifestsToGenerate():
+    return manifestsToGenerate
+
+def includeEvent(inclusionList, providerName, eventName):
+    if len(inclusionList) == 0:
+        return True
+    if providerName in inclusionList and eventName in inclusionList[providerName]:
+        return True
+    elif providerName in inclusionList and "*" in inclusionList[providerName]:
+        return True
+    elif "*" in inclusionList and eventName in inclusionList["*"]:
+        return True
+    elif "*" in inclusionList and "*" in inclusionList["*"]:
+        return True
+    else:
+        return False
 
 def generateEvent(eventNode, providerNode, outputFile, stringTable):
 
@@ -174,7 +180,9 @@ def generateEvent(eventNode, providerNode, outputFile, stringTable):
     writeOutput(outputFile, "}\n\n")
 
 
-def generateEvents(providerNode, outputFile, stringTable):
+def generateEvents(providerNode, outputFile, stringTable, inclusion_list):
+
+    providerName = providerNode.getAttribute("name")
 
     # Get the events element.
     for node in providerNode.getElementsByTagName("events"):
@@ -188,13 +196,21 @@ def generateEvents(providerNode, outputFile, stringTable):
     # key = eventID, value = version
     eventList = dict()
     for eventNode in eventNodes:
+        eventName    = eventNode.getAttribute('symbol')
+        if not includeEvent(inclusion_list, providerName, eventName):
+            continue
+
         eventID = eventNode.getAttribute("value")
         eventVersion = eventNode.getAttribute("version")
         eventList[eventID] = eventVersion
 
     # Iterate over each event node and process it.
     # Only emit events for the latest version of the event, otherwise EventSource initialization will fail.
     for eventNode in eventNodes:
+        eventName    = eventNode.getAttribute('symbol')
+        if not includeEvent(inclusion_list, providerName, eventName):
+            continue
+
         eventID = eventNode.getAttribute("value")
         eventVersion = eventNode.getAttribute("version")
         if eventID in eventList and eventList[eventID] == eventVersion:
@@ -305,7 +321,29 @@ def generateEnumTypeMap(providerNode):
 
     return typeMap
 
-def generateKeywordsClass(providerNode, outputFile):
+def generateKeywordsClass(providerNode, outputFile, inclusion_list):
+
+    providerName = providerNode.getAttribute("name")
+
+    # Get the events element.
+    for node in providerNode.getElementsByTagName("events"):
+        eventsNode = node
+        break
+
+    # Get the list of event nodes.
+    eventNodes = eventsNode.getElementsByTagName("event")
+
+    # Build the list of used keywords
+    keywordSet = set()
+    for eventNode in eventNodes:
+        eventName    = eventNode.getAttribute('symbol')
+        if not includeEvent(inclusion_list, providerName, eventName):
+            continue
+
+        # Not all events have keywords specified, and some have multiple keywords specified.
+        keywords = eventNode.getAttribute("keywords")
+        if keywords:
+            keywordSet = keywordSet.union(keywords.split())
 
     # Find the keywords element.
     for node in providerNode.getElementsByTagName("keywords"):
@@ -317,7 +355,11 @@ def generateKeywordsClass(providerNode, outputFile):
     increaseTabLevel()
 
     for keywordNode in keywordsNode.getElementsByTagName("keyword"):
-        writeOutput(outputFile, "public const EventKeywords " + keywordNode.getAttribute("name") + " = (EventKeywords)" + keywordNode.getAttribute("mask") + ";\n")
+        keywordName = keywordNode.getAttribute("name")
+        if keywordName not in keywordSet:
+            continue;
+
+        writeOutput(outputFile, "public const EventKeywords " + keywordName + " = (EventKeywords)" + keywordNode.getAttribute("mask") + ";\n")
 
     decreaseTabLevel()
     writeOutput(outputFile, "}\n\n")
@@ -338,7 +380,7 @@ def loadStringTable(manifest):
 
     return stringTable
 
-def generateEventSources(manifestFullPath, intermediatesDirFullPath, runtimeFlavor):
+def generateEventSources(manifestFullPath, intermediatesDirFullPath, inclusion_list):
 
     # Open the manifest for reading.
     manifest = DOM.parse(manifestFullPath)
@@ -347,7 +389,7 @@ def generateEventSources(manifestFullPath, intermediatesDirFullPath, runtimeFlav
     stringTable = loadStringTable(manifest)
 
     # Iterate over each provider that we want to generate an EventSource for.
-    for providerName, outputFileName in getManifestsToGenerate(runtimeFlavor).items():
+    for providerName, outputFileName in getManifestsToGenerate().items():
         for node in manifest.getElementsByTagName("provider"):
             if node.getAttribute("name") == providerName:
                 providerNode = node
@@ -379,7 +421,7 @@ def generateEventSources(manifestFullPath, intermediatesDirFullPath, runtimeFlav
             increaseTabLevel()
 
             # Write the keywords class.
-            generateKeywordsClass(providerNode, outputFile)
+            generateKeywordsClass(providerNode, outputFile, inclusion_list)
 
             #### Disable enums until they are needed ####
             # Generate the enum type map.
@@ -394,7 +436,7 @@ def generateEventSources(manifestFullPath, intermediatesDirFullPath, runtimeFlav
             #### Disable enums until they are needed ####
 
             # Generate events.
-            generateEvents(providerNode, outputFile, stringTable)
+            generateEvents(providerNode, outputFile, stringTable, inclusion_list)
 
             # Write the class footer.
             decreaseTabLevel()
@@ -413,16 +455,16 @@ def main(argv):
                           help='full path to manifest containing the description of events')
     required.add_argument('--intermediate', type=str, required=True,
                           help='full path to eventprovider intermediate directory')
-    required.add_argument('--runtimeflavor', type=str,default="CoreCLR",
-                          help='runtime flavor')
+    required.add_argument('--inc',  type=str,default="",
+                          help='full path to inclusion list')
     args, unknown = parser.parse_known_args(argv)
     if unknown:
         print('Unknown argument(s): ', ', '.join(unknown))
         return 1
 
     manifestFullPath = args.man
     intermediatesDirFullPath = args.intermediate
-    runtimeFlavor = RuntimeFlavor(args.runtimeflavor)
+    inclusion_filename = args.inc
 
     # Ensure the intermediates directory exists.
     try:
@@ -431,8 +473,10 @@ def main(argv):
         if not os.path.isdir(intermediatesDirFullPath):
             raise
 
+    inclusion_list = parseInclusionList(inclusion_filename)
+
     # Generate event sources.
-    generateEventSources(manifestFullPath, intermediatesDirFullPath, runtimeFlavor)
+    generateEventSources(manifestFullPath, intermediatesDirFullPath, inclusion_list)
     return 0
 
 if __name__ == '__main__':

diff --git a/...src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.Internal.cs b/...src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.Internal.cs
@@ -10,17 +10,6 @@ namespace System.Diagnostics.Tracing
     // It contains the runtime specific interop to native event sinks.
     internal sealed partial class NativeRuntimeEventSource : EventSource
     {
-#if NATIVEAOT
-        // We don't have these keywords defined from the genRuntimeEventSources.py, so we need to manually define them here.
-        public static partial class Keywords
-        {
-            public const EventKeywords ContentionKeyword = (EventKeywords)0x4000;
-            public const EventKeywords ThreadingKeyword = (EventKeywords)0x10000;
-            public const EventKeywords ThreadTransferKeyword = (EventKeywords)0x80000000;
-            public const EventKeywords WaitHandleKeyword = (EventKeywords)0x40000000000;
-        }
-#endif
-
         [NonEvent]
         [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogContentionLockCreated")]
         private static partial void LogContentionLockCreated(nint LockID, nint AssociatedObjectID, ushort ClrInstanceID);

diff --git a/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj
@@ -315,8 +315,8 @@
   <ItemGroup>
     <EventingGenerationScript Include="$(CoreClrProjectRoot)scripts/genRuntimeEventSources.py" />
     <EventManifestFile Include="$(CoreClrProjectRoot)vm/ClrEtwAll.man" />
-    <EventingSourceFile Include="$(IntermediateOutputPath)..\..\..\Eventing\$(TargetArchitecture)\$(Configuration)\NativeRuntimeEventSource.Mono.cs" Condition="'$(FeaturePerfTracing)' == 'true' ">
-      <Link>src\System\Diagnostics\Eventing\Generated\NativeRuntimeEventSource.Mono.cs</Link>
+    <EventingSourceFile Include="$(IntermediateOutputPath)NativeRuntimeEventSource.Generated.cs" Condition="'$(FeaturePerfTracing)' == 'true' ">
+      <Link>src\System\Diagnostics\Eventing\NativeRuntimeEventSource.Generated.cs</Link>
     </EventingSourceFile>
     <Compile Include="@(EventingSourceFile)" />
   </ItemGroup>
@@ -327,15 +327,15 @@
           DependsOnTargets="FindPython"
           BeforeTargets="BeforeCompile">
 
-    <Error Condition="'$(PYTHON)' == ''" Text="Unable to locate Python. NativeRuntimeEventSource.Mono.cs cannot be generated without Python installed on the machine. Either install Python in your path or point to it with the PYTHON environment variable." />
+    <Error Condition="'$(PYTHON)' == ''" Text="Unable to locate Python. NativeRuntimeEventSource.Generated.cs cannot be generated without Python installed on the machine. Either install Python in your path or point to it with the PYTHON environment variable." />
     <PropertyGroup>
       <_PythonWarningParameter>-Wall</_PythonWarningParameter>
       <_PythonWarningParameter Condition="'$(MSBuildTreatWarningsAsErrors)' == 'true'">$(_PythonWarningParameter) -Werror</_PythonWarningParameter>
       <_EventingSourceFileDirectory>%(EventingSourceFile.RootDir)%(EventingSourceFile.Directory)</_EventingSourceFileDirectory>
       <_EventingSourceFileDirectory Condition="HasTrailingSlash('$(_EventingSourceFileDirectory)')">$(_EventingSourceFileDirectory.TrimEnd('\'))</_EventingSourceFileDirectory>
     </PropertyGroup>
 
-    <Exec Command="&quot;$(PYTHON)&quot; -B $(_PythonWarningParameter) &quot;@(EventingGenerationScript)&quot; --man &quot;@(EventManifestFile)&quot; --intermediate &quot;$(_EventingSourceFileDirectory)&quot; --runtimeflavor mono" />
+    <Exec Command="&quot;$(PYTHON)&quot; -B $(_PythonWarningParameter) &quot;@(EventingGenerationScript)&quot; --man &quot;@(EventManifestFile)&quot; --intermediate &quot;$(_EventingSourceFileDirectory)&quot;" />
 
     <ItemGroup>
       <FileWrites Include="@(EventingSourceFile)" />

diff --git a/src/tests/issues.targets b/src/tests/issues.targets
@@ -686,9 +686,6 @@
 
     <!-- NativeAOT specific -->
     <ItemGroup Condition="'$(XunitTestBinBase)' != '' and '$(TestBuildMode)' == 'nativeaot' and '$(RuntimeFlavor)' == 'coreclr'">
-        <ExcludeList Include="$(XunitTestBinBase)/tracing/eventlistener/EventListenerThreadPool/**">
-            <Issue>https://github.com/dotnet/runtime/issues/105556</Issue>
-        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/baseservices/threading/regressions/beta2/437017/**">
             <Issue>https://github.com/dotnet/runtime/issues/102544</Issue>
         </ExcludeList>