Enable the new tokenizer for tooling tests

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version1_X/test/ModelExpressionPassTest.cs

@@ -8,7 +8,8 @@
 using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
-using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version1_X;
@@ -156,6 +157,7 @@ private RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version1_X/test/ViewComponentTagHelperPassTest.cs

@@ -7,6 +7,8 @@
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Extensions;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -219,6 +221,7 @@ private static RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
             b.Features.Add(new MvcViewDocumentClassifierPass());
 
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/InjectDirectiveTest.cs

@@ -3,9 +3,10 @@
 
 #nullable disable
 
-using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X;
@@ -180,6 +181,7 @@ private RazorEngine CreateEngine()
             // Notice we're not registering the InjectDirective.Pass here so we can run it on demand.
             b.AddDirective(InjectDirective.Directive);
             b.AddDirective(ModelDirective.Directive);
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/ModelExpressionPassTest.cs

@@ -7,6 +7,8 @@
 using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X;
@@ -154,6 +156,7 @@ private RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/PageDirectiveTest.cs

@@ -6,6 +6,8 @@
 using System.Collections.Immutable;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X;
@@ -128,6 +130,7 @@ private RazorEngine CreateEngine()
         return RazorProjectEngine.Create(b =>
         {
             PageDirective.Register(b);
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/ViewComponentTagHelperPassTest.cs

@@ -7,6 +7,8 @@
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Extensions;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -217,8 +219,8 @@ private static RazorProjectEngine CreateProjectEngine(params TagHelperDescriptor
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new MvcViewDocumentClassifierPass());
-
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         });
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions/test/ModelExpressionPassTest.cs

@@ -7,6 +7,8 @@
 using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions;
@@ -154,6 +156,7 @@ private RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions/test/PageDirectiveTest.cs

@@ -6,6 +6,8 @@
 using System.Collections.Immutable;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions;
@@ -128,6 +130,7 @@ private RazorEngine CreateEngine()
         return RazorProjectEngine.Create(b =>
         {
             PageDirective.Register(b);
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions/test/ViewComponentTagHelperPassTest.cs

@@ -7,6 +7,8 @@
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Extensions;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -217,8 +219,8 @@ private RazorProjectEngine CreateProjectEngine(params TagHelperDescriptor[] tagH
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new MvcViewDocumentClassifierPass());
-
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         });
     }
 

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorIntermediateNodeLoweringPhaseIntegrationTest.cs

@@ -9,6 +9,8 @@
 using Microsoft.AspNetCore.Razor.Language.Extensions;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
 using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Moq;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
@@ -465,6 +467,7 @@ private static DocumentIntermediateNode Lower(
             b.AddTagHelpers(tagHelpers);
 
             b.Features.Add(new DesignTimeOptionsFeature(designTime));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         };
 
         var projectEngine = RazorProjectEngine.Create(configureEngine);

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorTagHelperBinderPhaseTest.cs

@@ -9,6 +9,8 @@
 using Microsoft.AspNetCore.Razor.Language.Components;
 using Microsoft.AspNetCore.Razor.Language.Legacy;
 using Microsoft.AspNetCore.Razor.Language.Syntax;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -25,6 +27,7 @@ public void Execute_CanHandleSingleLengthAddTagHelperDirective()
         // Arrange
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             builder.AddTagHelpers([]);
         });
 
@@ -64,6 +67,7 @@ public void Execute_CanHandleSingleLengthRemoveTagHelperDirective()
         // Arrange
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             builder.AddTagHelpers([]);
         });
 
@@ -103,6 +107,7 @@ public void Execute_CanHandleSingleLengthTagHelperPrefix()
         // Arrange
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             builder.AddTagHelpers([]);
         });
 
@@ -142,6 +147,7 @@ public void Execute_RewritesTagHelpers()
         // Arrange
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             builder.AddTagHelpers(
             [
                 CreateTagHelperDescriptor(
@@ -242,7 +248,11 @@ public void Execute_NullTagHelperDescriptorsFromCodeDocument_FallsBackToTagHelpe
                 typeName: "TestInputTagHelper",
                 assemblyName: "TestAssembly"),
         };
-        var projectEngine = RazorProjectEngine.Create(builder => builder.AddTagHelpers(tagHelpers));
+        var projectEngine = RazorProjectEngine.Create(builder =>
+        {
+            builder.AddTagHelpers(tagHelpers);
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
+        });
 
         var discoveryPhase = new DefaultRazorTagHelperContextDiscoveryPhase()
         {
@@ -287,7 +297,11 @@ public void Execute_EmptyTagHelperDescriptorsFromCodeDocument_DoesNotFallbackToT
                 typeName: "TestInputTagHelper",
                 assemblyName: "TestAssembly"),
         };
-        var projectEngine = RazorProjectEngine.Create(builder => builder.AddTagHelpers(tagHelpers));
+        var projectEngine = RazorProjectEngine.Create(builder =>
+        {
+            builder.AddTagHelpers(tagHelpers);
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
+        });
 
         var phase = new DefaultRazorTagHelperContextDiscoveryPhase()
         {
@@ -334,6 +348,7 @@ public void Execute_DirectiveWithoutQuotes_RewritesTagHelpers_TagHelperMatchesEl
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
             builder.AddTagHelpers([descriptor]);
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         });
 
         var discoveryPhase = new DefaultRazorTagHelperContextDiscoveryPhase()
@@ -393,6 +408,7 @@ public void Execute_DirectiveWithQuotes_RewritesTagHelpers_TagHelperMatchesEleme
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
             builder.AddTagHelpers([descriptor]);
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         });
 
         var discoveryPhase = new DefaultRazorTagHelperContextDiscoveryPhase()
@@ -440,7 +456,11 @@ public void Execute_TagHelpersFromCodeDocumentAndFeature_PrefersCodeDocument()
                 typeName: "TestInputTagHelper",
                 assemblyName: "TestAssembly"),
         };
-        var projectEngine = RazorProjectEngine.Create(builder => builder.AddTagHelpers(featureTagHelpers));
+        var projectEngine = RazorProjectEngine.Create(builder =>
+        {
+            builder.AddTagHelpers(featureTagHelpers);
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
+        });
 
         var discoveryPhase = new DefaultRazorTagHelperContextDiscoveryPhase()
         {
@@ -508,6 +528,7 @@ public void Execute_NoopsWhenNoTagHelperDescriptorsAreResolved()
         // Arrange
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             builder.Features.Add(new TestTagHelperFeature());
         });
 
@@ -537,6 +558,7 @@ public void Execute_SetsTagHelperDocumentContext()
         // Arrange
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             builder.Features.Add(new TestTagHelperFeature());
         });
 
@@ -566,6 +588,7 @@ public void Execute_CombinesErrorsOnRewritingErrors()
         // Arrange
         var projectEngine = RazorProjectEngine.Create(builder =>
         {
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             builder.AddTagHelpers(
             [
                 CreateTagHelperDescriptor(

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DirectiveRemovalOptimizationPassTest.cs

@@ -3,9 +3,10 @@
 
 #nullable disable
 
-using System;
 using System.Linq;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.Intermediate.IntermediateNodeAssert;
 
@@ -23,6 +24,7 @@ public void Execute_Custom_RemovesDirectiveNodeFromDocument()
         var defaultEngine = RazorProjectEngine.Create(b =>
         {
             b.AddDirective(DirectiveDescriptor.CreateDirective("custom", DirectiveKind.SingleLine, d => d.AddStringToken()));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
         var documentNode = Lower(codeDocument, defaultEngine);
         var pass = new DirectiveRemovalOptimizationPass()
@@ -57,6 +59,7 @@ public void Execute_MultipleCustomDirectives_RemovesDirectiveNodesFromDocument()
         var defaultEngine = RazorProjectEngine.Create(b =>
         {
             b.AddDirective(DirectiveDescriptor.CreateDirective("custom", DirectiveKind.SingleLine, d => d.AddStringToken()));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
         var documentNode = Lower(codeDocument, defaultEngine);
         var pass = new DirectiveRemovalOptimizationPass()
@@ -88,6 +91,7 @@ public void Execute_DirectiveWithError_PreservesDiagnosticsAndRemovesDirectiveNo
         var codeDocument = RazorCodeDocument.Create(sourceDocument);
         var defaultEngine = RazorProjectEngine.Create(b =>
         {
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             b.AddDirective(DirectiveDescriptor.CreateDirective("custom", DirectiveKind.SingleLine, d => d.AddStringToken()));
         }).Engine;
         var documentNode = Lower(codeDocument, defaultEngine);

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Extensions/DefaultTagHelperOptimizationPassTest.cs

@@ -5,6 +5,8 @@
 
 using System.Linq;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -92,6 +94,7 @@ private RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Extensions/MetadataAttributePassTest.cs

@@ -6,6 +6,8 @@
 using System.Collections.Immutable;
 using Microsoft.AspNetCore.Razor.Language.Components;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.Intermediate.IntermediateNodeAssert;
 
@@ -452,6 +454,7 @@ private static RazorEngine CreateEngine()
     {
         return RazorProjectEngine.Create(b =>
         {
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             b.Features.Add(new DefaultMetadataIdentifierFeature());
         }).Engine;
     }

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Extensions/SectionDirectivePassTest.cs

@@ -4,6 +4,8 @@
 #nullable disable
 
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.Intermediate.IntermediateNodeAssert;
 
@@ -83,6 +85,7 @@ private static RazorProjectEngine CreateProjectEngine()
         return RazorProjectEngine.Create(b =>
         {
             SectionDirective.Register(b);
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         });
     }