Roslyn Tokenizer (#11086)

This adds the new roslyn-based tokenizer, off by default, which uses
Roslyn, instead of a native tokenizer, to tokenize C# code. This is a
major change and will be enabled by putting
`<Features>use-roslyn-tokenizer</Features>` in the project file.

Closes #10737
Closes #10568
Closes #7084

docs/Compiler Breaking Changes - DotNet 9.md

@@ -0,0 +1,56 @@
+# This document lists known breaking changes in Razor after .NET 9 all the way to .NET 10.
+
+## Preprocessor directive parsing breaks
+
+***Introduced in VS 17.13p1 and .NET 9.0.200***
+
+A new lexing mode was introduced for understanding the C# sections in razor files that brings increased compatibility with how C# is natively lexed. However, this
+also brings some breaking changes to the Razor compiler's understanding of C# preprocessing directives, which previously did not work consistently. Directives are
+now required to start at the beginning of a line in Razor files (only whitespace is allowed before them). Additionally, disabled sections are now properly disabled
+by the Razor compiler when `#if` preprocessor blocks are considered inactive.
+
+### Preprocessor blocks are required to start at the beginning of a line
+
+```razor
+@{ #if DEBUG /* Previously allowed, now triggers RZ1043 */ }
+<div>test</div>
+@{ #endif /* Previously allowed, now triggers RZ1043 */ }
+```
+
+To fix, move the directives to a new line. Only whitespace is allowed before the directive.
+
+```razor
+@{
+#if DEBUG /* This is allowed */
+}
+<div>test</div>
+@{
+    #endif /* This is allowed */
+}
+```
+
+### Disabled blocks are now considered properly in the Razor compiler
+
+Disabled blocks are now considered completely disabled by the Razor compiler, and no attempt to understand the block is made. When combined with the previous break,
+this means that if an `#else`, `#elif`, or `#endif` was not at the start of a line (modulo whitespace), a larger section of the file will be considered disabled than
+in older versions of the Razor compiler. To help diagnose potential breaks here, the Razor compiler will scan disabled text sections for potential misplaced preprocessor
+directives and report a warning if one is encountered.
+
+```razor
+@{
+#if false
+}
+
+This area is now properly considered disabled by the razor compiler, and no attempt to understand it as either C# or HTML is made. This
+can cause changes to how the output is rendered from previous versions of the Razor compiler.
+
+@{ #else
+    In previous versions of the Razor compiler, this directive would have been picked up. It is no longer picked up because it is not at
+    the start of a line. The Razor compiler will report a warning, RZ1044, to help diagnose any potential breaks in this area.
+}
+
+@{
+#endif
+}
+```
+

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version1_X/test/ModelExpressionPassTest.cs

@@ -8,7 +8,8 @@
 using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
-using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version1_X;
@@ -156,6 +157,7 @@ private RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             b.Features.Add(new RazorPageDocumentClassifierPass());
             b.Features.Add(new MvcViewDocumentClassifierPass());
         }).Engine;

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version1_X/test/ViewComponentTagHelperPassTest.cs

@@ -7,6 +7,8 @@
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Extensions;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -219,6 +221,7 @@ private static RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
             b.Features.Add(new MvcViewDocumentClassifierPass());
 
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/InjectDirectiveTest.cs

@@ -3,9 +3,10 @@
 
 #nullable disable
 
-using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X;
@@ -180,6 +181,7 @@ private RazorEngine CreateEngine()
             // Notice we're not registering the InjectDirective.Pass here so we can run it on demand.
             b.AddDirective(InjectDirective.Directive);
             b.AddDirective(ModelDirective.Directive);
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
 
             b.Features.Add(new RazorPageDocumentClassifierPass());
             b.Features.Add(new MvcViewDocumentClassifierPass());

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/ModelExpressionPassTest.cs

@@ -7,6 +7,8 @@
 using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X;
@@ -154,6 +156,7 @@ private RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             b.Features.Add(new RazorPageDocumentClassifierPass());
             b.Features.Add(new MvcViewDocumentClassifierPass());
         }).Engine;

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/PageDirectiveTest.cs

@@ -6,6 +6,8 @@
 using System.Collections.Immutable;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X;
@@ -128,6 +130,7 @@ private RazorEngine CreateEngine()
         return RazorProjectEngine.Create(b =>
         {
             PageDirective.Register(b);
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions.Version2_X/test/ViewComponentTagHelperPassTest.cs

@@ -7,6 +7,8 @@
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Extensions;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -217,8 +219,8 @@ private static RazorProjectEngine CreateProjectEngine(params TagHelperDescriptor
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new MvcViewDocumentClassifierPass());
-
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         });
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions/test/ModelExpressionPassTest.cs

@@ -7,6 +7,8 @@
 using System.Text;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions;
@@ -154,6 +156,7 @@ private RazorEngine CreateEngine(params TagHelperDescriptor[] tagHelpers)
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
             b.Features.Add(new RazorPageDocumentClassifierPass());
             b.Features.Add(new MvcViewDocumentClassifierPass());
         }).Engine;
@@ -171,7 +174,7 @@ private DocumentIntermediateNode CreateIRDocument(RazorEngine engine, RazorCodeD
             }
         }
 
-        return codeDocument.GetDocumentIntermediateNode(); 
+        return codeDocument.GetDocumentIntermediateNode();
     }
 
     private TagHelperIntermediateNode FindTagHelperNode(IntermediateNode node)

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions/test/PageDirectiveTest.cs

@@ -6,6 +6,8 @@
 using System.Collections.Immutable;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 
 namespace Microsoft.AspNetCore.Mvc.Razor.Extensions;
@@ -128,6 +130,7 @@ private RazorEngine CreateEngine()
         return RazorProjectEngine.Create(b =>
         {
             PageDirective.Register(b);
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         }).Engine;
     }
 

src/Compiler/Microsoft.AspNetCore.Mvc.Razor.Extensions/test/ViewComponentTagHelperPassTest.cs

@@ -7,6 +7,8 @@
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.Language.Extensions;
 using Microsoft.AspNetCore.Razor.Language.Intermediate;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Xunit;
 using static Microsoft.AspNetCore.Razor.Language.CommonMetadata;
 
@@ -217,8 +219,8 @@ private RazorProjectEngine CreateProjectEngine(params TagHelperDescriptor[] tagH
         return RazorProjectEngine.Create(b =>
         {
             b.Features.Add(new MvcViewDocumentClassifierPass());
-
             b.Features.Add(new TestTagHelperFeature(tagHelpers));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions.Default));
         });
     }
 

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpAutoCompleteTest.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpAutoCompleteTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpAutoCompleteTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void FunctionsDirectiveAutoCompleteAtEOF()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpBlockTest.cs

@@ -7,7 +7,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpBlockTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpBlockTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void CSharpBlock_SingleLineControlFlowStatement_Error()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpErrorTest.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpErrorTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpErrorTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void HandlesQuotesAfterTransition()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpExplicitExpressionTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpExplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpExplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void ShouldOutputZeroLengthCodeSpanIfExplicitExpressionIsEmpty()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpFunctionsTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpFunctionsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpFunctionsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void Functions_SingleLineControlFlowStatement_Error()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpImplicitExpressionTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpImplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpImplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void ParsesNullConditionalOperatorImplicitExpression_Bracket1()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpNestedStatementsTest.cs

@@ -7,7 +7,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpNestedStatementsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpNestedStatementsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void NestedSimpleStatement()