Roslyn Tokenizer Infrastructure

Directory.Packages.props

@@ -113,7 +113,7 @@
     <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="6.0.0" />
     <PackageVersion Include="System.Private.Uri" Version="4.3.2" />
     <PackageVersion Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataPackageVersion)" />
-    <PackageVersion Include="System.Security.Cryptography.Xml" Version="6.0.1" />
+    <PackageVersion Include="System.Security.Cryptography.Xml" Version="7.0.1" />
     <PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
     <PackageVersion Include="System.Text.Json" Version="6.0.0" />
     <PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.5.4" />

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpAutoCompleteTest.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpAutoCompleteTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpAutoCompleteTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void FunctionsDirectiveAutoCompleteAtEOF()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpBlockTest.cs

@@ -7,7 +7,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpBlockTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpBlockTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void CSharpBlock_SingleLineControlFlowStatement_Error()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpErrorTest.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpErrorTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpErrorTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void HandlesQuotesAfterTransition()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpExplicitExpressionTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpExplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpExplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void ShouldOutputZeroLengthCodeSpanIfExplicitExpressionIsEmpty()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpFunctionsTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpFunctionsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpFunctionsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void Functions_SingleLineControlFlowStatement_Error()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpImplicitExpressionTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpImplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpImplicitExpressionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void ParsesNullConditionalOperatorImplicitExpression_Bracket1()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpNestedStatementsTest.cs

@@ -7,7 +7,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpNestedStatementsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpNestedStatementsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void NestedSimpleStatement()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpRazorCommentsTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpRazorCommentsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpRazorCommentsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void UnterminatedRazorComment()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpReservedWordsTest.cs

@@ -7,7 +7,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpReservedWordsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpReservedWordsTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void ReservedWord()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpSectionTest.cs

@@ -10,7 +10,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpSectionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpSectionTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void CapturesNewlineImmediatelyFollowing()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpSpecialBlockTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpSpecialBlockTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpSpecialBlockTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void NonKeywordStatementInCodeBlockIsHandledCorrectly()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpStatementTest.cs

@@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 //  * Tests for various types of nested statements
 //  * Comment tests
 
-public class CSharpStatementTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpStatementTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void ForStatement()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpTemplateTest.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpTemplateTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpTemplateTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void HandlesSingleLineTemplate()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpToMarkupSwitchTest.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public class CSharpToMarkupSwitchTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true)
+public class CSharpToMarkupSwitchTest() : ParserTestBase(layer: TestProject.Layer.Compiler, validateSpanEditHandlers: true, useLegacyTokenizer: true)
 {
     [Fact]
     public void SingleAngleBracketDoesNotCauseSwitchIfOuterBlockIsTerminated()

src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/CSharpTokenizerCommentTest.cs

@@ -0,0 +1,96 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#nullable disable
+
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Razor.Language.Legacy;
+
+public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase
+{
+    private new SyntaxToken IgnoreRemaining => (SyntaxToken)base.IgnoreRemaining;
+
+    [Fact]
+    public void Next_Ignores_Star_At_EOF_In_RazorComment()
+    {
+        TestTokenizer(
+            "@* Foo * Bar * Baz *",
+            SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentLiteral, " Foo * Bar * Baz *"));
+    }
+
+    [Fact]
+    public void Next_Ignores_Star_Without_Trailing_At()
+    {
+        TestTokenizer(
+            "@* Foo * Bar * Baz *@",
+            SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentLiteral, " Foo * Bar * Baz "),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
+    }
+
+    [Fact]
+    public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
+    {
+        TestTokenizer(
+            "@* Foo Bar Baz *@",
+            SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentLiteral, " Foo Bar Baz "),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentStar, "*"),
+            SyntaxFactory.Token(SyntaxKind.RazorCommentTransition, "@"));
+    }
+
+    [Fact]
+    public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
+    {
+        TestTokenizer("// Foo Bar Baz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"));
+    }
+
+    [Fact]
+    public void Single_Line_Comment_Is_Terminated_By_Newline()
+    {
+        TestTokenizer("// Foo Bar Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
+    {
+        TestTokenizer("// Foo/*Bar*/ Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo/*Bar*/ Baz"), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
+    {
+        TestTokenizer("/* Foo\nBar\nBaz */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"));
+    }
+
+    [Fact]
+    public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
+    {
+        TestTokenizer("/* Foo\nBar\nBaz */a", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
+    {
+        TestTokenizer("/* Foo\nBar\nBaz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz"), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
+    {
+        TestTokenizer("/* Foo/*\nBar\nBaz*/ */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo/*\nBar\nBaz*/"), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
+    {
+        TestTokenizer("/* Foo\nBar\nBaz* */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz* */"), IgnoreRemaining);
+    }
+}