Razor fallback changes

src/Compilers/CSharp/Portable/Parser/Lexer.cs

@@ -411,6 +411,9 @@ private SyntaxToken Create(in TokenInfo info, SyntaxListBuilder? leading, Syntax
                     case SyntaxKind.EndOfFileToken:
                         token = SyntaxFactory.Token(leadingNode, info.Kind, trailingNode);
                         break;
+                    case SyntaxKind.RazorContentToken:
+                        token = SyntaxFactory.Token(leadingNode, info.Kind, info.Text, trailingNode);
+                        break;
                     case SyntaxKind.None:
                         token = SyntaxFactory.BadToken(leadingNode, info.Text, trailingNode);
                         break;
@@ -610,6 +613,19 @@ private void ScanSyntaxToken(ref TokenInfo info)
                         !this.ScanIdentifierOrKeyword(ref info))
                     {
                         Debug.Assert(TextWindow.PeekChar() == '@');
+
+                        if (TextWindow.PeekChar(1) == ':')
+                        {
+                            // Razor HTML transition. For best consumption by razor, we want to simply pretend it's a token and
+                            // consume all the way to the end of the line.
+                            info.Kind = SyntaxKind.RazorContentToken;
+                            this.AddError(TextWindow.Position + 1, width: 1, ErrorCode.ERR_ExpectedVerbatimLiteral);
+
+                            this.ScanToEndOfLine();
+                            info.Text = TextWindow.GetText(false);
+                            break;
+                        }
+
                         this.ConsumeAtSignSequence();
                         info.Text = TextWindow.GetText(intern: true);
                         this.AddError(ErrorCode.ERR_ExpectedVerbatimLiteral);
@@ -1945,14 +1961,6 @@ private void LexSyntaxTrivia(bool afterFirstToken, bool isTrailing, ref SyntaxLi
                             onlyWhitespaceOnLine = false;
                             break;
                         }
-                        else if (ch == ':')
-                        {
-                            // Razor HTML transition. We pretend it's a single-line comment for error recovery.
-                            this.AddError(TextWindow.Position, width: 1, ErrorCode.ERR_UnexpectedCharacter, '@');
-                            lexSingleLineComment(ref triviaList);
-                            onlyWhitespaceOnLine = false;
-                            break;
-                        }
                         else
                         {
                             return;

src/Compilers/CSharp/Portable/PublicAPI.Unshipped.txt

@@ -7,22 +7,25 @@ Microsoft.CodeAnalysis.CSharp.Conversion.IsCollectionExpression.get -> bool
 Microsoft.CodeAnalysis.CSharp.Syntax.CrefParameterSyntax.ReadOnlyKeyword.get -> Microsoft.CodeAnalysis.SyntaxToken
 Microsoft.CodeAnalysis.CSharp.Syntax.CrefParameterSyntax.Update(Microsoft.CodeAnalysis.SyntaxToken refKindKeyword, Microsoft.CodeAnalysis.SyntaxToken readOnlyKeyword, Microsoft.CodeAnalysis.CSharp.Syntax.TypeSyntax! type) -> Microsoft.CodeAnalysis.CSharp.Syntax.CrefParameterSyntax!
 Microsoft.CodeAnalysis.CSharp.Syntax.CrefParameterSyntax.WithReadOnlyKeyword(Microsoft.CodeAnalysis.SyntaxToken readOnlyKeyword) -> Microsoft.CodeAnalysis.CSharp.Syntax.CrefParameterSyntax!
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Dispose() -> void
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.ParseNextToken() -> Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.ResetTo(Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result result) -> void
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result.ContextualKind.get -> Microsoft.CodeAnalysis.CSharp.SyntaxKind
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result.Result() -> void
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result.Token.get -> Microsoft.CodeAnalysis.SyntaxToken
-Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.SkipForwardTo(int position) -> void
+Microsoft.CodeAnalysis.CSharp.SyntaxKind.RazorContentToken = 8523 -> Microsoft.CodeAnalysis.CSharp.SyntaxKind
 static Microsoft.CodeAnalysis.CSharp.CSharpExtensions.GetDeclaredSymbol(this Microsoft.CodeAnalysis.SemanticModel? semanticModel, Microsoft.CodeAnalysis.CSharp.Syntax.LocalFunctionStatementSyntax! node, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.IMethodSymbol?
 static Microsoft.CodeAnalysis.CSharp.CSharpExtensions.GetElementConversion(this Microsoft.CodeAnalysis.Operations.ISpreadOperation! spread) -> Microsoft.CodeAnalysis.CSharp.Conversion
-static Microsoft.CodeAnalysis.CSharp.SyntaxFactory.CreateTokenParser(Microsoft.CodeAnalysis.Text.SourceText! sourceText, Microsoft.CodeAnalysis.CSharp.CSharpParseOptions? options = null) -> Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser!
 static Microsoft.CodeAnalysis.CSharp.SyntaxFactory.CrefParameter(Microsoft.CodeAnalysis.SyntaxToken refKindKeyword, Microsoft.CodeAnalysis.SyntaxToken readOnlyKeyword, Microsoft.CodeAnalysis.CSharp.Syntax.TypeSyntax! type) -> Microsoft.CodeAnalysis.CSharp.Syntax.CrefParameterSyntax!
 [RSEXPERIMENTAL001]Microsoft.CodeAnalysis.CSharp.CSharpCompilation.GetSemanticModel(Microsoft.CodeAnalysis.SyntaxTree! syntaxTree, Microsoft.CodeAnalysis.SemanticModelOptions options) -> Microsoft.CodeAnalysis.SemanticModel!
 [RSEXPERIMENTAL002]override abstract Microsoft.CodeAnalysis.CSharp.InterceptableLocation.Equals(object? obj) -> bool
 [RSEXPERIMENTAL002]override abstract Microsoft.CodeAnalysis.CSharp.InterceptableLocation.GetHashCode() -> int
 [RSEXPERIMENTAL002]static Microsoft.CodeAnalysis.CSharp.CSharpExtensions.GetInterceptorMethod(this Microsoft.CodeAnalysis.SemanticModel? semanticModel, Microsoft.CodeAnalysis.CSharp.Syntax.InvocationExpressionSyntax! node, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.IMethodSymbol?
 [RSEXPERIMENTAL002]static Microsoft.CodeAnalysis.CSharp.CSharpExtensions.GetInterceptableLocation(this Microsoft.CodeAnalysis.SemanticModel? semanticModel, Microsoft.CodeAnalysis.CSharp.Syntax.InvocationExpressionSyntax! node, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.CSharp.InterceptableLocation?
 [RSEXPERIMENTAL002]static Microsoft.CodeAnalysis.CSharp.CSharpExtensions.GetInterceptsLocationAttributeSyntax(this Microsoft.CodeAnalysis.CSharp.InterceptableLocation! location) -> string!
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Dispose() -> void
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.ParseLeadingTrivia() -> Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.ParseNextToken() -> Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.ParseTrailingTrivia() -> Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.ResetTo(Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result result) -> void
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result.ContextualKind.get -> Microsoft.CodeAnalysis.CSharp.SyntaxKind
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result.Result() -> void
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.Result.Token.get -> Microsoft.CodeAnalysis.SyntaxToken
+[RSEXPERIMENTAL003]Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser.SkipForwardTo(int position) -> void
+[RSEXPERIMENTAL003]static Microsoft.CodeAnalysis.CSharp.SyntaxFactory.CreateTokenParser(Microsoft.CodeAnalysis.Text.SourceText! sourceText, Microsoft.CodeAnalysis.CSharp.CSharpParseOptions? options = null) -> Microsoft.CodeAnalysis.CSharp.SyntaxTokenParser!

src/Compilers/CSharp/Portable/Syntax/SyntaxFactory.cs

@@ -7,6 +7,7 @@
 using System.Collections.Immutable;
 using System.ComponentModel;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Text;
 using System.Threading;
@@ -1677,6 +1678,7 @@ public static IEnumerable<SyntaxToken> ParseTokens(string text, int offset = 0,
         /// </summary>
         /// <param name="sourceText">The source to parse tokens from.</param>
         /// <param name="options">Parse options for the source.</param>
+        [Experimental(RoslynExperiments.SyntaxTokenParser, UrlFormat = RoslynExperiments.SyntaxTokenParser_Url)]
         public static SyntaxTokenParser CreateTokenParser(SourceText sourceText, CSharpParseOptions? options = null)
         {
             return new SyntaxTokenParser(new InternalSyntax.Lexer(sourceText, options ?? CSharpParseOptions.Default));

src/Compilers/CSharp/Portable/Syntax/SyntaxKind.cs

@@ -526,6 +526,7 @@ public enum SyntaxKind : ushort
         Utf8StringLiteralToken = 8520,
         Utf8SingleLineRawStringLiteralToken = 8521,
         Utf8MultiLineRawStringLiteralToken = 8522,
+        RazorContentToken = 8523,
 
         // trivia
         EndOfLineTrivia = 8539,

src/Compilers/CSharp/Portable/Syntax/SourceTextTokenParser.cs → src/Compilers/CSharp/Portable/Syntax/SyntaxTokenParser.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Diagnostics.CodeAnalysis;
 using System.Threading;
 using InternalSyntax = Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax;
 
@@ -20,6 +21,7 @@ namespace Microsoft.CodeAnalysis.CSharp;
 /// <para />
 /// This type is not thread safe.
 /// </remarks>
+[Experimental(RoslynExperiments.SyntaxTokenParser, UrlFormat = RoslynExperiments.SyntaxTokenParser_Url)]
 public sealed class SyntaxTokenParser : IDisposable
 {
     private InternalSyntax.Lexer _lexer;
@@ -54,6 +56,36 @@ public Result ParseNextToken()
         return new Result(new SyntaxToken(parent: null, token, startingPosition, index: 0), startingDirectiveStack);
     }
 
+    /// <summary>
+    /// Parse the leading trivia of the next token from the input at the current position. This will advance the internal position of the
+    /// token parser to the end of the leading trivia of the next token. The returned result will have a token with <see cref="CSharpExtensions.Kind(SyntaxToken)"/>
+    /// of <see cref="SyntaxKind.None"/>, <see cref="SyntaxToken.IsMissing"/> set to <see langword="true"/>, and a parent of <see langword="null"/>. The
+    /// parsed trivia will be set as the <see cref="SyntaxToken.LeadingTrivia"/> of the token.
+    /// </summary>
+    public Result ParseLeadingTrivia()
+    {
+        var startingDirectiveStack = _lexer.Directives;
+        var startingPosition = _lexer.TextWindow.Position;
+        var leadingTrivia = _lexer.LexSyntaxLeadingTrivia();
+        var containingToken = InternalSyntax.SyntaxFactory.MissingToken(leading: leadingTrivia.Node, SyntaxKind.None, trailing: null);
+        return new Result(new SyntaxToken(parent: null, containingToken, startingPosition, index: 0), startingDirectiveStack);
+    }
+
+    /// <summary>
+    /// Parse syntax trivia from the current position, according to the rules of trailing syntax trivia. This will advance the internal position of the
+    /// token parser to the end of the trailing trivia from the current location. The returned result will have a token with <see cref="CSharpExtensions.Kind(SyntaxToken)"/>
+    /// of <see cref="SyntaxKind.None"/>, <see cref="SyntaxToken.IsMissing"/> set to <see langword="true"/>, and a parent of <see langword="null"/>. The
+    /// parsed trivia will be set as the <see cref="SyntaxToken.TrailingTrivia"/> of the token.
+    /// </summary>
+    public Result ParseTrailingTrivia()
+    {
+        var startingDirectiveStack = _lexer.Directives;
+        var startingPosition = _lexer.TextWindow.Position;
+        var trailingTrivia = _lexer.LexSyntaxTrailingTrivia();
+        var containingToken = InternalSyntax.SyntaxFactory.MissingToken(leading: null, SyntaxKind.None, trailing: trailingTrivia.Node);
+        return new Result(new SyntaxToken(parent: null, containingToken, startingPosition, index: 0), startingDirectiveStack);
+    }
+
     /// <summary>
     /// Skip forward in the input to the specified position. Current directive state is preserved during the skip.
     /// </summary>

src/Compilers/CSharp/Test/Syntax/LexicalAndXml/LexicalErrorTests.cs

@@ -573,48 +573,48 @@ public static int Main ()
         }
 
         [Fact]
-        public void CS1035AtColonParsedAsComment_01()
+        public void CS1035AtColonParsedAsBadRazorContent_01()
         {
             var test = """
                 var x = @:;
                 """;
 
             ParsingTests.ParseAndValidate(test,
-                // (1,9): error CS1056: Unexpected character '@'
+                // (1,9): error CS1525: Invalid expression term ''
                 // var x = @:;
-                Diagnostic(ErrorCode.ERR_UnexpectedCharacter, "@").WithArguments("@").WithLocation(1, 9),
-                // (1,12): error CS1733: Expected expression
+                Diagnostic(ErrorCode.ERR_InvalidExprTerm, "@:;").WithArguments("").WithLocation(1, 9),
+                // (1,10): error CS1646: Keyword, identifier, or string expected after verbatim specifier: @
                 // var x = @:;
-                Diagnostic(ErrorCode.ERR_ExpressionExpected, "").WithLocation(1, 12),
+                Diagnostic(ErrorCode.ERR_ExpectedVerbatimLiteral, ":").WithLocation(1, 10),
                 // (1,12): error CS1002: ; expected
                 // var x = @:;
                 Diagnostic(ErrorCode.ERR_SemicolonExpected, "").WithLocation(1, 12));
         }
 
         [Fact]
-        public void CS1035AtColonParsedAsComment_02()
+        public void CS1035AtColonParsedAsBadRazorContent_02()
         {
             var test = """
                 @:<div>test</div>
                 """;
 
             ParsingTests.ParseAndValidate(test,
-                // (1,1): error CS1056: Unexpected character '@'
+                // (1,2): error CS1646: Keyword, identifier, or string expected after verbatim specifier: @
                 // @:<div>test</div>
-                Diagnostic(ErrorCode.ERR_UnexpectedCharacter, "@").WithArguments("@").WithLocation(1, 1));
+                Diagnostic(ErrorCode.ERR_ExpectedVerbatimLiteral, ":").WithLocation(1, 2));
         }
 
         [Fact]
-        public void CS1035AtColonParsedAsComment_03()
+        public void CS1035AtColonParsedAsBadRazorContent_03()
         {
             var test = """
                 @: M() {}
                 """;
 
             ParsingTests.ParseAndValidate(test,
-                // (1,1): error CS1056: Unexpected character '@'
+                // (1,2): error CS1646: Keyword, identifier, or string expected after verbatim specifier: @
                 // @: M() {}
-                Diagnostic(ErrorCode.ERR_UnexpectedCharacter, "@").WithArguments("@").WithLocation(1, 1));
+                Diagnostic(ErrorCode.ERR_ExpectedVerbatimLiteral, ":").WithLocation(1, 2));
         }
 
         [Fact, WorkItem(526993, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/526993")]