Rejig semantic tokens service to not take VS protocol types, and extr…

…act C# functionality to a separate service

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Extensions/IServiceCollectionExtensions.cs

@@ -126,6 +126,7 @@ public static void AddSemanticTokensServices(this IServiceCollection services)
         services.AddHandlerWithCapabilities<SemanticTokensRangeEndpoint>();
         // Ensure that we don't add the default service if something else has added one.
         services.TryAddSingleton<IRazorSemanticTokensInfoService, RazorSemanticTokensInfoService>();
+        services.AddSingleton<ICSharpSemanticTokensProvider, LSPCSharpSemanticTokensProvider>();
 
         services.AddSingleton<RazorSemanticTokensLegendService>();
 

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Extensions/RazorCodeDocumentExtensions.cs

@@ -2,11 +2,11 @@
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
 using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.LanguageServer.Protocol;
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Workspaces.Extensions;
+using Microsoft.CodeAnalysis.Text;
 
 namespace Microsoft.AspNetCore.Razor.LanguageServer.Extensions;
 
@@ -20,7 +20,7 @@ public static IRazorGeneratedDocument GetGeneratedDocument(this RazorCodeDocumen
             _ => throw new System.InvalidOperationException(),
         };
 
-    public static bool TryGetMinimalCSharpRange(this RazorCodeDocument codeDocument, Range razorRange, [NotNullWhen(true)] out Range? csharpRange)
+    public static bool TryGetMinimalCSharpRange(this RazorCodeDocument codeDocument, LinePositionSpan razorRange, out LinePositionSpan csharpRange)
     {
         SourceSpan? minGeneratedSpan = null;
         SourceSpan? maxGeneratedSpan = null;
@@ -53,16 +53,16 @@ public static bool TryGetMinimalCSharpRange(this RazorCodeDocument codeDocument,
         if (minGeneratedSpan is not null && maxGeneratedSpan is not null)
         {
             var csharpSourceText = codeDocument.GetCSharpSourceText();
-            var startRange = minGeneratedSpan.Value.ToRange(csharpSourceText);
-            var endRange = maxGeneratedSpan.Value.ToRange(csharpSourceText);
+            var startRange = minGeneratedSpan.Value.ToLinePositionSpan(csharpSourceText);
+            var endRange = maxGeneratedSpan.Value.ToLinePositionSpan(csharpSourceText);
 
-            csharpRange = new Range { Start = startRange.Start, End = endRange.End };
+            csharpRange = new LinePositionSpan(startRange.Start, endRange.End);
             Debug.Assert(csharpRange.Start.CompareTo(csharpRange.End) <= 0, "Range.Start should not be larger than Range.End");
 
             return true;
         }
 
-        csharpRange = null;
+        csharpRange = default;
         return false;
     }
 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/InlayHints/InlayHintService.cs

@@ -27,11 +27,13 @@ internal sealed class InlayHintService(IRazorDocumentMappingService documentMapp
         var codeDocument = await documentContext.GetCodeDocumentAsync(cancellationToken).ConfigureAwait(false);
         var csharpDocument = codeDocument.GetCSharpDocument();
 
+        var span = range.ToLinePositionSpan();
+
         // We are given a range by the client, but our mapping only succeeds if the start and end of the range can both be mapped
         // to C#. Since that doesn't logically match what we want from inlay hints, we instead get the minimum range of mappable
         // C# to get hints for. We'll filter that later, to remove the sections that can't be mapped back.
-        if (!_documentMappingService.TryMapToGeneratedDocumentRange(csharpDocument, range, out var projectedRange) &&
-            !codeDocument.TryGetMinimalCSharpRange(range, out projectedRange))
+        if (!_documentMappingService.TryMapToGeneratedDocumentRange(csharpDocument, span, out var projectedLinePositionSpan) &&
+            !codeDocument.TryGetMinimalCSharpRange(span, out projectedLinePositionSpan))
         {
             // There's no C# in the range.
             return null;
@@ -41,7 +43,7 @@ internal sealed class InlayHintService(IRazorDocumentMappingService documentMapp
         // the results, much like folding ranges.
         var delegatedRequest = new DelegatedInlayHintParams(
             Identifier: documentContext.Identifier,
-            ProjectedRange: projectedRange,
+            ProjectedRange: projectedLinePositionSpan.ToRange(),
             ProjectedKind: RazorLanguageKind.CSharp
         );
 

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokensRangeEndpoint.cs

@@ -1,10 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Razor.LanguageServer.Common;
 using Microsoft.AspNetCore.Razor.LanguageServer.EndpointContracts;
+using Microsoft.AspNetCore.Razor.LanguageServer.Extensions;
+using Microsoft.AspNetCore.Razor.Telemetry;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
 
 namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
@@ -14,13 +17,13 @@ internal sealed class SemanticTokensRangeEndpoint(
     IRazorSemanticTokensInfoService semanticTokensInfoService,
     RazorSemanticTokensLegendService razorSemanticTokensLegendService,
     RazorLSPOptionsMonitor razorLSPOptionsMonitor,
-    IClientConnection clientConnection)
+    ITelemetryReporter telemetryReporter)
     : IRazorRequestHandler<SemanticTokensRangeParams, SemanticTokens?>, ICapabilitiesProvider
 {
     private readonly IRazorSemanticTokensInfoService _semanticTokensInfoService = semanticTokensInfoService;
     private readonly RazorSemanticTokensLegendService _razorSemanticTokensLegendService = razorSemanticTokensLegendService;
     private readonly RazorLSPOptionsMonitor _razorLSPOptionsMonitor = razorLSPOptionsMonitor;
-    private readonly IClientConnection _clientConnection = clientConnection;
+    private readonly ITelemetryReporter _telemetryReporter = telemetryReporter;
 
     public bool MutatesSolutionState { get; } = false;
 
@@ -39,8 +42,19 @@ public TextDocumentIdentifier GetTextDocumentIdentifier(SemanticTokensRangeParam
         var documentContext = requestContext.GetRequiredDocumentContext();
         var colorBackground = _razorLSPOptionsMonitor.CurrentValue.ColorBackground;
 
-        var semanticTokens = await _semanticTokensInfoService.GetSemanticTokensAsync(_clientConnection, request.TextDocument, request.Range, documentContext, colorBackground, cancellationToken).ConfigureAwait(false);
+        var correlationId = Guid.NewGuid();
+        using var _ = _telemetryReporter?.TrackLspRequest(Methods.TextDocumentSemanticTokensRangeName, LanguageServerConstants.RazorLanguageServerName, correlationId);
 
-        return semanticTokens;
+        var data = await _semanticTokensInfoService.GetSemanticTokensAsync(documentContext, request.Range.ToLinePositionSpan(), colorBackground, correlationId, cancellationToken).ConfigureAwait(false);
+
+        if (data is null)
+        {
+            return null;
+        }
+
+        return new SemanticTokens
+        {
+            Data = data,
+        };
     }
 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/Services/ICSharpSemanticTokensProvider.cs

@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Immutable;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Text;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
+
+internal interface ICSharpSemanticTokensProvider
+{
+    Task<int[]?> GetCSharpSemanticTokensResponseAsync(
+        VersionedDocumentContext documentContext,
+        ImmutableArray<LinePositionSpan> csharpSpans,
+        bool usePreciseSemanticTokenRanges,
+        Guid correlationId,
+        CancellationToken cancellationToken);
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/Services/IRazorSemanticTokenInfoService.cs

@@ -1,13 +1,14 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.VisualStudio.LanguageServer.Protocol;
+using Microsoft.CodeAnalysis.Text;
 
 namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
 
 internal interface IRazorSemanticTokensInfoService
 {
-    Task<SemanticTokens?> GetSemanticTokensAsync(IClientConnection clientConnection, TextDocumentIdentifier textDocumentIdentifier, Range range, VersionedDocumentContext documentContext, bool colorBackground, CancellationToken cancellationToken);
+    Task<int[]?> GetSemanticTokensAsync(VersionedDocumentContext documentContext, LinePositionSpan range, bool colorBackground, Guid correlationId, CancellationToken cancellationToken);
 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/Services/LSPCSharpSemanticTokensProvider.cs

@@ -0,0 +1,106 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Immutable;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Razor.LanguageServer.Common;
+using Microsoft.AspNetCore.Razor.LanguageServer.Extensions;
+using Microsoft.AspNetCore.Razor.LanguageServer.Semantic.Models;
+using Microsoft.AspNetCore.Razor.PooledObjects;
+using Microsoft.CodeAnalysis.Razor.Logging;
+using Microsoft.CodeAnalysis.Text;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
+
+internal class LSPCSharpSemanticTokensProvider(IClientConnection clientConnection, IRazorLoggerFactory loggerFactory) : ICSharpSemanticTokensProvider
+{
+    private readonly IClientConnection _clientConnection = clientConnection;
+    private readonly ILogger _logger = loggerFactory.CreateLogger<LSPCSharpSemanticTokensProvider>();
+
+    public async Task<int[]?> GetCSharpSemanticTokensResponseAsync(
+            VersionedDocumentContext documentContext,
+            ImmutableArray<LinePositionSpan> csharpSpans,
+            bool usePreciseSemanticTokenRanges,
+            Guid correlationId,
+            CancellationToken cancellationToken)
+    {
+        var documentVersion = documentContext.Version;
+
+        using var _ = ListPool<Range>.GetPooledObject(out var csharpRangeList);
+        foreach (var span in csharpSpans)
+        {
+            csharpRangeList.Add(span.ToRange());
+        }
+
+        var csharpRanges = csharpRangeList.ToArray();
+
+        var parameter = new ProvideSemanticTokensRangesParams(documentContext.Identifier.TextDocumentIdentifier, documentVersion, csharpRanges, correlationId);
+        ProvideSemanticTokensResponse? csharpResponse;
+        if (usePreciseSemanticTokenRanges)
+        {
+            csharpResponse = await GetCsharpResponseAsync(_clientConnection, parameter, CustomMessageNames.RazorProvidePreciseRangeSemanticTokensEndpoint, cancellationToken).ConfigureAwait(false);
+
+            // Likely the server doesn't support the new endpoint, fallback to the original one
+            if (csharpResponse?.Tokens is null && csharpRanges.Length > 1)
+            {
+                var minimalRange = new Range
+                {
+                    Start = csharpRanges[0].Start,
+                    End = csharpRanges[^1].End
+                };
+
+                var newParams = new ProvideSemanticTokensRangesParams(
+                    parameter.TextDocument,
+                    parameter.RequiredHostDocumentVersion,
+                    [minimalRange],
+                    parameter.CorrelationId);
+
+                csharpResponse = await GetCsharpResponseAsync(_clientConnection, newParams, CustomMessageNames.RazorProvideSemanticTokensRangeEndpoint, cancellationToken).ConfigureAwait(false);
+            }
+        }
+        else
+        {
+            csharpResponse = await GetCsharpResponseAsync(_clientConnection, parameter, CustomMessageNames.RazorProvideSemanticTokensRangeEndpoint, cancellationToken).ConfigureAwait(false);
+        }
+
+        if (csharpResponse is null)
+        {
+            // C# isn't ready yet, don't make Razor wait for it. Once C# is ready they'll send a refresh notification.
+            return [];
+        }
+
+        var csharpVersion = csharpResponse.HostDocumentSyncVersion;
+        if (csharpVersion != documentVersion)
+        {
+            // No C# response or C# is out of sync with us. Unrecoverable, return null to indicate no change.
+            // Once C# syncs up they'll send a refresh notification.
+            if (csharpVersion == -1)
+            {
+                _logger.LogWarning("Didn't get C# tokens because the virtual document wasn't found, or other problem. We were wanting {documentVersion} but C# could not get any version.", documentVersion);
+            }
+            else if (csharpVersion < documentVersion)
+            {
+                _logger.LogDebug("Didn't wait for Roslyn to get the C# version we were expecting. We are wanting {documentVersion} but C# is at {csharpVersion}.", documentVersion, csharpVersion);
+            }
+            else
+            {
+                _logger.LogWarning("We are behind the C# version which is surprising. Could be an old request that wasn't cancelled, but if not, expect most future requests to fail. We were wanting {documentVersion} but C# is at {csharpVersion}.", documentVersion, csharpVersion);
+            }
+
+            return null;
+        }
+
+        return csharpResponse.Tokens ?? [];
+    }
+
+    private static Task<ProvideSemanticTokensResponse?> GetCsharpResponseAsync(IClientConnection clientConnection, ProvideSemanticTokensRangesParams parameter, string lspMethodName, CancellationToken cancellationToken)
+    {
+        return clientConnection.SendRequestAsync<ProvideSemanticTokensRangesParams, ProvideSemanticTokensResponse?>(
+            lspMethodName,
+            parameter,
+            cancellationToken);
+    }
+}