Update semantic tokens service to better support The Future™

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/LanguageServer/RazorSemanticTokensBenchmark.cs

@@ -12,11 +12,13 @@
 using BenchmarkDotNet.Attributes;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.LanguageServer;
+using Microsoft.AspNetCore.Razor.LanguageServer.Extensions;
 using Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Logging;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
 using Microsoft.CodeAnalysis.Razor.Workspaces;
+using Microsoft.CodeAnalysis.Text;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
 
@@ -84,18 +86,12 @@ public async Task InitializeRazorSemanticAsync()
     [Benchmark(Description = "Razor Semantic Tokens Range Handling")]
     public async Task RazorSemanticTokensRangeAsync()
     {
-        var textDocumentIdentifier = new TextDocumentIdentifier
-        {
-            Uri = DocumentUri
-        };
         var cancellationToken = CancellationToken.None;
         var documentVersion = 1;
 
         await UpdateDocumentAsync(documentVersion, DocumentSnapshot, cancellationToken).ConfigureAwait(false);
 
-        var clientConnection = RazorLanguageServer.GetRequiredService<IClientConnection>();
-        await RazorSemanticTokenService.GetSemanticTokensAsync(
-            clientConnection, textDocumentIdentifier, Range, DocumentContext, colorBackground: false, cancellationToken).ConfigureAwait(false);
+        await RazorSemanticTokenService.GetSemanticTokensAsync(DocumentContext, Range.ToLinePositionSpan(), colorBackground: false, Guid.Empty, cancellationToken: cancellationToken).ConfigureAwait(false);
     }
 
     private async Task UpdateDocumentAsync(int newVersion, IDocumentSnapshot documentSnapshot, CancellationToken cancellationToken)
@@ -134,21 +130,18 @@ public TestRazorSemanticTokensInfoService(
             IRazorDocumentMappingService documentMappingService,
             RazorSemanticTokensLegendService razorSemanticTokensLegendService,
             IRazorLoggerFactory loggerFactory)
-            : base(documentMappingService, razorSemanticTokensLegendService, languageServerFeatureOptions, loggerFactory, telemetryReporter: null)
+            : base(documentMappingService, razorSemanticTokensLegendService, csharpSemanticTokensProvider: null!, languageServerFeatureOptions, loggerFactory)
         {
         }
 
         // We can't get C# responses without significant amounts of extra work, so let's just shim it for now, any non-Null result is fine.
-        internal override Task<ImmutableArray<SemanticRange>?> GetCSharpSemanticRangesAsync(
-            IClientConnection clientConnection,
+        protected override Task<ImmutableArray<SemanticRange>?> GetCSharpSemanticRangesAsync(
+            VersionedDocumentContext documentContext,
             RazorCodeDocument codeDocument,
-            TextDocumentIdentifier textDocumentIdentifier,
-            Range razorRange,
+            LinePositionSpan razorRange,
             bool colorBackground,
-            long documentVersion,
             Guid correlationId,
-            CancellationToken cancellationToken,
-            string previousResultId = null)
+            CancellationToken cancellationToken)
         {
             return Task.FromResult<ImmutableArray<SemanticRange>?>(ImmutableArray<SemanticRange>.Empty);
         }

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/LanguageServer/RazorSemanticTokensRangeEndpointBenchmark.cs

@@ -18,6 +18,7 @@
 using Microsoft.CodeAnalysis.Razor.Logging;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
 using Microsoft.CodeAnalysis.Razor.Workspaces;
+using Microsoft.CodeAnalysis.Text;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
 
@@ -75,10 +76,9 @@ public async Task InitializeRazorSemanticAsync()
         DocumentContext = new VersionedDocumentContext(documentUri, documentSnapshot, projectContext: null, version);
 
         var razorOptionsMonitor = RazorLanguageServer.GetRequiredService<RazorLSPOptionsMonitor>();
-        var clientConnection = RazorLanguageServer.GetRequiredService<IClientConnection>();
         var clientCapabilitiesService = new BenchmarkClientCapabilitiesService(new VSInternalClientCapabilities() { SupportsVisualStudioExtensions = true });
         var razorSemanticTokensLegendService = new RazorSemanticTokensLegendService(clientCapabilitiesService);
-        SemanticTokensRangeEndpoint = new SemanticTokensRangeEndpoint(RazorSemanticTokenService, razorSemanticTokensLegendService, razorOptionsMonitor, clientConnection);
+        SemanticTokensRangeEndpoint = new SemanticTokensRangeEndpoint(RazorSemanticTokenService, razorSemanticTokensLegendService, razorOptionsMonitor, telemetryReporter: null);
 
         var text = await DocumentContext.GetSourceTextAsync(CancellationToken.None).ConfigureAwait(false);
         Range = new Range
@@ -158,21 +158,18 @@ public TestCustomizableRazorSemanticTokensInfoService(
             IRazorDocumentMappingService documentMappingService,
             RazorSemanticTokensLegendService razorSemanticTokensLegendService,
             IRazorLoggerFactory loggerFactory)
-            : base(documentMappingService, razorSemanticTokensLegendService, languageServerFeatureOptions, loggerFactory, telemetryReporter: null)
+            : base(documentMappingService, razorSemanticTokensLegendService, csharpSemanticTokensProvider: null!, languageServerFeatureOptions, loggerFactory)
         {
         }
 
         // We can't get C# responses without significant amounts of extra work, so let's just shim it for now, any non-Null result is fine.
-        internal override Task<ImmutableArray<SemanticRange>?> GetCSharpSemanticRangesAsync(
-            IClientConnection clientConnection,
+        protected override Task<ImmutableArray<SemanticRange>?> GetCSharpSemanticRangesAsync(
+            VersionedDocumentContext documentContext,
             RazorCodeDocument codeDocument,
-            TextDocumentIdentifier textDocumentIdentifier,
-            Range razorRange,
+            LinePositionSpan razorSpan,
             bool colorBackground,
-            long documentVersion,
             Guid correlationId,
-            CancellationToken cancellationToken,
-            string previousResultId = null)
+            CancellationToken cancellationToken)
         {
             return Task.FromResult<ImmutableArray<SemanticRange>?>(PregeneratedRandomSemanticRanges);
         }

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/LanguageServer/RazorSemanticTokensScrollingBenchmark.cs

@@ -12,6 +12,7 @@
 using Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
+using Microsoft.CodeAnalysis.Text;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
 using static Microsoft.AspNetCore.Razor.Microbenchmarks.LanguageServer.RazorSemanticTokensBenchmark;
@@ -76,34 +77,23 @@ public async Task InitializeRazorSemanticAsync()
     [Benchmark(Description = "Razor Semantic Tokens Range Scrolling")]
     public async Task RazorSemanticTokensRangeScrollingAsync()
     {
-        var textDocumentIdentifier = new TextDocumentIdentifier()
-        {
-            Uri = DocumentUri
-        };
         var cancellationToken = CancellationToken.None;
         var documentVersion = 1;
 
         await UpdateDocumentAsync(documentVersion, DocumentSnapshot).ConfigureAwait(false);
 
         var documentLineCount = Range.End.Line;
 
-        var clientConnection = RazorLanguageServer.GetRequiredService<IClientConnection>();
-
         var lineCount = 0;
         while (lineCount != documentLineCount)
         {
             var newLineCount = Math.Min(lineCount + WindowSize, documentLineCount);
-            var range = new Range
-            {
-                Start = new Position(lineCount, 0),
-                End = new Position(newLineCount, 0)
-            };
+            var span = new LinePositionSpan(new LinePosition(lineCount, 0), new LinePosition(newLineCount, 0));
             await RazorSemanticTokenService!.GetSemanticTokensAsync(
-                clientConnection,
-                textDocumentIdentifier,
-                range,
                 DocumentContext,
+                span,
                 colorBackground: false,
+                Guid.Empty,
                 cancellationToken);
 
             lineCount = newLineCount;

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Extensions/IServiceCollectionExtensions.cs

@@ -126,6 +126,7 @@ public static void AddSemanticTokensServices(this IServiceCollection services)
         services.AddHandlerWithCapabilities<SemanticTokensRangeEndpoint>();
         // Ensure that we don't add the default service if something else has added one.
         services.TryAddSingleton<IRazorSemanticTokensInfoService, RazorSemanticTokensInfoService>();
+        services.AddSingleton<ICSharpSemanticTokensProvider, LSPCSharpSemanticTokensProvider>();
 
         services.AddSingleton<RazorSemanticTokensLegendService>();
 

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Extensions/RazorCodeDocumentExtensions.cs

@@ -2,11 +2,11 @@
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
 using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.LanguageServer.Protocol;
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Workspaces.Extensions;
+using Microsoft.CodeAnalysis.Text;
 
 namespace Microsoft.AspNetCore.Razor.LanguageServer.Extensions;
 
@@ -20,7 +20,7 @@ public static IRazorGeneratedDocument GetGeneratedDocument(this RazorCodeDocumen
             _ => throw new System.InvalidOperationException(),
         };
 
-    public static bool TryGetMinimalCSharpRange(this RazorCodeDocument codeDocument, Range razorRange, [NotNullWhen(true)] out Range? csharpRange)
+    public static bool TryGetMinimalCSharpRange(this RazorCodeDocument codeDocument, LinePositionSpan razorRange, out LinePositionSpan csharpRange)
     {
         SourceSpan? minGeneratedSpan = null;
         SourceSpan? maxGeneratedSpan = null;
@@ -53,16 +53,16 @@ public static bool TryGetMinimalCSharpRange(this RazorCodeDocument codeDocument,
         if (minGeneratedSpan is not null && maxGeneratedSpan is not null)
         {
             var csharpSourceText = codeDocument.GetCSharpSourceText();
-            var startRange = minGeneratedSpan.Value.ToRange(csharpSourceText);
-            var endRange = maxGeneratedSpan.Value.ToRange(csharpSourceText);
+            var startRange = minGeneratedSpan.Value.ToLinePositionSpan(csharpSourceText);
+            var endRange = maxGeneratedSpan.Value.ToLinePositionSpan(csharpSourceText);
 
-            csharpRange = new Range { Start = startRange.Start, End = endRange.End };
+            csharpRange = new LinePositionSpan(startRange.Start, endRange.End);
             Debug.Assert(csharpRange.Start.CompareTo(csharpRange.End) <= 0, "Range.Start should not be larger than Range.End");
 
             return true;
         }
 
-        csharpRange = null;
+        csharpRange = default;
         return false;
     }
 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/InlayHints/InlayHintService.cs

@@ -27,11 +27,13 @@ internal sealed class InlayHintService(IRazorDocumentMappingService documentMapp
         var codeDocument = await documentContext.GetCodeDocumentAsync(cancellationToken).ConfigureAwait(false);
         var csharpDocument = codeDocument.GetCSharpDocument();
 
+        var span = range.ToLinePositionSpan();
+
         // We are given a range by the client, but our mapping only succeeds if the start and end of the range can both be mapped
         // to C#. Since that doesn't logically match what we want from inlay hints, we instead get the minimum range of mappable
         // C# to get hints for. We'll filter that later, to remove the sections that can't be mapped back.
-        if (!_documentMappingService.TryMapToGeneratedDocumentRange(csharpDocument, range, out var projectedRange) &&
-            !codeDocument.TryGetMinimalCSharpRange(range, out projectedRange))
+        if (!_documentMappingService.TryMapToGeneratedDocumentRange(csharpDocument, span, out var projectedLinePositionSpan) &&
+            !codeDocument.TryGetMinimalCSharpRange(span, out projectedLinePositionSpan))
         {
             // There's no C# in the range.
             return null;
@@ -41,7 +43,7 @@ internal sealed class InlayHintService(IRazorDocumentMappingService documentMapp
         // the results, much like folding ranges.
         var delegatedRequest = new DelegatedInlayHintParams(
             Identifier: documentContext.Identifier,
-            ProjectedRange: projectedRange,
+            ProjectedRange: projectedLinePositionSpan.ToRange(),
             ProjectedKind: RazorLanguageKind.CSharp
         );
 

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokensRangeEndpoint.cs

@@ -1,10 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Razor.LanguageServer.Common;
 using Microsoft.AspNetCore.Razor.LanguageServer.EndpointContracts;
+using Microsoft.AspNetCore.Razor.LanguageServer.Extensions;
+using Microsoft.AspNetCore.Razor.Telemetry;
 using Microsoft.VisualStudio.LanguageServer.Protocol;
 
 namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
@@ -14,13 +17,13 @@ internal sealed class SemanticTokensRangeEndpoint(
     IRazorSemanticTokensInfoService semanticTokensInfoService,
     RazorSemanticTokensLegendService razorSemanticTokensLegendService,
     RazorLSPOptionsMonitor razorLSPOptionsMonitor,
-    IClientConnection clientConnection)
+    ITelemetryReporter? telemetryReporter)
     : IRazorRequestHandler<SemanticTokensRangeParams, SemanticTokens?>, ICapabilitiesProvider
 {
     private readonly IRazorSemanticTokensInfoService _semanticTokensInfoService = semanticTokensInfoService;
     private readonly RazorSemanticTokensLegendService _razorSemanticTokensLegendService = razorSemanticTokensLegendService;
     private readonly RazorLSPOptionsMonitor _razorLSPOptionsMonitor = razorLSPOptionsMonitor;
-    private readonly IClientConnection _clientConnection = clientConnection;
+    private readonly ITelemetryReporter? _telemetryReporter = telemetryReporter;
 
     public bool MutatesSolutionState { get; } = false;
 
@@ -39,8 +42,19 @@ public TextDocumentIdentifier GetTextDocumentIdentifier(SemanticTokensRangeParam
         var documentContext = requestContext.GetRequiredDocumentContext();
         var colorBackground = _razorLSPOptionsMonitor.CurrentValue.ColorBackground;
 
-        var semanticTokens = await _semanticTokensInfoService.GetSemanticTokensAsync(_clientConnection, request.TextDocument, request.Range, documentContext, colorBackground, cancellationToken).ConfigureAwait(false);
+        var correlationId = Guid.NewGuid();
+        using var _ = _telemetryReporter?.TrackLspRequest(Methods.TextDocumentSemanticTokensRangeName, LanguageServerConstants.RazorLanguageServerName, correlationId);
 
-        return semanticTokens;
+        var data = await _semanticTokensInfoService.GetSemanticTokensAsync(documentContext, request.Range.ToLinePositionSpan(), colorBackground, correlationId, cancellationToken).ConfigureAwait(false);
+
+        if (data is null)
+        {
+            return null;
+        }
+
+        return new SemanticTokens
+        {
+            Data = data,
+        };
     }
 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/Services/ICSharpSemanticTokensProvider.cs

@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Immutable;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Text;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
+
+internal interface ICSharpSemanticTokensProvider
+{
+    Task<int[]?> GetCSharpSemanticTokensResponseAsync(
+        VersionedDocumentContext documentContext,
+        ImmutableArray<LinePositionSpan> csharpSpans,
+        bool usePreciseSemanticTokenRanges,
+        Guid correlationId,
+        CancellationToken cancellationToken);
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/Services/IRazorSemanticTokenInfoService.cs

@@ -1,13 +1,18 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.VisualStudio.LanguageServer.Protocol;
+using Microsoft.CodeAnalysis.Text;
 
 namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
 
 internal interface IRazorSemanticTokensInfoService
 {
-    Task<SemanticTokens?> GetSemanticTokensAsync(IClientConnection clientConnection, TextDocumentIdentifier textDocumentIdentifier, Range range, VersionedDocumentContext documentContext, bool colorBackground, CancellationToken cancellationToken);
+    /// <summary>
+    /// Gets the int array representing the semantic tokens for the given range.
+    /// </summary>
+    /// <remarks>See https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens for details about the int array</remarks>
+    Task<int[]?> GetSemanticTokensAsync(VersionedDocumentContext documentContext, LinePositionSpan range, bool colorBackground, Guid correlationId, CancellationToken cancellationToken);
 }