Update project configuration from Roslyn info

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/CSharp/ParseOptionsExtensions.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
+
+internal static class ParseOptionsExtensions
+{
+    public static bool UseRoslynTokenizer(this ParseOptions parseOptions)
+        => parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRoslynTokenizerValue)
+           && string.Equals(useRoslynTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);
+}

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerator.RazorProviders.cs

@@ -56,8 +56,7 @@ public partial class RazorSourceGenerator
             var razorConfiguration = new RazorConfiguration(razorLanguageVersion, configurationName ?? "default", Extensions: [], UseConsolidatedMvcViews: true, SuppressAddComponentParameter: !isComponentParameterSupported);
 
             // We use the new tokenizer only when requested for now.
-            var useRoslynTokenizer = parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRoslynTokenizerValue)
-                                    && string.Equals(useRoslynTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);
+            var useRoslynTokenizer = parseOptions.UseRoslynTokenizer();
 
             var razorSourceGenerationOptions = new RazorSourceGenerationOptions()
             {

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/LanguageServer/RazorLanguageServerBenchmarkBase.cs

@@ -73,7 +73,7 @@ await projectManager.UpdateAsync(
             {
                 updater.ProjectAdded(hostProject);
                 var tagHelpers = CommonResources.LegacyTagHelpers;
-                var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, CodeAnalysis.CSharp.LanguageVersion.CSharp11);
+                var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, useRoslynTokenizer: false, CodeAnalysis.CSharp.LanguageVersion.CSharp11);
                 updater.ProjectWorkspaceStateChanged(hostProject.Key, projectWorkspaceState);
                 updater.DocumentAdded(hostProject.Key, hostDocument, textLoader);
             },

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/Resources/Telerik/Kendo.Mvc.Examples.project.razor.json

@@ -1,5 +1,5 @@
 {
-  "__Version": 6,
+  "__Version": 7,
   "ProjectKey": "C:\\Users\\admin\\location\\Kendo.Mvc.Examples\\obj\\Debug\\net7.0\\",
   "FilePath": "C:\\Users\\admin\\location\\Kendo.Mvc.Examples\\Kendo.Mvc.Examples.csproj",
   "Configuration": {
@@ -159527,6 +159527,7 @@
         }
       }
     ],
+    "UseRoslynTokenizer": false,
     "CSharpLanguageVersion": 1100
   },
   "RootNamespace": "Kendo.Mvc.Examples",

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/Resources/project.razor.json

@@ -1,5 +1,5 @@
 {
-  "__Version": 6,
+  "__Version": 7,
   "ProjectKey": "C:\\Users\\admin\\location\\blazorserver\\obj\\Debug\\net7.0\\",
   "FilePath": "C:\\Users\\admin\\location\\blazorserver\\blazorserver.csproj",
   "Configuration": {
@@ -16126,6 +16126,7 @@
         }
       }
     ],
+    "UseRoslynTokenizer": false,
     "CSharpLanguageVersion": 800
   },
   "RootNamespace": "blazorserver",

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/ProjectSystem/ProjectWorkspaceState.cs

@@ -12,45 +12,51 @@ namespace Microsoft.AspNetCore.Razor.ProjectSystem;
 
 internal sealed class ProjectWorkspaceState : IEquatable<ProjectWorkspaceState>
 {
-    public static readonly ProjectWorkspaceState Default = new(ImmutableArray<TagHelperDescriptor>.Empty, LanguageVersion.Default);
+    public static readonly ProjectWorkspaceState Default = new(ImmutableArray<TagHelperDescriptor>.Empty, useRoslynTokenizer: false, LanguageVersion.Default);
 
     public ImmutableArray<TagHelperDescriptor> TagHelpers { get; }
     public LanguageVersion CSharpLanguageVersion { get; }
+    public bool UseRoslynTokenizer { get; }
 
     private ProjectWorkspaceState(
         ImmutableArray<TagHelperDescriptor> tagHelpers,
+        bool useRoslynTokenizer,
         LanguageVersion csharpLanguageVersion)
     {
         TagHelpers = tagHelpers;
         CSharpLanguageVersion = csharpLanguageVersion;
+        UseRoslynTokenizer = useRoslynTokenizer;
     }
 
     public static ProjectWorkspaceState Create(
         ImmutableArray<TagHelperDescriptor> tagHelpers,
+        bool useRoslynTokenizer = false,
         LanguageVersion csharpLanguageVersion = LanguageVersion.Default)
         => tagHelpers.IsEmpty && csharpLanguageVersion == LanguageVersion.Default
             ? Default
-            : new(tagHelpers, csharpLanguageVersion);
+            : new(tagHelpers, useRoslynTokenizer, csharpLanguageVersion);
 
-    public static ProjectWorkspaceState Create(LanguageVersion csharpLanguageVersion)
+    public static ProjectWorkspaceState Create(LanguageVersion csharpLanguageVersion, bool useRoslynTokenizer = false)
         => csharpLanguageVersion == LanguageVersion.Default
             ? Default
-            : new(ImmutableArray<TagHelperDescriptor>.Empty, csharpLanguageVersion);
+            : new(ImmutableArray<TagHelperDescriptor>.Empty, useRoslynTokenizer, csharpLanguageVersion);
 
     public override bool Equals(object? obj)
         => obj is ProjectWorkspaceState other && Equals(other);
 
     public bool Equals(ProjectWorkspaceState? other)
         => other is not null &&
            TagHelpers.SequenceEqual(other.TagHelpers) &&
-           CSharpLanguageVersion == other.CSharpLanguageVersion;
+           CSharpLanguageVersion == other.CSharpLanguageVersion &&
+           UseRoslynTokenizer == other.UseRoslynTokenizer;
 
     public override int GetHashCode()
     {
         var hash = HashCodeCombiner.Start();
 
         hash.Add(TagHelpers);
         hash.Add(CSharpLanguageVersion);
+        hash.Add(UseRoslynTokenizer);
 
         return hash.CombinedHash;
     }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/Formatters/ProjectWorkspaceStateFormatter.cs

@@ -22,7 +22,7 @@ private ProjectWorkspaceStateFormatter()
 
     public override ProjectWorkspaceState Deserialize(ref MessagePackReader reader, SerializerCachingOptions options)
     {
-        reader.ReadArrayHeaderAndVerify(3);
+        reader.ReadArrayHeaderAndVerify(4);
 
         var checksums = reader.Deserialize<ImmutableArray<Checksum>>(options);
 
@@ -47,19 +47,21 @@ public override ProjectWorkspaceState Deserialize(ref MessagePackReader reader,
         }
 
         var tagHelpers = builder.DrainToImmutable();
+        var useRoslynTokenizer = reader.ReadBoolean();
         var csharpLanguageVersion = (LanguageVersion)reader.ReadInt32();
 
-        return ProjectWorkspaceState.Create(tagHelpers, csharpLanguageVersion);
+        return ProjectWorkspaceState.Create(tagHelpers, useRoslynTokenizer, csharpLanguageVersion);
     }
 
     public override void Serialize(ref MessagePackWriter writer, ProjectWorkspaceState value, SerializerCachingOptions options)
     {
-        writer.WriteArrayHeader(3);
+        writer.WriteArrayHeader(4);
 
         var checksums = value.TagHelpers.SelectAsArray(x => x.Checksum);
 
         writer.Serialize(checksums, options);
         writer.Serialize(value.TagHelpers, options);
+        writer.Write(value.UseRoslynTokenizer);
         writer.Write((int)value.CSharpLanguageVersion);
     }
 }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/SerializationFormat.cs

@@ -9,5 +9,5 @@ internal static class SerializationFormat
     // or any of the types that compose it changes. This includes: RazorConfiguration,
     // ProjectWorkspaceState, TagHelperDescriptor, and DocumentSnapshotHandle.
     // NOTE: If this version is changed, a coordinated insertion is required between Roslyn and Razor for the C# extension.
-    public const int Version = 6;
+    public const int Version = 7;
 }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Utilities/RazorProjectInfoFactory.cs

@@ -19,6 +19,7 @@
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 
 namespace Microsoft.AspNetCore.Razor.Utilities;
 
@@ -56,7 +57,9 @@ static RazorProjectInfoFactory()
             return null;
         }
 
-        var csharpLanguageVersion = (project.ParseOptions as CSharpParseOptions)?.LanguageVersion ?? LanguageVersion.Default;
+        var cSharpParseOptions = project.ParseOptions as CSharpParseOptions ?? CSharpParseOptions.Default;
+        var csharpLanguageVersion = cSharpParseOptions.LanguageVersion;
+        var useRoslynTokenizer = cSharpParseOptions.UseRoslynTokenizer();
 
         var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
         if (compilation is null)
@@ -77,6 +80,7 @@ static RazorProjectInfoFactory()
 
             builder.SetCSharpLanguageVersion(csharpLanguageVersion);
             builder.SetSupportLocalizedComponentNames(); // ProjectState in MS.CA.Razor.Workspaces does this, so I'm doing it too!
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer, cSharpParseOptions));
         };
 
         var engineFactory = ProjectEngineFactories.DefaultProvider.GetFactory(configuration);
@@ -88,7 +92,7 @@ static RazorProjectInfoFactory()
 
         var tagHelpers = await project.GetTagHelpersAsync(engine, NoOpTelemetryReporter.Instance, cancellationToken).ConfigureAwait(false);
 
-        var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, csharpLanguageVersion);
+        var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, useRoslynTokenizer, csharpLanguageVersion);
 
         return new RazorProjectInfo(
             projectKey: new ProjectKey(intermediateOutputPath),

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Utilities/StreamExtensions.NetCore.cs

@@ -1,14 +1,14 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
-using Microsoft.AspNetCore.Razor.ProjectSystem;
 using System;
 using System.Buffers;
 using System.Diagnostics;
 using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Razor.ProjectSystem;
 
 namespace Microsoft.AspNetCore.Razor.Utilities;
 

src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/ProjectSystem/ProjectState.cs

@@ -13,6 +13,7 @@
 using Microsoft.AspNetCore.Razor.Utilities;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Text;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 
 namespace Microsoft.CodeAnalysis.Razor.ProjectSystem;
 
@@ -150,14 +151,15 @@ private ProjectState(
         else
         {
             ProjectWorkspaceStateVersion = Version;
-        }
 
-        if ((difference & ClearProjectWorkspaceStateVersionMask) != 0 &&
-            CSharpLanguageVersion != older.CSharpLanguageVersion)
-        {
-            // C# language version changed. This impacts the ProjectEngine, reset it.
-            _projectEngine = null;
-            ConfigurationVersion = Version;
+            // CSharpLanguageVersion and UseRoslynTokenizer are part of the ProjectWorkspaceState, but they affect the project engine
+            // so we check for those specifically changing, and clear that.
+            if (CSharpLanguageVersion != older.CSharpLanguageVersion ||
+                ProjectWorkspaceState.UseRoslynTokenizer != older.ProjectWorkspaceState.UseRoslynTokenizer)
+            {
+                _projectEngine = null;
+                ConfigurationVersion = Version;
+            }
         }
     }
 
@@ -209,6 +211,7 @@ RazorProjectEngine CreateProjectEngine()
                     builder.SetRootNamespace(HostProject.RootNamespace);
                     builder.SetCSharpLanguageVersion(CSharpLanguageVersion);
                     builder.SetSupportLocalizedComponentNames();
+                    builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: ProjectWorkspaceState.UseRoslynTokenizer, CSharpParseOptions.Default));
                 });
             }
         }

src/Razor/src/Microsoft.CodeAnalysis.Remote.Razor/ProjectSystem/RemoteProjectSnapshot.cs

@@ -19,6 +19,7 @@
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Microsoft.VisualStudio.Threading;
 
 namespace Microsoft.CodeAnalysis.Remote.Razor.ProjectSystem;
@@ -51,6 +52,7 @@ public RemoteProjectSnapshot(Project project, RemoteSolutionSnapshot solutionSna
         _lazyProjectEngine = new AsyncLazy<RazorProjectEngine>(async () =>
         {
             var configuration = await _lazyConfiguration.GetValueAsync();
+            var csharpParseOptions = project.ParseOptions as CSharpParseOptions ?? CSharpParseOptions.Default;
             return ProjectEngineFactories.DefaultProvider.Create(
                 configuration,
                 rootDirectoryPath: Path.GetDirectoryName(FilePath).AssumeNotNull(),
@@ -59,6 +61,7 @@ public RemoteProjectSnapshot(Project project, RemoteSolutionSnapshot solutionSna
                     builder.SetRootNamespace(RootNamespace);
                     builder.SetCSharpLanguageVersion(CSharpLanguageVersion);
                     builder.SetSupportLocalizedComponentNames();
+                    builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: csharpParseOptions.UseRoslynTokenizer(), csharpParseOptions));
                 });
         },
         joinableTaskFactory: null);

src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/LiveShare/Host/ProjectSnapshotManagerProxy.cs

@@ -101,7 +101,7 @@ internal async Task<ProjectSnapshotManagerProxyState> CalculateUpdatedStateAsync
         }
 
         var tagHelpers = await project.GetTagHelpersAsync(CancellationToken.None).ConfigureAwait(false);
-        var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, project.CSharpLanguageVersion);
+        var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, project.ProjectWorkspaceState.UseRoslynTokenizer, project.CSharpLanguageVersion);
         var projectFilePath = _session.ConvertLocalPathToSharedUri(project.FilePath);
         var intermediateOutputPath = _session.ConvertLocalPathToSharedUri(project.IntermediateOutputPath);
         var projectHandleProxy = new ProjectSnapshotHandleProxy(projectFilePath, intermediateOutputPath, project.Configuration, project.RootNamespace, projectWorkspaceState);

src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/ProjectWorkspaceStateGenerator.cs

@@ -6,11 +6,13 @@
 using System.ComponentModel.Composition;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.PooledObjects;
 using Microsoft.AspNetCore.Razor.ProjectSystem;
 using Microsoft.AspNetCore.Razor.Telemetry;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
 using Microsoft.CodeAnalysis.Razor.Logging;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
 using Microsoft.CodeAnalysis.Razor.Workspaces;
@@ -138,9 +140,7 @@ private async Task UpdateWorkspaceStateAsync(Project? workspaceProject, IProject
                 return;
             }
 
-            var workspaceState = await GetProjectWorkspaceStateAsync(workspaceProject, projectSnapshot, cancellationToken);
-
-            if (workspaceState is null)
+            if (await GetProjectWorkspaceStateAndConfigurationAsync(workspaceProject, projectSnapshot, cancellationToken) is not var (workspaceState, configuration))
             {
                 _logger.LogTrace($"Didn't receive {nameof(ProjectWorkspaceState)} for '{projectKey}'");
                 return;
@@ -153,21 +153,26 @@ private async Task UpdateWorkspaceStateAsync(Project? workspaceProject, IProject
 
             _logger.LogTrace($"Received {nameof(ProjectWorkspaceState)} with {workspaceState.TagHelpers.Length} tag helper(s) for '{projectKey}'");
 
+            // We got a possibly changed configuration, but need a host project to update. Fortunately everything will no-op if this
+            // is unchanged
+            var hostProject = new HostProject(projectSnapshot.FilePath, projectSnapshot.IntermediateOutputPath, configuration, projectSnapshot.RootNamespace, projectSnapshot.DisplayName);
+
             await _projectManager
                 .UpdateAsync(
                     static (updater, state) =>
                     {
-                        var (projectKey, workspaceState, logger, cancellationToken) = state;
+                        var (projectKey, workspaceState, hostProject, logger, cancellationToken) = state;
 
                         if (cancellationToken.IsCancellationRequested)
                         {
                             return;
                         }
 
                         logger.LogTrace($"Updating project with {workspaceState.TagHelpers.Length} tag helper(s) for '{projectKey}'");
+                        updater.ProjectConfigurationChanged(hostProject);
                         updater.ProjectWorkspaceStateChanged(projectKey, workspaceState);
                     },
-                    state: (projectKey, workspaceState, _logger, cancellationToken),
+                    state: (projectKey, workspaceState, hostProject, _logger, cancellationToken),
                     cancellationToken)
                 .ConfigureAwait(false);
         }
@@ -256,7 +261,7 @@ private void ReleaseSemaphore(ProjectKey projectKey)
     ///  Attempts to produce a <see cref="ProjectWorkspaceState"/> from the provide <see cref="Project"/> and <see cref="IProjectSnapshot"/>.
     ///  Returns <see langword="null"/> if an error is encountered.
     /// </summary>
-    private async Task<ProjectWorkspaceState?> GetProjectWorkspaceStateAsync(
+    private async Task<(ProjectWorkspaceState, RazorConfiguration)?> GetProjectWorkspaceStateAndConfigurationAsync(
         Project? workspaceProject,
         IProjectSnapshot projectSnapshot,
         CancellationToken cancellationToken)
@@ -265,7 +270,7 @@ private void ReleaseSemaphore(ProjectKey projectKey)
         // we just a default ProjectWorkspaceState.
         if (workspaceProject is null)
         {
-            return ProjectWorkspaceState.Default;
+            return (ProjectWorkspaceState.Default, RazorConfiguration.Default);
         }
 
         _logger.LogTrace($"Starting tag helper discovery for {projectSnapshot.FilePath}");
@@ -279,9 +284,13 @@ private void ReleaseSemaphore(ProjectKey projectKey)
 
         try
         {
-            var csharpLanguageVersion = workspaceProject.ParseOptions is CSharpParseOptions csharpParseOptions
-                ? csharpParseOptions.LanguageVersion
-                : LanguageVersion.Default;
+            var csharpParseOptions = workspaceProject.ParseOptions as CSharpParseOptions ?? CSharpParseOptions.Default;
+            var csharpLanguageVersion = csharpParseOptions.LanguageVersion;
+            var useRoslynTokenizer = csharpParseOptions.UseRoslynTokenizer();
+
+            var compilation = await workspaceProject.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
+            var suppressAddComponentParameter = compilation is not null && !compilation.HasAddComponentParameter();
+            var configuration = projectSnapshot.Configuration with { SuppressAddComponentParameter = suppressAddComponentParameter };
 
             using var _ = StopwatchPool.GetPooledObject(out var watch);
 
@@ -305,7 +314,7 @@ private void ReleaseSemaphore(ProjectKey projectKey)
                 Project: {projectSnapshot.FilePath}
                 """);
 
-            return ProjectWorkspaceState.Create(tagHelpers, csharpLanguageVersion);
+            return (ProjectWorkspaceState.Create(tagHelpers, useRoslynTokenizer, csharpLanguageVersion), configuration);
         }
         catch (OperationCanceledException)
         {