Fix null ref in SyntaxTokenCache (dotnet/aspnetcore#30978) (dotnet/as…

…pnetcore#30988)

Fixes dotnet/aspnetcore#27154

Commit migrated from dotnet/aspnetcore@dec60c8e37b9

src/Microsoft.AspNetCore.Razor.Language/src/Syntax/InternalSyntax/SyntaxFactory.cs

@@ -1,17 +1,15 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System;
-
 namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
 {
     internal static partial class SyntaxFactory
     {
         internal static SyntaxToken Token(SyntaxKind kind, string content, params RazorDiagnostic[] diagnostics)
         {
-            if (SyntaxTokenCache.CanBeCached(kind, diagnostics))
+            if (SyntaxTokenCache.Instance.CanBeCached(kind, diagnostics))
             {
-                return SyntaxTokenCache.GetCachedToken(kind, content);
+                return SyntaxTokenCache.Instance.GetCachedToken(kind, content);
             }
 
             return new SyntaxToken(kind, content, diagnostics);

src/Microsoft.AspNetCore.Razor.Language/src/Syntax/InternalSyntax/SyntaxTokenCache.cs

@@ -1,23 +1,27 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+#nullable enable
+
 using System;
-using Microsoft.Extensions.Internal;
 
 namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
 {
     // Simplified version of Roslyn's SyntaxNodeCache
-    internal static class SyntaxTokenCache
+    internal sealed class SyntaxTokenCache
     {
         private const int CacheSizeBits = 16;
         private const int CacheSize = 1 << CacheSizeBits;
         private const int CacheMask = CacheSize - 1;
+        public static readonly SyntaxTokenCache Instance = new();
         private static readonly Entry[] s_cache = new Entry[CacheSize];
 
-        private struct Entry
+        internal SyntaxTokenCache() { }
+
+        private readonly struct Entry
         {
             public int Hash { get; }
-            public SyntaxToken Token { get; }
+            public SyntaxToken? Token { get; }
 
             internal Entry(int hash, SyntaxToken token)
             {
@@ -26,7 +30,7 @@ internal Entry(int hash, SyntaxToken token)
             }
         }
 
-        public static bool CanBeCached(SyntaxKind kind, params RazorDiagnostic[] diagnostics)
+        public bool CanBeCached(SyntaxKind kind, params RazorDiagnostic[] diagnostics)
         {
             if (diagnostics.Length == 0)
             {
@@ -50,7 +54,7 @@ public static bool CanBeCached(SyntaxKind kind, params RazorDiagnostic[] diagnos
             return false;
         }
 
-        public static SyntaxToken GetCachedToken(SyntaxKind kind, string content)
+        public SyntaxToken GetCachedToken(SyntaxKind kind, string content)
         {
             var hash = (kind, content).GetHashCode();
 
@@ -60,7 +64,7 @@ public static SyntaxToken GetCachedToken(SyntaxKind kind, string content)
             var idx = indexableHash & CacheMask;
             var e = s_cache[idx];
 
-            if (e.Hash == hash && e.Token.Kind == kind && e.Token.Content == content)
+            if (e.Hash == hash && e.Token != null && e.Token.Kind == kind && e.Token.Content == content)
             {
                 return e.Token;
             }

src/Microsoft.AspNetCore.Razor.Language/test/Syntax/SyntaxTokenCacheTest.cs

@@ -0,0 +1,78 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Xunit;
+
+namespace Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax
+{
+    public class SyntaxTokenCacheTest
+    {
+        // Regression test for https://github.com/dotnet/aspnetcore/issues/27154
+        [Fact]
+        public void GetCachedToken_ReturnsNewEntry()
+        {
+            // Arrange
+            var cache = new SyntaxTokenCache();
+
+            // Act
+            var token = cache.GetCachedToken(SyntaxKind.Whitespace, "Hello world");
+
+            // Assert
+            Assert.Equal(SyntaxKind.Whitespace, token.Kind);
+            Assert.Equal("Hello world", token.Content);
+            Assert.Empty(token.GetDiagnostics());
+        }
+
+        [Fact]
+        public void GetCachedToken_ReturnsCachedToken()
+        {
+            // Arrange
+            var cache = new SyntaxTokenCache();
+
+            // Act
+            var token1 = cache.GetCachedToken(SyntaxKind.Whitespace, "Hello world");
+            var token2 = cache.GetCachedToken(SyntaxKind.Whitespace, "Hello world");
+
+            // Assert
+            Assert.Same(token1, token2);
+        }
+
+        [Fact]
+        public void GetCachedToken_ReturnsDifferentEntries_IfKindsAreDifferent()
+        {
+            // Arrange
+            var cache = new SyntaxTokenCache();
+
+            // Act
+            var token1 = cache.GetCachedToken(SyntaxKind.Whitespace, "Hello world");
+            var token2 = cache.GetCachedToken(SyntaxKind.Keyword, "Hello world");
+
+            // Assert
+            Assert.NotSame(token1, token2);
+            Assert.Equal(SyntaxKind.Whitespace, token1.Kind);
+            Assert.Equal("Hello world", token1.Content);
+
+            Assert.Equal(SyntaxKind.Keyword, token2.Kind);
+            Assert.Equal("Hello world", token2.Content);
+        }
+
+        [Fact]
+        public void GetCachedToken_ReturnsDifferentEntries_IfContentsAreDifferent()
+        {
+            // Arrange
+            var cache = new SyntaxTokenCache();
+
+            // Act
+            var token1 = cache.GetCachedToken(SyntaxKind.Keyword, "Text1");
+            var token2 = cache.GetCachedToken(SyntaxKind.Keyword, "Text2");
+
+            // Assert
+            Assert.NotSame(token1, token2);
+            Assert.Equal(SyntaxKind.Keyword, token1.Kind);
+            Assert.Equal("Text1", token1.Content);
+
+            Assert.Equal(SyntaxKind.Keyword, token2.Kind);
+            Assert.Equal("Text2", token2.Content);
+        }
+    }
+}