From b325cf96c7bca177a072ea423b54814030b36709 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Tue, 7 Nov 2023 14:22:58 -0800 Subject: [PATCH 1/9] in progress --- eng/Versions.props | 1 + .../Microsoft.CodeAnalysis.Workspaces.csproj | 1 + .../v2/SQLitePersistentStorageConstants.cs | 3 +- .../Portable/Workspace/Solution/Checksum.cs | 16 +- .../Workspace/Solution/Checksum_Factory.cs | 147 +++++++++--------- 5 files changed, 81 insertions(+), 87 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index e96575b7c2b9..4f79481ec965 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -245,6 +245,7 @@ 7.0.0 4.3.0 4.3.0 + 8.0.0-rc.2.23479.6 5.0.0 7.0.0 7.0.0 diff --git a/src/Workspaces/Core/Portable/Microsoft.CodeAnalysis.Workspaces.csproj b/src/Workspaces/Core/Portable/Microsoft.CodeAnalysis.Workspaces.csproj index 134a10a305d9..f283c28c8cbb 100644 --- a/src/Workspaces/Core/Portable/Microsoft.CodeAnalysis.Workspaces.csproj +++ b/src/Workspaces/Core/Portable/Microsoft.CodeAnalysis.Workspaces.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Workspaces/Core/Portable/Storage/SQLite/v2/SQLitePersistentStorageConstants.cs b/src/Workspaces/Core/Portable/Storage/SQLite/v2/SQLitePersistentStorageConstants.cs index be5bd0a3b811..98d820834134 100644 --- a/src/Workspaces/Core/Portable/Storage/SQLite/v2/SQLitePersistentStorageConstants.cs +++ b/src/Workspaces/Core/Portable/Storage/SQLite/v2/SQLitePersistentStorageConstants.cs @@ -18,7 +18,8 @@ internal static class SQLitePersistentStorageConstants // 6. Use compression in some features. Need to move to a different table since the blob // format will be different and we don't want different VS versions (that do/don't support // compression constantly stomping on each other. - private const string Version = "6"; + // 7. Checksum size changed from 20 bytes to 16 bytes long. + private const string Version = "7"; /// /// Inside the DB we have a table dedicated to storing strings that also provides a unique diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs index 8f2d05b8461d..8e5cc5ddff60 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs @@ -13,6 +13,7 @@ using System.Runtime.Serialization; using Microsoft.CodeAnalysis.Serialization; using Roslyn.Utilities; +using System.IO.Hashing; namespace Microsoft.CodeAnalysis { @@ -27,7 +28,7 @@ internal sealed partial record class Checksum( /// /// The intended size of the structure. /// - public const int HashSize = 20; + public const int HashSize = 16; public static readonly Checksum Null = new(Hash: default); @@ -119,29 +120,24 @@ public override int GetHashCode() [DataContract, StructLayout(LayoutKind.Explicit, Size = HashSize)] public readonly record struct HashData( [field: FieldOffset(0)][property: DataMember(Order = 0)] long Data1, - [field: FieldOffset(8)][property: DataMember(Order = 1)] long Data2, - [field: FieldOffset(16)][property: DataMember(Order = 2)] int Data3) + [field: FieldOffset(8)][property: DataMember(Order = 1)] long Data2) { public void WriteTo(ObjectWriter writer) { writer.WriteInt64(Data1); writer.WriteInt64(Data2); - writer.WriteInt32(Data3); } public void WriteTo(Span span) { - Contract.ThrowIfFalse(span.Length >= HashSize); + Contract.ThrowIfTrue(span.Length < HashSize); #pragma warning disable CS9191 // The 'ref' modifier for an argument corresponding to 'in' parameter is equivalent to 'in'. Consider using 'in' instead. Contract.ThrowIfFalse(MemoryMarshal.TryWrite(span, ref Unsafe.AsRef(in this))); #pragma warning restore CS9191 } - public static unsafe HashData FromPointer(HashData* hash) - => new(hash->Data1, hash->Data2, hash->Data3); - public static HashData ReadFrom(ObjectReader reader) - => new(reader.ReadInt64(), reader.ReadInt64(), reader.ReadInt32()); + => new(reader.ReadInt64(), reader.ReadInt64()); public override int GetHashCode() { @@ -152,7 +148,7 @@ public override int GetHashCode() // Explicitly implement this method as default jit for records on netfx doesn't properly devirtualize the // standard calls to EqualityComparer.Default.Equals public bool Equals(HashData other) - => this.Data1 == other.Data1 && this.Data2 == other.Data2 && this.Data3 == other.Data3; + => this.Data1 == other.Data1 && this.Data2 == other.Data2; } } diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs index 2934104d7888..03c467242d73 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs @@ -7,9 +7,9 @@ using System.Collections.Immutable; using System.Diagnostics; using System.IO; +using System.IO.Hashing; using System.Linq; using System.Runtime.InteropServices; -using System.Security.Cryptography; using System.Threading; using Microsoft.CodeAnalysis.PooledObjects; using Microsoft.CodeAnalysis.Serialization; @@ -21,15 +21,10 @@ namespace Microsoft.CodeAnalysis internal partial record class Checksum { // https://github.com/dotnet/runtime/blob/f2db6d6093c54e5eeb9db2d8dcbe15b2db92ad8c/src/libraries/System.Security.Cryptography.Algorithms/src/System/Security/Cryptography/SHA256.cs#L18-L19 - private const int SHA256HashSizeBytes = 256 / 8; + private const int XXHash128SizeBytes = 128 / 8; -#if NET - private static readonly ObjectPool s_incrementalHashPool = - new(() => IncrementalHash.CreateHash(HashAlgorithmName.SHA256), size: 20); -#else - private static readonly ObjectPool s_incrementalHashPool = - new(SHA256.Create, size: 20); -#endif + private static readonly ObjectPool s_incrementalHashPool = + new(() => new(), size: 20); #if !NET // Dedicated pools for the byte[]s we use to create checksums from two or three existing checksums. Sized to @@ -43,52 +38,52 @@ internal partial record class Checksum public static Checksum Create(IEnumerable values) { -#if NET +// #if NET using var pooledHash = s_incrementalHashPool.GetPooledObject(); foreach (var value in values) { - pooledHash.Object.AppendData(MemoryMarshal.AsBytes(value.AsSpan())); - pooledHash.Object.AppendData(MemoryMarshal.AsBytes("\0".AsSpan())); + pooledHash.Object.Append(MemoryMarshal.AsBytes(value.AsSpan())); + pooledHash.Object.Append(MemoryMarshal.AsBytes("\0".AsSpan())); } - Span hash = stackalloc byte[SHA256HashSizeBytes]; + Span hash = stackalloc byte[XXHash128SizeBytes]; pooledHash.Object.GetHashAndReset(hash); return From(hash); -#else - using var pooledHash = s_incrementalHashPool.GetPooledObject(); - using var pooledBuffer = SharedPools.ByteArray.GetPooledObject(); - var hash = pooledHash.Object; - - hash.Initialize(); - foreach (var value in values) - { - AppendData(hash, pooledBuffer.Object, value); - AppendData(hash, pooledBuffer.Object, "\0"); - } - - hash.TransformFinalBlock(Array.Empty(), 0, 0); - return From(hash.Hash); -#endif +//#else +// using var pooledHash = s_incrementalHashPool.GetPooledObject(); +// using var pooledBuffer = SharedPools.ByteArray.GetPooledObject(); +// var hash = pooledHash.Object; + +// hash.Reset(); +// foreach (var value in values) +// { +// AppendData(hash, pooledBuffer.Object, value); +// AppendData(hash, pooledBuffer.Object, "\0"); +// } + +// hash.TransformFinalBlock(Array.Empty(), 0, 0); +// return From(hash.Hash); +//#endif } public static Checksum Create(string value) { -#if NET - Span hash = stackalloc byte[SHA256HashSizeBytes]; - SHA256.HashData(MemoryMarshal.AsBytes(value.AsSpan()), hash); - return From(hash); -#else - using var pooledHash = s_incrementalHashPool.GetPooledObject(); - using var pooledBuffer = SharedPools.ByteArray.GetPooledObject(); - var hash = pooledHash.Object; - hash.Initialize(); - - AppendData(hash, pooledBuffer.Object, value); - - hash.TransformFinalBlock(Array.Empty(), 0, 0); - return From(hash.Hash); -#endif +// #if NET + Span destination = stackalloc byte[XXHash128SizeBytes]; + Contract.ThrowIfFalse(XxHash128.TryHash(MemoryMarshal.AsBytes(value.AsSpan()), destination, out _)); + return From(destination); +//#else +// using var pooledHash = s_incrementalHashPool.GetPooledObject(); +// using var pooledBuffer = SharedPools.ByteArray.GetPooledObject(); +// var hash = pooledHash.Object; +// hash.Reset(); + +// AppendData(hash, pooledBuffer.Object, value); +// value.assp +// hash.TransformFinalBlock(Array.Empty(), 0, 0); +// return From(hash.Hash); +//#endif } public static Checksum Create(Stream stream) @@ -107,12 +102,12 @@ public static Checksum Create(Stream stream) bytesRead = stream.Read(buffer); if (bytesRead > 0) { - pooledHash.Object.AppendData(buffer[..bytesRead]); + pooledHash.Object.Append(buffer[..bytesRead]); } } while (bytesRead > 0); - Span hash = stackalloc byte[SHA256HashSizeBytes]; + Span hash = stackalloc byte[XXHash128SizeBytes]; pooledHash.Object.GetHashAndReset(hash); return From(hash); #else @@ -192,42 +187,42 @@ public static Checksum Create(ReadOnlySpan hashes) #endif } -#if !NET +//#if !NET - private static PooledObject GetPooledByteArray(int checksumCount) - { - var objectPool = s_checksumByteArrayPool[checksumCount]; - return objectPool.GetPooledObject(); - } +// private static PooledObject GetPooledByteArray(int checksumCount) +// { +// var objectPool = s_checksumByteArrayPool[checksumCount]; +// return objectPool.GetPooledObject(); +// } - private static Checksum CreateUsingByteArrays(ReadOnlySpan checksums) - { - using var bytes = GetPooledByteArray(checksumCount: checksums.Length); +// private static Checksum CreateUsingByteArrays(ReadOnlySpan checksums) +// { +// using var bytes = GetPooledByteArray(checksumCount: checksums.Length); - var bytesSpan = bytes.Object.AsSpan(); - var index = 0; - foreach (var checksum in checksums) - { - checksum.WriteTo(bytesSpan.Slice(HashSize * index)); - index++; - } +// var bytesSpan = bytes.Object.AsSpan(); +// var index = 0; +// foreach (var checksum in checksums) +// { +// checksum.WriteTo(bytesSpan.Slice(HashSize * index)); +// index++; +// } - using var hash = s_incrementalHashPool.GetPooledObject(); - hash.Object.Initialize(); +// using var hash = s_incrementalHashPool.GetPooledObject(); +// hash.Object.Initialize(); - hash.Object.TransformBlock(bytes.Object, 0, bytes.Object.Length, null, 0); +// hash.Object.TransformBlock(bytes.Object, 0, bytes.Object.Length, null, 0); - hash.Object.TransformFinalBlock(Array.Empty(), 0, 0); - return From(hash.Object.Hash); - } +// hash.Object.TransformFinalBlock(Array.Empty(), 0, 0); +// return From(hash.Object.Hash); +// } - private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2) - => CreateUsingByteArrays(stackalloc[] { checksum1.Hash, checksum2.Hash }); +// private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2) +// => CreateUsingByteArrays(stackalloc[] { checksum1.Hash, checksum2.Hash }); - private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2, Checksum checksum3) - => CreateUsingByteArrays(stackalloc[] { checksum1.Hash, checksum2.Hash, checksum3.Hash }); +// private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2, Checksum checksum3) +// => CreateUsingByteArrays(stackalloc[] { checksum1.Hash, checksum2.Hash, checksum3.Hash }); -#else +//#else // Optimized helpers that do not need to allocate any arrays to combine hashes. @@ -240,11 +235,11 @@ private static Checksum CreateUsingSpans(Checksum checksum1, Checksum checksum2, private static Checksum CreateUsingSpans( ReadOnlySpan hashes) { - Span hashResultSpan = stackalloc byte[SHA256HashSizeBytes]; + Span destination = stackalloc byte[XXHash128SizeBytes]; - SHA256.HashData(MemoryMarshal.AsBytes(hashes), hashResultSpan); + Contract.ThrowIfFalse(XxHash128.TryHash(MemoryMarshal.AsBytes(hashes), destination, out _)); - return From(hashResultSpan); + return From(destination); } #endif @@ -319,7 +314,7 @@ public static Checksum Create(ParseOptions value, ISerializerService serializer) } #if !NET - private static void AppendData(SHA256 hash, byte[] buffer, string value) + private static void AppendData(XxHash128 hash, byte[] buffer, string value) { var stringBytes = MemoryMarshal.AsBytes(value.AsSpan()); Debug.Assert(stringBytes.Length == value.Length * 2); From e20234eb4c5487239952d8c46089f41d4cab0ad3 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Tue, 7 Nov 2023 14:40:11 -0800 Subject: [PATCH 2/9] Swithc entirely over --- .../Workspace/Solution/Checksum_Factory.cs | 186 +----------------- src/Workspaces/CoreTest/ChecksumTests.cs | 6 +- 2 files changed, 6 insertions(+), 186 deletions(-) diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs index 03c467242d73..38bec8acee74 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs @@ -5,10 +5,8 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; using System.IO; using System.IO.Hashing; -using System.Linq; using System.Runtime.InteropServices; using System.Threading; using Microsoft.CodeAnalysis.PooledObjects; @@ -26,19 +24,8 @@ internal partial record class Checksum private static readonly ObjectPool s_incrementalHashPool = new(() => new(), size: 20); -#if !NET - // Dedicated pools for the byte[]s we use to create checksums from two or three existing checksums. Sized to - // exactly the space needed to splat the existing checksum data into the array and then hash it. - - // Note: number of elements here should be computed based on what we need from our various collection-with-children objects. - private static readonly ObjectPool[] s_checksumByteArrayPool = - Enumerable.Range(0, 11).Select(i => new ObjectPool(() => new byte[HashSize * i])).ToArray(); - -#endif - public static Checksum Create(IEnumerable values) { -// #if NET using var pooledHash = s_incrementalHashPool.GetPooledObject(); foreach (var value in values) @@ -50,101 +37,23 @@ public static Checksum Create(IEnumerable values) Span hash = stackalloc byte[XXHash128SizeBytes]; pooledHash.Object.GetHashAndReset(hash); return From(hash); -//#else -// using var pooledHash = s_incrementalHashPool.GetPooledObject(); -// using var pooledBuffer = SharedPools.ByteArray.GetPooledObject(); -// var hash = pooledHash.Object; - -// hash.Reset(); -// foreach (var value in values) -// { -// AppendData(hash, pooledBuffer.Object, value); -// AppendData(hash, pooledBuffer.Object, "\0"); -// } - -// hash.TransformFinalBlock(Array.Empty(), 0, 0); -// return From(hash.Hash); -//#endif } public static Checksum Create(string value) { -// #if NET Span destination = stackalloc byte[XXHash128SizeBytes]; Contract.ThrowIfFalse(XxHash128.TryHash(MemoryMarshal.AsBytes(value.AsSpan()), destination, out _)); return From(destination); -//#else -// using var pooledHash = s_incrementalHashPool.GetPooledObject(); -// using var pooledBuffer = SharedPools.ByteArray.GetPooledObject(); -// var hash = pooledHash.Object; -// hash.Reset(); - -// AppendData(hash, pooledBuffer.Object, value); -// value.assp -// hash.TransformFinalBlock(Array.Empty(), 0, 0); -// return From(hash.Hash); -//#endif } public static Checksum Create(Stream stream) { -#if NET7_0_OR_GREATER - Span hash = stackalloc byte[SHA256HashSizeBytes]; - SHA256.HashData(stream, hash); - return From(hash); -#elif NET using var pooledHash = s_incrementalHashPool.GetPooledObject(); - Span buffer = stackalloc byte[SharedPools.ByteBufferSize]; - - int bytesRead; - do - { - bytesRead = stream.Read(buffer); - if (bytesRead > 0) - { - pooledHash.Object.Append(buffer[..bytesRead]); - } - } - while (bytesRead > 0); + pooledHash.Object.Append(stream); Span hash = stackalloc byte[XXHash128SizeBytes]; pooledHash.Object.GetHashAndReset(hash); return From(hash); -#else - using var pooledHash = s_incrementalHashPool.GetPooledObject(); - using var pooledBuffer = SharedPools.ByteArray.GetPooledObject(); - - var hash = pooledHash.Object; - hash.Initialize(); - - var buffer = pooledBuffer.Object; - var bufferLength = buffer.Length; - int bytesRead; - do - { - bytesRead = stream.Read(buffer, 0, bufferLength); - if (bytesRead > 0) - { - hash.TransformBlock(buffer, 0, bytesRead, null, 0); - } - } - while (bytesRead > 0); - - hash.TransformFinalBlock(Array.Empty(), 0, 0); - var bytes = hash.Hash; - - // if bytes array is bigger than certain size, checksum - // will truncate it to predetermined size. for more detail, - // see the Checksum type - // - // the truncation can happen since different hash algorithm or - // same algorithm on different platform can have different hash size - // which might be bigger than the Checksum HashSize. - // - // hash algorithm used here should remain functionally correct even - // after the truncation - return From(bytes); -#endif } public static Checksum Create(T @object, Action writeObject) @@ -161,80 +70,13 @@ public static Checksum Create(T @object, Action writeObject) } public static Checksum Create(Checksum checksum1, Checksum checksum2) - { -#if NET - return CreateUsingSpans(checksum1, checksum2); -#else - return CreateUsingByteArrays(checksum1, checksum2); -#endif - } + => Create(stackalloc[] { checksum1.Hash, checksum2.Hash }); public static Checksum Create(Checksum checksum1, Checksum checksum2, Checksum checksum3) - { -#if NET - return CreateUsingSpans(checksum1, checksum2, checksum3); -#else - return CreateUsingByteArrays(checksum1, checksum2, checksum3); -#endif - } + => Create(stackalloc[] { checksum1.Hash, checksum2.Hash, checksum3.Hash }); public static Checksum Create(ReadOnlySpan hashes) { -#if NET - return CreateUsingSpans(hashes); -#else - return CreateUsingByteArrays(hashes); -#endif - } - -//#if !NET - -// private static PooledObject GetPooledByteArray(int checksumCount) -// { -// var objectPool = s_checksumByteArrayPool[checksumCount]; -// return objectPool.GetPooledObject(); -// } - -// private static Checksum CreateUsingByteArrays(ReadOnlySpan checksums) -// { -// using var bytes = GetPooledByteArray(checksumCount: checksums.Length); - -// var bytesSpan = bytes.Object.AsSpan(); -// var index = 0; -// foreach (var checksum in checksums) -// { -// checksum.WriteTo(bytesSpan.Slice(HashSize * index)); -// index++; -// } - -// using var hash = s_incrementalHashPool.GetPooledObject(); -// hash.Object.Initialize(); - -// hash.Object.TransformBlock(bytes.Object, 0, bytes.Object.Length, null, 0); - -// hash.Object.TransformFinalBlock(Array.Empty(), 0, 0); -// return From(hash.Object.Hash); -// } - -// private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2) -// => CreateUsingByteArrays(stackalloc[] { checksum1.Hash, checksum2.Hash }); - -// private static Checksum CreateUsingByteArrays(Checksum checksum1, Checksum checksum2, Checksum checksum3) -// => CreateUsingByteArrays(stackalloc[] { checksum1.Hash, checksum2.Hash, checksum3.Hash }); - -//#else - - // Optimized helpers that do not need to allocate any arrays to combine hashes. - - private static Checksum CreateUsingSpans(Checksum checksum1, Checksum checksum2) - => CreateUsingSpans(stackalloc[] { checksum1.Hash, checksum2.Hash }); - - private static Checksum CreateUsingSpans(Checksum checksum1, Checksum checksum2, Checksum checksum3) - => CreateUsingSpans(stackalloc[] { checksum1.Hash, checksum2.Hash, checksum3.Hash }); - - private static Checksum CreateUsingSpans( - ReadOnlySpan hashes) - { Span destination = stackalloc byte[XXHash128SizeBytes]; Contract.ThrowIfFalse(XxHash128.TryHash(MemoryMarshal.AsBytes(hashes), destination, out _)); @@ -242,8 +84,6 @@ private static Checksum CreateUsingSpans( return From(destination); } -#endif - public static Checksum Create(ArrayBuilder checksums) { using var stream = SerializableBytes.CreateWritableStream(); @@ -312,25 +152,5 @@ public static Checksum Create(ParseOptions value, ISerializerService serializer) stream.Position = 0; return Create(stream); } - -#if !NET - private static void AppendData(XxHash128 hash, byte[] buffer, string value) - { - var stringBytes = MemoryMarshal.AsBytes(value.AsSpan()); - Debug.Assert(stringBytes.Length == value.Length * 2); - - var index = 0; - while (index < stringBytes.Length) - { - var remaining = stringBytes.Length - index; - var toCopy = Math.Min(remaining, buffer.Length); - - stringBytes.Slice(index, toCopy).CopyTo(buffer); - hash.TransformBlock(buffer, 0, toCopy, null, 0); - - index += toCopy; - } - } -#endif } } diff --git a/src/Workspaces/CoreTest/ChecksumTests.cs b/src/Workspaces/CoreTest/ChecksumTests.cs index f81ec447f72a..8ada730b6bb0 100644 --- a/src/Workspaces/CoreTest/ChecksumTests.cs +++ b/src/Workspaces/CoreTest/ChecksumTests.cs @@ -19,7 +19,7 @@ public void ValidateChecksumFromSpanSameAsChecksumFromBytes1() var checksumA = Checksum.Create(checksum1, checksum2); // Running this test on multiple target frameworks with the same expectation ensures the results match - Assert.Equal(Checksum.FromBase64String("N30m5jwVeMZzWpy9cbQbtSYHoXU="), checksumA); + Assert.Equal(Checksum.FromBase64String("RRbspG+E4ziBC5hOWyrfCQ=="), checksumA); Assert.NotEqual(checksum1, checksum2); Assert.NotEqual(checksum1, checksumA); @@ -36,7 +36,7 @@ public void ValidateChecksumFromSpanSameAsChecksumFromBytes2() var checksumA = Checksum.Create(checksum1, checksum2, checksum3); // Running this test on multiple target frameworks with the same expectation ensures the results match - Assert.Equal(Checksum.FromBase64String("NEfIznmqkIqi4VJl12KxycWt7uo="), checksumA); + Assert.Equal(Checksum.FromBase64String("DvHp7gyz/hBKY1/Q7A1NCg=="), checksumA); Assert.NotEqual(checksum1, checksum2); Assert.NotEqual(checksum2, checksum3); @@ -56,7 +56,7 @@ public void ValidateChecksumFromSpanSameAsChecksumFromBytes10() var checksumA = Checksum.Create(checksums.Select(c => c.Hash).ToArray().AsSpan()); // Running this test on multiple target frameworks with the same expectation ensures the results match - Assert.Equal(Checksum.FromBase64String("yOnAG9SVuhK/+wCM/WlpTl5e7h8="), checksumA); + Assert.Equal(Checksum.FromBase64String("umt6tOdNsvIArs4OY7MFCg=="), checksumA); for (var i = 0; i < max; i++) { From 1eb8cfaabf0d0e29b04e10181df4b345641cffe6 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Tue, 7 Nov 2023 15:19:36 -0800 Subject: [PATCH 3/9] Simplify --- .../Core/Portable/Workspace/Solution/Checksum_Factory.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs index 38bec8acee74..246c1a5bd0bd 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs @@ -42,7 +42,7 @@ public static Checksum Create(IEnumerable values) public static Checksum Create(string value) { Span destination = stackalloc byte[XXHash128SizeBytes]; - Contract.ThrowIfFalse(XxHash128.TryHash(MemoryMarshal.AsBytes(value.AsSpan()), destination, out _)); + XxHash128.Hash(MemoryMarshal.AsBytes(value.AsSpan()), destination); return From(destination); } @@ -78,9 +78,7 @@ public static Checksum Create(Checksum checksum1, Checksum checksum2, Checksum c public static Checksum Create(ReadOnlySpan hashes) { Span destination = stackalloc byte[XXHash128SizeBytes]; - - Contract.ThrowIfFalse(XxHash128.TryHash(MemoryMarshal.AsBytes(hashes), destination, out _)); - + XxHash128.Hash(MemoryMarshal.AsBytes(hashes), destination); return From(destination); } From eb0bcfcd28fdc056ef944b97a7f64b8ecfaca72e Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Tue, 7 Nov 2023 15:40:29 -0800 Subject: [PATCH 4/9] Update file --- src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj b/src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj index dde68248f848..92801965a73c 100644 --- a/src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj +++ b/src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj @@ -125,6 +125,7 @@ <_Dependency Remove="System.Collections.Immutable"/> <_Dependency Remove="System.Diagnostics.DiagnosticSource"/> <_Dependency Remove="System.Drawing.Common"/> + <_Dependency Remove="System.IO.Hashing"/> <_Dependency Remove="System.IO.Pipelines"/> <_Dependency Remove="System.Memory"/> <_Dependency Remove="System.Numerics.Vectors"/> From d3875e0580be54e7e0991ae86aaa3a1176ef4165 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 8 Nov 2023 09:02:52 -0800 Subject: [PATCH 5/9] remove --- src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs index 8e5cc5ddff60..f2ab91eeece1 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum.cs @@ -13,7 +13,6 @@ using System.Runtime.Serialization; using Microsoft.CodeAnalysis.Serialization; using Roslyn.Utilities; -using System.IO.Hashing; namespace Microsoft.CodeAnalysis { From def56fc6689d878dfe3321d41cbd4ca62ecba6a8 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 8 Nov 2023 09:13:54 -0800 Subject: [PATCH 6/9] Add test --- src/Workspaces/CoreTest/ChecksumTests.cs | 100 +++++++++++++---------- 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/src/Workspaces/CoreTest/ChecksumTests.cs b/src/Workspaces/CoreTest/ChecksumTests.cs index 8ada730b6bb0..b9d9b7e8d837 100644 --- a/src/Workspaces/CoreTest/ChecksumTests.cs +++ b/src/Workspaces/CoreTest/ChecksumTests.cs @@ -6,67 +6,77 @@ using System.Linq; using Xunit; -namespace Microsoft.CodeAnalysis.UnitTests +namespace Microsoft.CodeAnalysis.UnitTests; + +public class ChecksumTests { - public class ChecksumTests + [Fact] + public void ValidateChecksumFromSpanSameAsChecksumFromBytes1() { - [Fact] - public void ValidateChecksumFromSpanSameAsChecksumFromBytes1() - { - var checksum1 = Checksum.Create("Goo"); - var checksum2 = Checksum.Create("Bar"); + var checksum1 = Checksum.Create("Goo"); + var checksum2 = Checksum.Create("Bar"); - var checksumA = Checksum.Create(checksum1, checksum2); + var checksumA = Checksum.Create(checksum1, checksum2); - // Running this test on multiple target frameworks with the same expectation ensures the results match - Assert.Equal(Checksum.FromBase64String("RRbspG+E4ziBC5hOWyrfCQ=="), checksumA); + // Running this test on multiple target frameworks with the same expectation ensures the results match + Assert.Equal(Checksum.FromBase64String("RRbspG+E4ziBC5hOWyrfCQ=="), checksumA); - Assert.NotEqual(checksum1, checksum2); - Assert.NotEqual(checksum1, checksumA); - Assert.NotEqual(checksum2, checksumA); - } + Assert.NotEqual(checksum1, checksum2); + Assert.NotEqual(checksum1, checksumA); + Assert.NotEqual(checksum2, checksumA); + } - [Fact] - public void ValidateChecksumFromSpanSameAsChecksumFromBytes2() - { - var checksum1 = Checksum.Create("Goo"); - var checksum2 = Checksum.Create("Bar"); - var checksum3 = Checksum.Create("Baz"); + [Fact] + public void ValidateChecksumFromSpanSameAsChecksumFromBytes2() + { + var checksum1 = Checksum.Create("Goo"); + var checksum2 = Checksum.Create("Bar"); + var checksum3 = Checksum.Create("Baz"); - var checksumA = Checksum.Create(checksum1, checksum2, checksum3); + var checksumA = Checksum.Create(checksum1, checksum2, checksum3); - // Running this test on multiple target frameworks with the same expectation ensures the results match - Assert.Equal(Checksum.FromBase64String("DvHp7gyz/hBKY1/Q7A1NCg=="), checksumA); + // Running this test on multiple target frameworks with the same expectation ensures the results match + Assert.Equal(Checksum.FromBase64String("DvHp7gyz/hBKY1/Q7A1NCg=="), checksumA); - Assert.NotEqual(checksum1, checksum2); - Assert.NotEqual(checksum2, checksum3); - Assert.NotEqual(checksum3, checksum1); + Assert.NotEqual(checksum1, checksum2); + Assert.NotEqual(checksum2, checksum3); + Assert.NotEqual(checksum3, checksum1); - Assert.NotEqual(checksum1, checksumA); - Assert.NotEqual(checksum2, checksumA); - Assert.NotEqual(checksum3, checksumA); - } + Assert.NotEqual(checksum1, checksumA); + Assert.NotEqual(checksum2, checksumA); + Assert.NotEqual(checksum3, checksumA); + } - [Fact] - public void ValidateChecksumFromSpanSameAsChecksumFromBytes10() - { - const int max = 10; - var checksums = Enumerable.Range(0, max).Select(i => Checksum.Create($"{i}")).ToArray(); + [Fact] + public void ValidateChecksumFromSpanSameAsChecksumFromBytes10() + { + const int max = 10; + var checksums = Enumerable.Range(0, max).Select(i => Checksum.Create($"{i}")).ToArray(); - var checksumA = Checksum.Create(checksums.Select(c => c.Hash).ToArray().AsSpan()); + var checksumA = Checksum.Create(checksums.Select(c => c.Hash).ToArray().AsSpan()); - // Running this test on multiple target frameworks with the same expectation ensures the results match - Assert.Equal(Checksum.FromBase64String("umt6tOdNsvIArs4OY7MFCg=="), checksumA); + // Running this test on multiple target frameworks with the same expectation ensures the results match + Assert.Equal(Checksum.FromBase64String("umt6tOdNsvIArs4OY7MFCg=="), checksumA); - for (var i = 0; i < max; i++) + for (var i = 0; i < max; i++) + { + for (var j = 0; j < max; j++) { - for (var j = 0; j < max; j++) - { - Assert.True((i == j) == (checksums[i] == checksums[j])); - } - - Assert.NotEqual(checksums[i], checksumA); + Assert.True((i == j) == (checksums[i] == checksums[j])); } + + Assert.NotEqual(checksums[i], checksumA); } } + + [Fact] + public void StringArraysProduceDifferentResultsThanConcatenation() + { + var checksum1 = Checksum.Create(["goo", "bar"]); + var checksum2 = Checksum.Create(["go", "obar"]); + var checksum3 = Checksum.Create("goobar"); + Assert.NotEqual(checksum1, checksum2); + Assert.NotEqual(checksum2, checksum3); + Assert.NotEqual(checksum3, checksum1); + } } From 2f5f6ac4193b1a1829ae5d5f870e245cea1614c0 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 8 Nov 2023 11:33:06 -0800 Subject: [PATCH 7/9] Update project --- .../Roslyn.VisualStudio.Setup.Dependencies.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/VisualStudio/Setup.Dependencies/Roslyn.VisualStudio.Setup.Dependencies.csproj b/src/VisualStudio/Setup.Dependencies/Roslyn.VisualStudio.Setup.Dependencies.csproj index 45475844c1cc..46eca982c9a7 100644 --- a/src/VisualStudio/Setup.Dependencies/Roslyn.VisualStudio.Setup.Dependencies.csproj +++ b/src/VisualStudio/Setup.Dependencies/Roslyn.VisualStudio.Setup.Dependencies.csproj @@ -32,6 +32,7 @@ + @@ -75,6 +76,7 @@ + From 9639beda79b1257acf19abdad855097463008df3 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Tue, 14 Nov 2023 08:33:06 -0800 Subject: [PATCH 8/9] Update eng/Versions.props --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 4f79481ec965..96a6fc644a52 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -245,7 +245,7 @@ 7.0.0 4.3.0 4.3.0 - 8.0.0-rc.2.23479.6 + 8.0.0 5.0.0 7.0.0 7.0.0 From 2c1117dd0b7e59ac3a20356a6bf7b404ea2a1518 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Tue, 14 Nov 2023 12:26:10 -0800 Subject: [PATCH 9/9] Remove comment --- .../Core/Portable/Workspace/Solution/Checksum_Factory.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs index 246c1a5bd0bd..119216a81e09 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/Checksum_Factory.cs @@ -18,7 +18,6 @@ namespace Microsoft.CodeAnalysis // various factory methods. all these are just helper methods internal partial record class Checksum { - // https://github.com/dotnet/runtime/blob/f2db6d6093c54e5eeb9db2d8dcbe15b2db92ad8c/src/libraries/System.Security.Cryptography.Algorithms/src/System/Security/Cryptography/SHA256.cs#L18-L19 private const int XXHash128SizeBytes = 128 / 8; private static readonly ObjectPool s_incrementalHashPool =