From 8d9115a63427425c71831045a2a55bf126dd43e5 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Wed, 19 Jul 2023 18:20:53 -0700 Subject: [PATCH 01/13] factored out CastCache to be able create several. --- .../Runtime/CompilerServices/CastHelpers.cs | 18 ++-- .../src/System/Runtime/TypeCast.cs | 7 +- .../Runtime/CompilerServices/CastCache.cs | 6 +- .../Runtime/CompilerServices/CastCache.cs | 89 ++++++++++--------- 4 files changed, 68 insertions(+), 52 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 435782b8b754d8..a18879bddbfaaa 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -9,8 +9,16 @@ namespace System.Runtime.CompilerServices { + internal static class CastCache + { + // In coreclr the table is allocated and written to on the native side. + internal static int[]? s_table; + } + internal static unsafe class CastHelpers { + private static CastCacheImpl CastCacheInstance => new CastCacheImpl(CastCache.s_table!); + [MethodImpl(MethodImplOptions.InternalCall)] private static extern object IsInstanceOfAny_NoCacheLookup(void* toTypeHnd, object obj); @@ -36,7 +44,7 @@ internal static unsafe class CastHelpers void* mt = RuntimeHelpers.GetMethodTable(obj); if (mt != toTypeHnd) { - CastResult result = CastCache.TryGet((nuint)mt, (nuint)toTypeHnd); + CastResult result = CastCacheInstance.TryGet((nuint)mt, (nuint)toTypeHnd); if (result == CastResult.CanCast) { // do nothing @@ -186,7 +194,7 @@ internal static unsafe class CastHelpers [MethodImpl(MethodImplOptions.NoInlining)] private static object? IsInstance_Helper(void* toTypeHnd, object obj) { - CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); + CastResult result = CastCacheInstance.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); if (result == CastResult.CanCast) { return obj; @@ -215,7 +223,7 @@ internal static unsafe class CastHelpers void* mt = RuntimeHelpers.GetMethodTable(obj); if (mt != toTypeHnd) { - result = CastCache.TryGet((nuint)mt, (nuint)toTypeHnd); + result = CastCacheInstance.TryGet((nuint)mt, (nuint)toTypeHnd); if (result != CastResult.CanCast) { goto slowPath; @@ -239,7 +247,7 @@ internal static unsafe class CastHelpers [MethodImpl(MethodImplOptions.NoInlining)] private static object? ChkCast_Helper(void* toTypeHnd, object obj) { - CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); + CastResult result = CastCacheInstance.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); if (result == CastResult.CanCast) { return obj; @@ -456,7 +464,7 @@ private static void StelemRef(Array array, nint index, object? obj) [MethodImpl(MethodImplOptions.NoInlining)] private static void StelemRef_Helper(ref object? element, void* elementType, object obj) { - CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType); + CastResult result = CastCacheInstance.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType); if (result == CastResult.CanCast) { WriteBarrier(ref element, obj); diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs index 3cd06a108b2bb2..2c850604ce9977 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs @@ -21,8 +21,11 @@ namespace System.Runtime // ///////////////////////////////////////////////////////////////////////////////////////////////////// + [EagerStaticClassConstruction] internal static class TypeCast { + private static CastCacheImpl s_castCache = new CastCacheImpl(); + [Flags] internal enum AssignmentVariation { @@ -1159,7 +1162,7 @@ public static unsafe bool AreTypesAssignableInternal(MethodTable* pSourceType, M return true; nuint sourceAndVariation = (nuint)pSourceType + (uint)variation; - CastResult result = CastCache.TryGet(sourceAndVariation, (nuint)(pTargetType)); + CastResult result = s_castCache.TryGet(sourceAndVariation, (nuint)(pTargetType)); if (result != CastResult.MaybeCast) { return result == CastResult.CanCast; @@ -1187,7 +1190,7 @@ private static unsafe bool CacheMiss(MethodTable* pSourceType, MethodTable* pTar // Update the cache // nuint sourceAndVariation = (nuint)pSourceType + (uint)variation; - CastCache.TrySet(sourceAndVariation, (nuint)pTargetType, result); + s_castCache.TrySet(sourceAndVariation, (nuint)pTargetType, result); return result; } diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index 58de4e50ea2bd6..d5a93bf41cc793 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -8,14 +8,14 @@ internal enum CastResult } // trivial implementation of the cast cache - internal static unsafe class CastCache + internal unsafe struct CastCacheImpl { - internal static CastResult TryGet(nuint source, nuint target) + internal CastResult TryGet(nuint source, nuint target) { return CastResult.MaybeCast; } - internal static void TrySet(nuint source, nuint target, bool result) + internal void TrySet(nuint source, nuint target, bool result) { } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index a333b710d012c1..43c0613554177c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -15,19 +15,9 @@ internal enum CastResult MaybeCast = 2 } -#if NATIVEAOT - [EagerStaticClassConstruction] -#endif - internal static unsafe class CastCache + internal unsafe struct CastCacheImpl { - -#if CORECLR - // In coreclr the table is written to only on the native side. T - // This is all we need to implement TryGet. - private static int[]? s_table; -#else - - #if DEBUG +#if DEBUG private const int INITIAL_CACHE_SIZE = 8; // MUST BE A POWER OF TWO private const int MAXIMUM_CACHE_SIZE = 512; // make this lower than release to make it easier to reach this in tests. #else @@ -37,25 +27,39 @@ internal static unsafe class CastCache private const int VERSION_NUM_SIZE = 29; private const uint VERSION_NUM_MASK = (1 << VERSION_NUM_SIZE) - 1; + private const int BUCKET_SIZE = 8; - // A trivial 2-elements table used for "flushing" the cache. Nothing is ever stored in this table. - // It is required that we are able to allocate this. - private static int[] s_sentinelTable = CreateCastCache(2, throwOnFail: true)!; - - // when flushing, remember the last size. - private static int s_lastFlushSize = INITIAL_CACHE_SIZE; + // nothing is ever stored into this, so we can use a static instance. + private static int[]? s_sentinelTable; // The actual storage. - // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. - private static int[] s_table = - #if !DEBUG - CreateCastCache(INITIAL_CACHE_SIZE) ?? - #endif - s_sentinelTable; + private int[] _table; -#endif // CORECLR + // when flushing, remember the last size. + private int _lastFlushSize; - private const int BUCKET_SIZE = 8; + // wraps existing table + public CastCacheImpl(int[] table) + { + _table = table; + } + + // creates a new cache instance + public CastCacheImpl() + { + // A trivial 2-elements table used for "flushing" the cache. + // Nothing is ever stored in such a small table and identity of the sentinel is not important. + // It is required that we are able to allocate this, we may need this in OOM cases. + s_sentinelTable ??= CreateCastCache(2, throwOnFail: true); + + _table = +#if !DEBUG + // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. + CreateCastCache(INITIAL_CACHE_SIZE) ?? +#endif + s_sentinelTable!; + _lastFlushSize = INITIAL_CACHE_SIZE; + } [StructLayout(LayoutKind.Sequential)] private struct CastCacheEntry @@ -139,10 +143,10 @@ private static ref CastCacheEntry Element(ref int tableData, int index) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static CastResult TryGet(nuint source, nuint target) + internal CastResult TryGet(nuint source, nuint target) { // table is always initialized and is not null. - ref int tableData = ref TableData(s_table!); + ref int tableData = ref TableData(_table!); int index = KeyToBucket(ref tableData, source, target); for (int i = 0; i < BUCKET_SIZE;) @@ -159,7 +163,7 @@ internal static CastResult TryGet(nuint source, nuint target) if (entrySource == source) { - // in CoreCLR we do ordinary reads of the entry parts and + // we do ordinary reads of the entry parts and // Interlocked.ReadMemoryBarrier() before reading the version nuint entryTargetAndResult = pEntry._targetAndResult; // target never has its lower bit set. @@ -204,7 +208,6 @@ internal static CastResult TryGet(nuint source, nuint target) // in CoreClr the cache is only updated in the native code // // The following helpers must match native implementations in castcache.h and castcache.cpp -#if !CORECLR // we generally do not OOM in casts, just return null unless throwOnFail is specified. private static int[]? CreateCastCache(int size, bool throwOnFail = false) @@ -252,20 +255,20 @@ internal static CastResult TryGet(nuint source, nuint target) return table; } - internal static void TrySet(nuint source, nuint target, bool result) + internal void TrySet(nuint source, nuint target, bool result) { int bucket; ref int tableData = ref *(int*)0; do { - tableData = ref TableData(s_table); + tableData = ref TableData(_table); if (TableMask(ref tableData) == 1) { // 2-element table is used as a sentinel. // we did not allocate a real table yet or have flushed it. // try replacing the table, but do not insert anything. - MaybeReplaceCacheWithLarger(s_lastFlushSize); + MaybeReplaceCacheWithLarger(_lastFlushSize); return; } @@ -333,7 +336,7 @@ internal static void TrySet(nuint source, nuint target, bool result) } while (TryGrow(ref tableData)); // reread tableData after TryGrow. - tableData = ref TableData(s_table); + tableData = ref TableData(_table); if (TableMask(ref tableData) == 1) { @@ -381,19 +384,22 @@ private static int CacheElementCount(ref int tableData) return TableMask(ref tableData) + 1; } - private static void FlushCurrentCache() + private void FlushCurrentCache() { - ref int tableData = ref TableData(s_table); + ref int tableData = ref TableData(_table); int lastSize = CacheElementCount(ref tableData); if (lastSize < INITIAL_CACHE_SIZE) lastSize = INITIAL_CACHE_SIZE; - s_lastFlushSize = lastSize; + // store the last size to use when creating a new table + // it is just a hint, not needed for correctness, so no synchronization + // with the writing of the table + _lastFlushSize = lastSize; // flushing is just replacing the table with a sentinel. - s_table = s_sentinelTable; + _table = s_sentinelTable!; } - private static bool MaybeReplaceCacheWithLarger(int size) + private bool MaybeReplaceCacheWithLarger(int size) { int[]? newTable = CreateCastCache(size); if (newTable == null) @@ -401,11 +407,11 @@ private static bool MaybeReplaceCacheWithLarger(int size) return false; } - s_table = newTable; + _table = newTable; return true; } - private static bool TryGrow(ref int tableData) + private bool TryGrow(ref int tableData) { int newSize = CacheElementCount(ref tableData) * 2; if (newSize <= MAXIMUM_CACHE_SIZE) @@ -415,6 +421,5 @@ private static bool TryGrow(ref int tableData) return false; } -#endif // !CORECLR } } From 8330e46c3b59f81c6fe6012912c93bc4136deff8 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:37:13 -0700 Subject: [PATCH 02/13] Cache impl --- .../Runtime/CompilerServices/CastCache.cs | 417 ++++++++++++++++++ 1 file changed, 417 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index 43c0613554177c..4fd17225d4796d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -1,7 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections; +using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Numerics; using System.Runtime.InteropServices; using System.Threading; @@ -422,4 +425,418 @@ private bool TryGrow(ref int tableData) return false; } } + + internal unsafe struct Cache + { +#if DEBUG + private const int INITIAL_CACHE_SIZE = 8; // MUST BE A POWER OF TWO + private const int MAXIMUM_CACHE_SIZE = 512; // make this lower than release to make it easier to reach this in tests. +#else + private const int INITIAL_CACHE_SIZE = 128; // MUST BE A POWER OF TWO + private const int MAXIMUM_CACHE_SIZE = 4096; // Same as in CastCache. We will rarely need this much though. +#endif // DEBUG + + private const int VERSION_NUM_SIZE = 29; + private const uint VERSION_NUM_MASK = (1 << VERSION_NUM_SIZE) - 1; + private const int BUCKET_SIZE = 8; + + // nothing is ever stored into this, so we can use a static instance. + private static Entry[]? s_sentinelTable; + + // The actual storage. + private Entry[] _table; + + // when flushing, remember the last size. + private int _lastFlushSize; + + // creates a new cache instance + public Cache() + { + // A trivial 2-elements table used for "flushing" the cache. + // Nothing is ever stored in such a small table and identity of the sentinel is not important. + // It is required that we are able to allocate this, we may need this in OOM cases. + s_sentinelTable ??= CreateCastCache(2, throwOnFail: true); + + _table = +#if !DEBUG + // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. + CreateCastCache(INITIAL_CACHE_SIZE) ?? +#endif + s_sentinelTable!; + _lastFlushSize = INITIAL_CACHE_SIZE; + } + + [StructLayout(LayoutKind.Explicit)] + private struct UnmanagedPart + { + // version has the following structure: + // [ distance:3bit | versionNum:29bit ] + // + // distance is how many iterations the entry is from it ideal position. + // we use that for preemption. + // + // versionNum is a monotonically increasing numerical tag. + // Writer "claims" entry by atomically incrementing the tag. Thus odd number indicates an entry in progress. + // Upon completion of adding an entry the tag is incremented again making it even. Even number indicates a complete entry. + // + // Readers will read the version twice before and after retrieving the entry. + // To have a usable entry both reads must yield the same even version. + // + [FieldOffset(0)] + internal uint _version; + [FieldOffset(sizeof(int))] + internal int _hash; + + // AuxData + [FieldOffset(0)] + internal int tableMask; + [FieldOffset(sizeof(int))] + internal byte hashShift; + [FieldOffset(sizeof(int) + 1)] + internal byte victimCounter; + } + + [StructLayout(LayoutKind.Sequential)] + private struct Entry + { + internal UnmanagedPart _unmanagedPart; + internal TKey _key; + internal TValue _value; + + [UnscopedRef] + public ref uint Version => ref _unmanagedPart._version; + [UnscopedRef] + public ref int Hash => ref _unmanagedPart._hash; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int KeyToBucket(ref Entry tableData, TKey key) + { + int hashShift = HashShift(ref tableData); + int hash = key!.GetHashCode(); +#if TARGET_64BIT + return (int)(((ulong)hash * 11400714819323198485ul) >> hashShift); +#else + return (int)(((uint)hash * 2654435769u) >> hashShift); +#endif + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref Entry TableData(Entry[] table) + { + // element 0 is used for embedded aux data + return ref table[0]; + // return ref Unsafe.As(ref Unsafe.AddByteOffset(ref table.GetRawData(), (nint)sizeof(nint))); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref byte HashShift(ref Entry tableData) + { + return ref tableData._unmanagedPart.hashShift; + } + + // TableMask is "size - 1" + // we need that more often that we need size + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref int TableMask(ref Entry tableData) + { + return ref tableData._unmanagedPart.tableMask; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref byte VictimCounter(ref Entry tableData) + { + return ref tableData._unmanagedPart.victimCounter; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref Entry Element(ref Entry tableData, int index) + { + // element 0 is used for embedded aux data, skip it + return ref Unsafe.Add(ref tableData, index + 1); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGet(TKey key, out TValue? value) + { + // table is always initialized and is not null. + ref Entry tableData = ref TableData(_table!); + int hash = key!.GetHashCode(); + + int index = KeyToBucket(ref tableData, key); + for (int i = 0; i < BUCKET_SIZE;) + { + ref Entry pEntry = ref Element(ref tableData, index); + + // we must read in this order: version -> [entry parts] -> version + // if version is odd or changes, the entry is inconsistent and thus ignored + uint version = Volatile.Read(ref pEntry.Version); + + // mask the lower version bit to make it even. + // This way we can check if version is odd or changing in just one compare. + version &= unchecked((uint)~1); + + if (hash == pEntry.Hash && EqualityComparer.Default.Equals(key, pEntry._key)) + { + // we do ordinary reads of the value and + // Interlocked.ReadMemoryBarrier() before reading the version + value = pEntry._value; + + // make sure the second read of 'version' happens after reading 'source' and 'targetAndResults' + // + // We can either: + // - use acquires for both _source and _targetAndResults or + // - issue a load barrier before reading _version + // benchmarks on available hardware (Jan 2020) show that use of a read barrier is cheaper. + Interlocked.ReadMemoryBarrier(); + + if (version != pEntry.Version) + { + // oh, so close, the entry is in inconsistent state. + // it is either changing or has changed while we were reading. + // treat it as a miss. + break; + } + + return true; + } + + if (version == 0) + { + // the rest of the bucket is unclaimed, no point to search further + break; + } + + // quadratic reprobe + i++; + index = (index + i) & TableMask(ref tableData); + } + + value = default; + return false; + } + + // the rest is the support for updating the cache. + // in CoreClr the cache is only updated in the native code + // + // The following helpers must match native implementations in castcache.h and castcache.cpp + + // we generally do not OOM in casts, just return null unless throwOnFail is specified. + private static Entry[]? CreateCastCache(int size, bool throwOnFail = false) + { + // size must be positive + Debug.Assert(size > 1); + // size must be a power of two + Debug.Assert((size & (size - 1)) == 0); + + Entry[]? table = null; + try + { + table = new Entry[size + 1]; + } + catch (OutOfMemoryException) when (!throwOnFail) + { + } + + if (table == null) + { + size = INITIAL_CACHE_SIZE; + try + { + table = new Entry[size + 1]; + } + catch (OutOfMemoryException) + { + } + } + + if (table == null) + { + return table; + } + + ref Entry tableData = ref TableData(table); + + // set the table mask. we need it often, do not want to compute each time. + TableMask(ref tableData) = size - 1; + + // Fibonacci hash reduces the value into desired range by shifting right by the number of leading zeroes in 'size-1' + byte shift = (byte)BitOperations.LeadingZeroCount(size - 1); + HashShift(ref tableData) = shift; + + return table; + } + + internal void TrySet(TKey key, TValue value) + { + int bucket; + int hash = key!.GetHashCode(); + ref Entry tableData = ref Unsafe.NullRef(); + + do + { + tableData = ref TableData(_table); + if (TableMask(ref tableData) == 1) + { + // 2-element table is used as a sentinel. + // we did not allocate a real table yet or have flushed it. + // try replacing the table, but do not insert anything. + MaybeReplaceCacheWithLarger(_lastFlushSize); + return; + } + + bucket = KeyToBucket(ref tableData, key); + int index = bucket; + ref Entry pEntry = ref Element(ref tableData, index); + + for (int i = 0; i < BUCKET_SIZE;) + { + // claim the entry if unused or is more distant than us from its origin. + // Note - someone familiar with Robin Hood hashing will notice that + // we do the opposite - we are "robbing the poor". + // Robin Hood strategy improves average lookup in a lossles dictionary by reducing + // outliers via giving preference to more distant entries. + // What we have here is a lossy cache with outliers bounded by the bucket size. + // We improve average lookup by giving preference to the "richer" entries. + // If we used Robin Hood strategy we could eventually end up with all + // entries in the table being maximally "poor". + + uint version = pEntry.Version; + + // mask the lower version bit to make it even. + // This way we will detect both if version is changing (odd) or has changed (even, but different). + version &= unchecked((uint)~1); + + if ((version & VERSION_NUM_MASK) >= (VERSION_NUM_MASK - 2)) + { + // If exactly VERSION_NUM_MASK updates happens between here and publishing, we may not recognize a race. + // It is extremely unlikely, but to not worry about the possibility, lets not allow version to go this high and just get a new cache. + // This will not happen often. + FlushCurrentCache(); + return; + } + + if (version == 0 || (version >> VERSION_NUM_SIZE) > i) + { + uint newVersion = ((uint)i << VERSION_NUM_SIZE) + (version & VERSION_NUM_MASK) + 1; + uint versionOrig = Interlocked.CompareExchange(ref pEntry.Version, newVersion, version); + if (versionOrig == version) + { + pEntry.Hash = hash; + pEntry._key = key; + pEntry._value = value; + + // entry is in inconsistent state and cannot be read or written to until we + // update the version, which is the last thing we do here + Volatile.Write(ref pEntry.Version, newVersion + 1); + return; + } + // someone snatched the entry. try the next one in the bucket. + } + + if (hash == pEntry.Hash && EqualityComparer.Default.Equals(key, pEntry._key)) + { + // looks like we already have an entry for this. + // duplicate entries are harmless, but a bit of a waste. + return; + } + + // quadratic reprobe + i++; + index += i; + pEntry = ref Element(ref tableData, index & TableMask(ref tableData)); + } + + // bucket is full. + } while (TryGrow(ref tableData)); + + // reread tableData after TryGrow. + tableData = ref TableData(_table); + + if (TableMask(ref tableData) == 1) + { + // do not insert into a sentinel. + return; + } + + // pick a victim somewhat randomly within a bucket + // NB: ++ is not interlocked. We are ok if we lose counts here. It is just a number that changes. + byte victimDistance = (byte)(VictimCounter(ref tableData)++ & (BUCKET_SIZE - 1)); + // position the victim in a quadratic reprobe bucket + int victim = (victimDistance * victimDistance + victimDistance) / 2; + + { + ref Entry pEntry = ref Element(ref tableData, (bucket + victim) & TableMask(ref tableData)); + + uint version = pEntry.Version; + + // mask the lower version bit to make it even. + // This way we will detect both if version is changing (odd) or has changed (even, but different). + version &= unchecked((uint)~1); + + if ((version & VERSION_NUM_MASK) >= (VERSION_NUM_MASK - 2)) + { + // If exactly VERSION_NUM_MASK updates happens between here and publishing, we may not recognize a race. + // It is extremely unlikely, but to not worry about the possibility, lets not allow version to go this high and just get a new cache. + // This will not happen often. + FlushCurrentCache(); + return; + } + + uint newVersion = (uint)((victimDistance << VERSION_NUM_SIZE) + (version & VERSION_NUM_MASK) + 1); + uint versionOrig = Interlocked.CompareExchange(ref pEntry.Version, newVersion, version); + + if (versionOrig == version) + { + pEntry.Hash = hash; + pEntry._key = key; + pEntry._value = value; + Volatile.Write(ref pEntry.Version, newVersion + 1); + } + } + } + + private static int CacheElementCount(ref Entry tableData) + { + return TableMask(ref tableData) + 1; + } + + private void FlushCurrentCache() + { + ref Entry tableData = ref TableData(_table); + int lastSize = CacheElementCount(ref tableData); + if (lastSize < INITIAL_CACHE_SIZE) + lastSize = INITIAL_CACHE_SIZE; + + // store the last size to use when creating a new table + // it is just a hint, not needed for correctness, so no synchronization + // with the writing of the table + _lastFlushSize = lastSize; + // flushing is just replacing the table with a sentinel. + _table = s_sentinelTable!; + } + + private bool MaybeReplaceCacheWithLarger(int size) + { + Entry[]? newTable = CreateCastCache(size); + if (newTable == null) + { + return false; + } + + _table = newTable; + return true; + } + + private bool TryGrow(ref Entry tableData) + { + int newSize = CacheElementCount(ref tableData) * 2; + if (newSize <= MAXIMUM_CACHE_SIZE) + { + return MaybeReplaceCacheWithLarger(newSize); + } + + return false; + } + } } From b09dff1657b561d6fd31b02c01e6abf8e3dd60a3 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:16:28 -0700 Subject: [PATCH 03/13] generic cache --- .../src/System/Runtime/TypeLoaderExports.cs | 322 ++++++------------ 1 file changed, 105 insertions(+), 217 deletions(-) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs index 431c83b42d0321..a8b556db29950c 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs @@ -28,143 +28,173 @@ public static unsafe void ActivatorCreateInstanceAny(ref object ptrToData, IntPt // 3- Update ptrToData to point to that newly allocated object ptrToData = RuntimeImports.RhNewObject(pEEType); - Entry entry = LookupInCache(s_cache, pEETypePtr, pEETypePtr); - entry ??= CacheMiss(pEETypePtr, pEETypePtr, - (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => - { - IntPtr result = RuntimeAugments.TypeLoaderCallbacks.TryGetDefaultConstructorForType(new RuntimeTypeHandle(new EETypePtr(context))); - if (result == IntPtr.Zero) - result = RuntimeAugments.GetFallbackDefaultConstructor(); - return result; - }); - RawCalliHelper.Call(entry.Result, ptrToData); + if (!LookupInCache(pEETypePtr, pEETypePtr, out var v)) + { + v = CacheMiss(pEETypePtr, pEETypePtr, + (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => + { + IntPtr result = RuntimeAugments.TypeLoaderCallbacks.TryGetDefaultConstructorForType(new RuntimeTypeHandle(new EETypePtr(context))); + if (result == IntPtr.Zero) + result = RuntimeAugments.GetFallbackDefaultConstructor(); + return result; + }); + } + + RawCalliHelper.Call(v.Result, ptrToData); } // // Generic lookup cache // - private class Entry + private struct Key : IEquatable { public IntPtr Context; public IntPtr Signature; + + public Key(nint context, nint signature) + { + Context = context; + Signature = signature; + } + + public bool Equals(Key other) + { + return Context == other.Context && Signature == other.Signature; + } + + public override int GetHashCode() + { + // TODO: VS shift/roll + return Context.GetHashCode() ^ Signature.GetHashCode(); + } + + public override bool Equals(object obj) + { + return obj is Key && Equals((Key)obj); + } + } + + private struct Value + { public IntPtr Result; public IntPtr AuxResult; - public Entry Next; + + public Value(IntPtr context, IntPtr signature) + { + Result = context; + AuxResult = signature; + } } // Initialize the cache eagerly to avoid null checks. - // Use array with just single element to make this pay-for-play. The actual cache will be allocated only - // once the lazy lookups are actually needed. - private static Entry[] s_cache; - - private static Lock s_lock; - private static GCHandle s_previousCache; - + private static Cache s_cache; internal static void Initialize() { - s_cache = new Entry[1]; + s_cache = new Cache(); + } + + private static Value LookupOrAdd(IntPtr context, IntPtr signature) + { + if (!LookupInCache(context, signature, out var v)) + { + v = CacheMiss(context, signature); + } + + return v; } public static IntPtr GenericLookup(IntPtr context, IntPtr signature) { - Entry entry = LookupInCache(s_cache, context, signature); - entry ??= CacheMiss(context, signature); - return entry.Result; + if (!LookupInCache(context, signature, out var v)) + { + v = CacheMiss(context, signature); + } + + return v.Result; } public static void GenericLookupAndCallCtor(object arg, IntPtr context, IntPtr signature) { - Entry entry = LookupInCache(s_cache, context, signature); - entry ??= CacheMiss(context, signature); - RawCalliHelper.Call(entry.Result, arg); + Value v = LookupOrAdd(context, signature); + RawCalliHelper.Call(v.Result, arg); } public static object GenericLookupAndAllocObject(IntPtr context, IntPtr signature) { - Entry entry = LookupInCache(s_cache, context, signature); - entry ??= CacheMiss(context, signature); - return RawCalliHelper.Call(entry.Result, entry.AuxResult); + Value v = LookupOrAdd(context, signature); + return RawCalliHelper.Call(v.Result, v.AuxResult); } public static object GenericLookupAndAllocArray(IntPtr context, IntPtr arg, IntPtr signature) { - Entry entry = LookupInCache(s_cache, context, signature); - entry ??= CacheMiss(context, signature); - return RawCalliHelper.Call(entry.Result, entry.AuxResult, arg); + Value v = LookupOrAdd(context, signature); + return RawCalliHelper.Call(v.Result, v.AuxResult, arg); } public static void GenericLookupAndCheckArrayElemType(IntPtr context, object arg, IntPtr signature) { - Entry entry = LookupInCache(s_cache, context, signature); - entry ??= CacheMiss(context, signature); - RawCalliHelper.Call(entry.Result, entry.AuxResult, arg); + Value v = LookupOrAdd(context, signature); + RawCalliHelper.Call(v.Result, v.AuxResult, arg); } public static object GenericLookupAndCast(object arg, IntPtr context, IntPtr signature) { - Entry entry = LookupInCache(s_cache, context, signature); - entry ??= CacheMiss(context, signature); - return RawCalliHelper.Call(entry.Result, arg, entry.AuxResult); + Value v = LookupOrAdd(context, signature); + return RawCalliHelper.Call(v.Result, arg, v.AuxResult); } public static unsafe IntPtr GVMLookupForSlot(object obj, RuntimeMethodHandle slot) { - Entry entry = LookupInCache(s_cache, (IntPtr)obj.GetMethodTable(), RuntimeMethodHandle.ToIntPtr(slot)); - if (entry != null) - return entry.Result; + if (LookupInCache((IntPtr)obj.GetMethodTable(), RuntimeMethodHandle.ToIntPtr(slot), out var v)) + return v.Result; return GVMLookupForSlotSlow(obj, slot); } private static unsafe IntPtr GVMLookupForSlotSlow(object obj, RuntimeMethodHandle slot) { - Entry entry = CacheMiss((IntPtr)obj.GetMethodTable(), RuntimeMethodHandle.ToIntPtr(slot), + Value v = CacheMiss((IntPtr)obj.GetMethodTable(), RuntimeMethodHandle.ToIntPtr(slot), (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => RuntimeAugments.TypeLoaderCallbacks.ResolveGenericVirtualMethodTarget(new RuntimeTypeHandle(new EETypePtr(context)), *(RuntimeMethodHandle*)&signature)); - return entry.Result; + return v.Result; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static unsafe IntPtr OpenInstanceMethodLookup(IntPtr openResolver, object obj) { - Entry entry = LookupInCache(s_cache, (IntPtr)obj.GetMethodTable(), openResolver); - entry ??= CacheMiss((IntPtr)obj.GetMethodTable(), openResolver, - (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) - => Internal.Runtime.CompilerServices.OpenMethodResolver.ResolveMethodWorker(signature, contextObject), - obj); - return entry.Result; + if (!LookupInCache((IntPtr)obj.GetMethodTable(), openResolver, out var v)) + { + v = CacheMiss((IntPtr)obj.GetMethodTable(), openResolver, + (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) + => Internal.Runtime.CompilerServices.OpenMethodResolver.ResolveMethodWorker(signature, contextObject), + obj); + } + + return v.Result; } [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] - private static Entry LookupInCache(Entry[] cache, IntPtr context, IntPtr signature) + private static bool LookupInCache(IntPtr context, IntPtr signature, out Value entry) { - int key = ((context.GetHashCode() >> 4) ^ signature.GetHashCode()) & (cache.Length - 1); -#if DEBUG - Entry entry = cache[key]; -#else - Entry entry = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(cache), key); -#endif - while (entry != null) - { - if (entry.Context == context && entry.Signature == signature) - break; - entry = entry.Next; - } - return entry; + Key k = new Key(context, signature); + return s_cache.TryGet(k, out entry); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static IntPtr RuntimeCacheLookupInCache(IntPtr context, IntPtr signature, RuntimeObjectFactory factory, object contextObject, out IntPtr auxResult) { - Entry entry = LookupInCache(s_cache, context, signature); - entry ??= CacheMiss(context, signature, factory, contextObject); - auxResult = entry.AuxResult; - return entry.Result; + if (!LookupInCache(context, signature, out var v)) + { + v = CacheMiss(context, signature, factory, contextObject); + } + + auxResult = v.AuxResult; + return v.Result; } - private static Entry CacheMiss(IntPtr ctx, IntPtr sig) + private static Value CacheMiss(IntPtr ctx, IntPtr sig) { return CacheMiss(ctx, sig, (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => @@ -172,161 +202,19 @@ private static Entry CacheMiss(IntPtr ctx, IntPtr sig) ); } - private static unsafe Entry CacheMiss(IntPtr context, IntPtr signature, RuntimeObjectFactory factory, object contextObject = null) + private static unsafe Value CacheMiss(IntPtr context, IntPtr signature, RuntimeObjectFactory factory, object contextObject = null) { - IntPtr result = IntPtr.Zero, auxResult = IntPtr.Zero; - bool previouslyCached = false; - - // - // Try to find the entry in the previous version of the cache that is kept alive by weak reference - // - if (s_previousCache.IsAllocated) - { - Entry[]? previousCache = (Entry[]?)s_previousCache.Target; - if (previousCache != null) - { - Entry previousEntry = LookupInCache(previousCache, context, signature); - if (previousEntry != null) - { - result = previousEntry.Result; - auxResult = previousEntry.AuxResult; - previouslyCached = true; - } - } - } - // // Call into the type loader to compute the target // - if (!previouslyCached) - { - result = factory(context, signature, contextObject, ref auxResult); - } - - // - // Update the cache under the lock - // - if (s_lock == null) - Interlocked.CompareExchange(ref s_lock, new Lock(), null); - - s_lock.Acquire(); - try - { - // Avoid duplicate entries - Entry existingEntry = LookupInCache(s_cache, context, signature); - if (existingEntry != null) - return existingEntry; - - // Resize cache as necessary - Entry[] cache = ResizeCacheForNewEntryAsNecessary(); - - int key = ((context.GetHashCode() >> 4) ^ signature.GetHashCode()) & (cache.Length - 1); - - Entry newEntry = new Entry() { Context = context, Signature = signature, Result = result, AuxResult = auxResult, Next = cache[key] }; - cache[key] = newEntry; - return newEntry; - } - finally - { - s_lock.Release(); - } - } - - // - // Parameters and state used by generic lookup cache resizing algorithm - // - - private const int InitialCacheSize = 128; // MUST BE A POWER OF TWO - private const int DefaultCacheSize = 1024; - private const int MaximumCacheSize = 128 * 1024; - - private static long s_tickCountOfLastOverflow; - private static int s_entries; - private static bool s_roundRobinFlushing; - - private static Entry[] ResizeCacheForNewEntryAsNecessary() - { - Entry[] cache = s_cache; - - if (cache.Length < InitialCacheSize) - { - // Start with small cache size so that the cache entries used by startup one-time only initialization will get flushed soon - return s_cache = new Entry[InitialCacheSize]; - } - - int entries = s_entries++; - - // If the cache has spare space, we are done - if (2 * entries < cache.Length) - { - if (s_roundRobinFlushing) - { - cache[2 * entries] = null; - cache[2 * entries + 1] = null; - } - return cache; - } - - // - // Now, we have cache that is overflowing with the stuff. We need to decide whether to resize it or start flushing the old entries instead - // + IntPtr auxResult = default; + IntPtr result = factory(context, signature, contextObject, ref auxResult); - // Start over counting the entries - s_entries = 0; + Key k = new Key(context, signature); + Value v = new Value(result, auxResult); - // See how long it has been since the last time the cache was overflowing - long tickCount = Environment.TickCount64; - long tickCountSinceLastOverflow = tickCount - s_tickCountOfLastOverflow; - s_tickCountOfLastOverflow = tickCount; - - bool shrinkCache = false; - bool growCache = false; - - if (cache.Length < DefaultCacheSize) - { - // If the cache have not reached the default size, just grow it without thinking about it much - growCache = true; - } - else - { - if (tickCountSinceLastOverflow < cache.Length / 128) - { - // If the fill rate of the cache is faster than ~0.01ms per entry, grow it - if (cache.Length < MaximumCacheSize) - growCache = true; - } - else - if (tickCountSinceLastOverflow > cache.Length * 16) - { - // If the fill rate of the cache is slower than 16ms per entry, shrink it - if (cache.Length > DefaultCacheSize) - shrinkCache = true; - } - // Otherwise, keep the current size and just keep flushing the entries round robin - } - - if (growCache || shrinkCache) - { - s_roundRobinFlushing = false; - - // Keep the reference to the old cache in a weak handle. We will try to use to avoid - // hitting the type loader until GC collects it. - if (s_previousCache.IsAllocated) - { - s_previousCache.Target = cache; - } - else - { - s_previousCache = GCHandle.Alloc(cache, GCHandleType.Weak); - } - - return s_cache = new Entry[shrinkCache ? (cache.Length / 2) : (cache.Length * 2)]; - } - else - { - s_roundRobinFlushing = true; - return cache; - } + s_cache.TrySet(k, v); + return v; } } From 2f8b6efaae6eaf9bc9f194e084b5f94460cdf88b Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:46:19 -0700 Subject: [PATCH 04/13] some refactoring --- .../src/System/Runtime/TypeLoaderExports.cs | 101 ++++++++------ .../Runtime/CompilerServices/CastCache.cs | 132 ++++++++---------- 2 files changed, 118 insertions(+), 115 deletions(-) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs index a8b556db29950c..d0e11b26aef0aa 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Numerics; namespace System.Runtime { @@ -28,7 +29,7 @@ public static unsafe void ActivatorCreateInstanceAny(ref object ptrToData, IntPt // 3- Update ptrToData to point to that newly allocated object ptrToData = RuntimeImports.RhNewObject(pEEType); - if (!LookupInCache(pEETypePtr, pEETypePtr, out var v)) + if (!TryGetFromCache(pEETypePtr, pEETypePtr, out var v)) { v = CacheMiss(pEETypePtr, pEETypePtr, (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => @@ -40,7 +41,7 @@ public static unsafe void ActivatorCreateInstanceAny(ref object ptrToData, IntPt }); } - RawCalliHelper.Call(v.Result, ptrToData); + RawCalliHelper.Call(v._result, ptrToData); } // @@ -49,24 +50,26 @@ public static unsafe void ActivatorCreateInstanceAny(ref object ptrToData, IntPt private struct Key : IEquatable { - public IntPtr Context; - public IntPtr Signature; + public IntPtr _context; + public IntPtr _signature; public Key(nint context, nint signature) { - Context = context; - Signature = signature; + _context = context; + _signature = signature; } public bool Equals(Key other) { - return Context == other.Context && Signature == other.Signature; + return _context == other._context && _signature == other._signature; } public override int GetHashCode() { - // TODO: VS shift/roll - return Context.GetHashCode() ^ Signature.GetHashCode(); + // pointers will likely match and cancel out in the upper bits + // we will rotate context by 16 bit to keep more varying bits in the hash + IntPtr context = (IntPtr)BitOperations.RotateLeft((nuint)_context, 16); + return (context ^ _signature).GetHashCode(); } public override bool Equals(object obj) @@ -77,26 +80,40 @@ public override bool Equals(object obj) private struct Value { - public IntPtr Result; - public IntPtr AuxResult; + public IntPtr _result; + public IntPtr _auxResult; - public Value(IntPtr context, IntPtr signature) + public Value(IntPtr result, IntPtr auxResult) { - Result = context; - AuxResult = signature; + _result = result; + _auxResult = auxResult; } } + // + // Parameters and state used by generic lookup cache resizing algorithm + // + +#if DEBUG + // use smaller numbers to hit resizing/preempting logic in debug + private const int InitialCacheSize = 8; // MUST BE A POWER OF TWO + private const int MaximumCacheSize = 512; +#else + private const int InitialCacheSize = 128; // MUST BE A POWER OF TWO + private const int MaximumCacheSize = 128 * 1024; +#endif // DEBUG + + // Initialize the cache eagerly to avoid null checks. private static Cache s_cache; internal static void Initialize() { - s_cache = new Cache(); + s_cache = new Cache(InitialCacheSize, MaximumCacheSize); } private static Value LookupOrAdd(IntPtr context, IntPtr signature) { - if (!LookupInCache(context, signature, out var v)) + if (!TryGetFromCache(context, signature, out var v)) { v = CacheMiss(context, signature); } @@ -106,48 +123,48 @@ private static Value LookupOrAdd(IntPtr context, IntPtr signature) public static IntPtr GenericLookup(IntPtr context, IntPtr signature) { - if (!LookupInCache(context, signature, out var v)) + if (!TryGetFromCache(context, signature, out var v)) { v = CacheMiss(context, signature); } - return v.Result; + return v._result; } public static void GenericLookupAndCallCtor(object arg, IntPtr context, IntPtr signature) { Value v = LookupOrAdd(context, signature); - RawCalliHelper.Call(v.Result, arg); + RawCalliHelper.Call(v._result, arg); } public static object GenericLookupAndAllocObject(IntPtr context, IntPtr signature) { Value v = LookupOrAdd(context, signature); - return RawCalliHelper.Call(v.Result, v.AuxResult); + return RawCalliHelper.Call(v._result, v._auxResult); } public static object GenericLookupAndAllocArray(IntPtr context, IntPtr arg, IntPtr signature) { Value v = LookupOrAdd(context, signature); - return RawCalliHelper.Call(v.Result, v.AuxResult, arg); + return RawCalliHelper.Call(v._result, v._auxResult, arg); } public static void GenericLookupAndCheckArrayElemType(IntPtr context, object arg, IntPtr signature) { Value v = LookupOrAdd(context, signature); - RawCalliHelper.Call(v.Result, v.AuxResult, arg); + RawCalliHelper.Call(v._result, v._auxResult, arg); } public static object GenericLookupAndCast(object arg, IntPtr context, IntPtr signature) { Value v = LookupOrAdd(context, signature); - return RawCalliHelper.Call(v.Result, arg, v.AuxResult); + return RawCalliHelper.Call(v._result, arg, v._auxResult); } public static unsafe IntPtr GVMLookupForSlot(object obj, RuntimeMethodHandle slot) { - if (LookupInCache((IntPtr)obj.GetMethodTable(), RuntimeMethodHandle.ToIntPtr(slot), out var v)) - return v.Result; + if (TryGetFromCache((IntPtr)obj.GetMethodTable(), RuntimeMethodHandle.ToIntPtr(slot), out var v)) + return v._result; return GVMLookupForSlotSlow(obj, slot); } @@ -158,13 +175,13 @@ private static unsafe IntPtr GVMLookupForSlotSlow(object obj, RuntimeMethodHandl (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => RuntimeAugments.TypeLoaderCallbacks.ResolveGenericVirtualMethodTarget(new RuntimeTypeHandle(new EETypePtr(context)), *(RuntimeMethodHandle*)&signature)); - return v.Result; + return v._result; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static unsafe IntPtr OpenInstanceMethodLookup(IntPtr openResolver, object obj) { - if (!LookupInCache((IntPtr)obj.GetMethodTable(), openResolver, out var v)) + if (!TryGetFromCache((IntPtr)obj.GetMethodTable(), openResolver, out var v)) { v = CacheMiss((IntPtr)obj.GetMethodTable(), openResolver, (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) @@ -172,11 +189,11 @@ internal static unsafe IntPtr OpenInstanceMethodLookup(IntPtr openResolver, obje obj); } - return v.Result; + return v._result; } - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] - private static bool LookupInCache(IntPtr context, IntPtr signature, out Value entry) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryGetFromCache(IntPtr context, IntPtr signature, out Value entry) { Key k = new Key(context, signature); return s_cache.TryGet(k, out entry); @@ -185,13 +202,13 @@ private static bool LookupInCache(IntPtr context, IntPtr signature, out Value en [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static IntPtr RuntimeCacheLookupInCache(IntPtr context, IntPtr signature, RuntimeObjectFactory factory, object contextObject, out IntPtr auxResult) { - if (!LookupInCache(context, signature, out var v)) + if (!TryGetFromCache(context, signature, out var v)) { v = CacheMiss(context, signature, factory, contextObject); } - auxResult = v.AuxResult; - return v.Result; + auxResult = v._auxResult; + return v._result; } private static Value CacheMiss(IntPtr ctx, IntPtr sig) @@ -222,39 +239,39 @@ private static unsafe Value CacheMiss(IntPtr context, IntPtr signature, RuntimeO internal static unsafe class RawCalliHelper { - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Call(System.IntPtr pfn, ref byte data) => ((delegate*)pfn)(ref data); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static T Call(System.IntPtr pfn, IntPtr arg) => ((delegate*)pfn)(arg); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Call(System.IntPtr pfn, object arg) => ((delegate*)pfn)(arg); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static T Call(System.IntPtr pfn, IntPtr arg1, IntPtr arg2) => ((delegate*)pfn)(arg1, arg2); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static T Call(System.IntPtr pfn, IntPtr arg1, IntPtr arg2, object arg3, out IntPtr arg4) => ((delegate*)pfn)(arg1, arg2, arg3, out arg4); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Call(System.IntPtr pfn, IntPtr arg1, object arg2) => ((delegate*)pfn)(arg1, arg2); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static T Call(System.IntPtr pfn, object arg1, IntPtr arg2) => ((delegate*)pfn)(arg1, arg2); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static T Call(IntPtr pfn, string[] arg0) => ((delegate*)pfn)(arg0); - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static ref byte Call(IntPtr pfn, void* arg1, ref byte arg2, ref byte arg3, void* arg4) => ref ((delegate*)pfn)(arg1, ref arg2, ref arg3, arg4); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index 4fd17225d4796d..95e3f193abd220 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -426,16 +426,41 @@ private bool TryGrow(ref int tableData) } } - internal unsafe struct Cache + [StructLayout(LayoutKind.Explicit)] + internal struct UnmanagedPart { -#if DEBUG - private const int INITIAL_CACHE_SIZE = 8; // MUST BE A POWER OF TWO - private const int MAXIMUM_CACHE_SIZE = 512; // make this lower than release to make it easier to reach this in tests. -#else - private const int INITIAL_CACHE_SIZE = 128; // MUST BE A POWER OF TWO - private const int MAXIMUM_CACHE_SIZE = 4096; // Same as in CastCache. We will rarely need this much though. -#endif // DEBUG + // version has the following structure: + // [ distance:3bit | versionNum:29bit ] + // + // distance is how many iterations the entry is from it ideal position. + // we use that for preemption. + // + // versionNum is a monotonically increasing numerical tag. + // Writer "claims" entry by atomically incrementing the tag. Thus odd number indicates an entry in progress. + // Upon completion of adding an entry the tag is incremented again making it even. Even number indicates a complete entry. + // + // Readers will read the version twice before and after retrieving the entry. + // To have a usable entry both reads must yield the same even version. + // + [FieldOffset(0)] + internal uint _version; + [FieldOffset(sizeof(int))] + internal int _hash; + + // AuxData + [FieldOffset(0)] + internal int tableMask; + [FieldOffset(sizeof(int))] + internal byte hashShift; + [FieldOffset(sizeof(int) + 1)] + internal byte victimCounter; + } + // TKey may contain references, but we want it to be a struct, + // so that equality is devirtualized. + internal unsafe struct Cache + where TKey: struct, IEquatable + { private const int VERSION_NUM_SIZE = 29; private const uint VERSION_NUM_MASK = (1 << VERSION_NUM_SIZE) - 1; private const int BUCKET_SIZE = 8; @@ -449,9 +474,15 @@ internal unsafe struct Cache // when flushing, remember the last size. private int _lastFlushSize; + private int _initialCacheSize; + private int _maxCacheSize; + // creates a new cache instance - public Cache() + public Cache(int initialCacheSize, int maxCacheSize) { + _initialCacheSize = initialCacheSize; + _maxCacheSize = maxCacheSize; + // A trivial 2-elements table used for "flushing" the cache. // Nothing is ever stored in such a small table and identity of the sentinel is not important. // It is required that we are able to allocate this, we may need this in OOM cases. @@ -460,43 +491,12 @@ public Cache() _table = #if !DEBUG // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. - CreateCastCache(INITIAL_CACHE_SIZE) ?? + CreateCastCache(initialCacheSize) ?? #endif s_sentinelTable!; - _lastFlushSize = INITIAL_CACHE_SIZE; - } - - [StructLayout(LayoutKind.Explicit)] - private struct UnmanagedPart - { - // version has the following structure: - // [ distance:3bit | versionNum:29bit ] - // - // distance is how many iterations the entry is from it ideal position. - // we use that for preemption. - // - // versionNum is a monotonically increasing numerical tag. - // Writer "claims" entry by atomically incrementing the tag. Thus odd number indicates an entry in progress. - // Upon completion of adding an entry the tag is incremented again making it even. Even number indicates a complete entry. - // - // Readers will read the version twice before and after retrieving the entry. - // To have a usable entry both reads must yield the same even version. - // - [FieldOffset(0)] - internal uint _version; - [FieldOffset(sizeof(int))] - internal int _hash; - - // AuxData - [FieldOffset(0)] - internal int tableMask; - [FieldOffset(sizeof(int))] - internal byte hashShift; - [FieldOffset(sizeof(int) + 1)] - internal byte victimCounter; + _lastFlushSize = initialCacheSize; } - [StructLayout(LayoutKind.Sequential)] private struct Entry { internal UnmanagedPart _unmanagedPart; @@ -510,10 +510,9 @@ private struct Entry } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int KeyToBucket(ref Entry tableData, TKey key) + private static int HashToBucket(ref Entry tableData, int hash) { - int hashShift = HashShift(ref tableData); - int hash = key!.GetHashCode(); + byte hashShift = HashShift(ref tableData); #if TARGET_64BIT return (int)(((ulong)hash * 11400714819323198485ul) >> hashShift); #else @@ -524,9 +523,8 @@ private static int KeyToBucket(ref Entry tableData, TKey key) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static ref Entry TableData(Entry[] table) { - // element 0 is used for embedded aux data - return ref table[0]; - // return ref Unsafe.As(ref Unsafe.AddByteOffset(ref table.GetRawData(), (nint)sizeof(nint))); + // points to element 0, which is used for embedded aux data + return ref Unsafe.As(ref Unsafe.As(table).Data); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -562,8 +560,7 @@ internal bool TryGet(TKey key, out TValue? value) // table is always initialized and is not null. ref Entry tableData = ref TableData(_table!); int hash = key!.GetHashCode(); - - int index = KeyToBucket(ref tableData, key); + int index = HashToBucket(ref tableData, hash); for (int i = 0; i < BUCKET_SIZE;) { ref Entry pEntry = ref Element(ref tableData, index); @@ -572,24 +569,18 @@ internal bool TryGet(TKey key, out TValue? value) // if version is odd or changes, the entry is inconsistent and thus ignored uint version = Volatile.Read(ref pEntry.Version); - // mask the lower version bit to make it even. - // This way we can check if version is odd or changing in just one compare. - version &= unchecked((uint)~1); - - if (hash == pEntry.Hash && EqualityComparer.Default.Equals(key, pEntry._key)) + if (hash == pEntry.Hash && key.Equals(pEntry._key)) { // we do ordinary reads of the value and // Interlocked.ReadMemoryBarrier() before reading the version value = pEntry._value; - // make sure the second read of 'version' happens after reading 'source' and 'targetAndResults' - // - // We can either: - // - use acquires for both _source and _targetAndResults or - // - issue a load barrier before reading _version - // benchmarks on available hardware (Jan 2020) show that use of a read barrier is cheaper. + // make sure the second read of 'version' happens after reading '_value' Interlocked.ReadMemoryBarrier(); + // mask the lower version bit to make it even. + // This way we can check if version is odd or changing in just one compare. + version &= unchecked((uint)~1); if (version != pEntry.Version) { // oh, so close, the entry is in inconsistent state. @@ -616,13 +607,8 @@ internal bool TryGet(TKey key, out TValue? value) return false; } - // the rest is the support for updating the cache. - // in CoreClr the cache is only updated in the native code - // - // The following helpers must match native implementations in castcache.h and castcache.cpp - // we generally do not OOM in casts, just return null unless throwOnFail is specified. - private static Entry[]? CreateCastCache(int size, bool throwOnFail = false) + private Entry[]? CreateCastCache(int size, bool throwOnFail = false) { // size must be positive Debug.Assert(size > 1); @@ -640,7 +626,7 @@ internal bool TryGet(TKey key, out TValue? value) if (table == null) { - size = INITIAL_CACHE_SIZE; + size = _initialCacheSize; try { table = new Entry[size + 1]; @@ -685,7 +671,7 @@ internal void TrySet(TKey key, TValue value) return; } - bucket = KeyToBucket(ref tableData, key); + bucket = HashToBucket(ref tableData, hash); int index = bucket; ref Entry pEntry = ref Element(ref tableData, index); @@ -734,7 +720,7 @@ internal void TrySet(TKey key, TValue value) // someone snatched the entry. try the next one in the bucket. } - if (hash == pEntry.Hash && EqualityComparer.Default.Equals(key, pEntry._key)) + if (hash == pEntry.Hash && key.Equals(pEntry._key)) { // looks like we already have an entry for this. // duplicate entries are harmless, but a bit of a waste. @@ -805,8 +791,8 @@ private void FlushCurrentCache() { ref Entry tableData = ref TableData(_table); int lastSize = CacheElementCount(ref tableData); - if (lastSize < INITIAL_CACHE_SIZE) - lastSize = INITIAL_CACHE_SIZE; + if (lastSize < _initialCacheSize) + lastSize = _initialCacheSize; // store the last size to use when creating a new table // it is just a hint, not needed for correctness, so no synchronization @@ -831,7 +817,7 @@ private bool MaybeReplaceCacheWithLarger(int size) private bool TryGrow(ref Entry tableData) { int newSize = CacheElementCount(ref tableData) * 2; - if (newSize <= MAXIMUM_CACHE_SIZE) + if (newSize <= _maxCacheSize) { return MaybeReplaceCacheWithLarger(newSize); } From 3ff8edcf5cb0edf2e436f1f059cdca2c7c4c118e Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Fri, 21 Jul 2023 12:32:24 -0700 Subject: [PATCH 05/13] separated GenericCache --- .../Runtime/CompilerServices/CastHelpers.cs | 4 +- .../src/System/Runtime/TypeCast.cs | 2 +- .../src/System/Runtime/TypeLoaderExports.cs | 4 +- .../Runtime/CompilerServices/CastCache.cs | 2 +- .../System.Private.CoreLib.Shared.projitems | 3 +- .../Runtime/CompilerServices/CastCache.cs | 406 +---------------- .../Runtime/CompilerServices/GenericCache.cs | 415 ++++++++++++++++++ 7 files changed, 426 insertions(+), 410 deletions(-) create mode 100644 src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index a18879bddbfaaa..415387d0b3ed98 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -9,7 +9,7 @@ namespace System.Runtime.CompilerServices { - internal static class CastCache + internal static class CastCacheContainer { // In coreclr the table is allocated and written to on the native side. internal static int[]? s_table; @@ -17,7 +17,7 @@ internal static class CastCache internal static unsafe class CastHelpers { - private static CastCacheImpl CastCacheInstance => new CastCacheImpl(CastCache.s_table!); + private static CastCache CastCacheInstance => new CastCache(CastCacheContainer.s_table!); [MethodImpl(MethodImplOptions.InternalCall)] private static extern object IsInstanceOfAny_NoCacheLookup(void* toTypeHnd, object obj); diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs index 2c850604ce9977..88924ed1d85864 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs @@ -24,7 +24,7 @@ namespace System.Runtime [EagerStaticClassConstruction] internal static class TypeCast { - private static CastCacheImpl s_castCache = new CastCacheImpl(); + private static CastCache s_castCache = new CastCache(); [Flags] internal enum AssignmentVariation diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs index d0e11b26aef0aa..61df0731af5373 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs @@ -105,10 +105,10 @@ public Value(IntPtr result, IntPtr auxResult) // Initialize the cache eagerly to avoid null checks. - private static Cache s_cache; + private static GenericCache s_cache; internal static void Initialize() { - s_cache = new Cache(InitialCacheSize, MaximumCacheSize); + s_cache = new GenericCache(InitialCacheSize, MaximumCacheSize); } private static Value LookupOrAdd(IntPtr context, IntPtr signature) diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index d5a93bf41cc793..1fc4aa1240c109 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -8,7 +8,7 @@ internal enum CastResult } // trivial implementation of the cast cache - internal unsafe struct CastCacheImpl + internal unsafe struct CastCache { internal CastResult TryGet(nuint source, nuint target) { diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index b15194e5f8c370..c0993c7572fa0d 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -775,6 +775,7 @@ + @@ -2675,4 +2676,4 @@ - \ No newline at end of file + diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index 95e3f193abd220..61196425a2843c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -18,7 +18,7 @@ internal enum CastResult MaybeCast = 2 } - internal unsafe struct CastCacheImpl + internal unsafe struct CastCache { #if DEBUG private const int INITIAL_CACHE_SIZE = 8; // MUST BE A POWER OF TWO @@ -42,13 +42,13 @@ internal unsafe struct CastCacheImpl private int _lastFlushSize; // wraps existing table - public CastCacheImpl(int[] table) + public CastCache(int[] table) { _table = table; } // creates a new cache instance - public CastCacheImpl() + public CastCache() { // A trivial 2-elements table used for "flushing" the cache. // Nothing is ever stored in such a small table and identity of the sentinel is not important. @@ -425,404 +425,4 @@ private bool TryGrow(ref int tableData) return false; } } - - [StructLayout(LayoutKind.Explicit)] - internal struct UnmanagedPart - { - // version has the following structure: - // [ distance:3bit | versionNum:29bit ] - // - // distance is how many iterations the entry is from it ideal position. - // we use that for preemption. - // - // versionNum is a monotonically increasing numerical tag. - // Writer "claims" entry by atomically incrementing the tag. Thus odd number indicates an entry in progress. - // Upon completion of adding an entry the tag is incremented again making it even. Even number indicates a complete entry. - // - // Readers will read the version twice before and after retrieving the entry. - // To have a usable entry both reads must yield the same even version. - // - [FieldOffset(0)] - internal uint _version; - [FieldOffset(sizeof(int))] - internal int _hash; - - // AuxData - [FieldOffset(0)] - internal int tableMask; - [FieldOffset(sizeof(int))] - internal byte hashShift; - [FieldOffset(sizeof(int) + 1)] - internal byte victimCounter; - } - - // TKey may contain references, but we want it to be a struct, - // so that equality is devirtualized. - internal unsafe struct Cache - where TKey: struct, IEquatable - { - private const int VERSION_NUM_SIZE = 29; - private const uint VERSION_NUM_MASK = (1 << VERSION_NUM_SIZE) - 1; - private const int BUCKET_SIZE = 8; - - // nothing is ever stored into this, so we can use a static instance. - private static Entry[]? s_sentinelTable; - - // The actual storage. - private Entry[] _table; - - // when flushing, remember the last size. - private int _lastFlushSize; - - private int _initialCacheSize; - private int _maxCacheSize; - - // creates a new cache instance - public Cache(int initialCacheSize, int maxCacheSize) - { - _initialCacheSize = initialCacheSize; - _maxCacheSize = maxCacheSize; - - // A trivial 2-elements table used for "flushing" the cache. - // Nothing is ever stored in such a small table and identity of the sentinel is not important. - // It is required that we are able to allocate this, we may need this in OOM cases. - s_sentinelTable ??= CreateCastCache(2, throwOnFail: true); - - _table = -#if !DEBUG - // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. - CreateCastCache(initialCacheSize) ?? -#endif - s_sentinelTable!; - _lastFlushSize = initialCacheSize; - } - - private struct Entry - { - internal UnmanagedPart _unmanagedPart; - internal TKey _key; - internal TValue _value; - - [UnscopedRef] - public ref uint Version => ref _unmanagedPart._version; - [UnscopedRef] - public ref int Hash => ref _unmanagedPart._hash; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int HashToBucket(ref Entry tableData, int hash) - { - byte hashShift = HashShift(ref tableData); -#if TARGET_64BIT - return (int)(((ulong)hash * 11400714819323198485ul) >> hashShift); -#else - return (int)(((uint)hash * 2654435769u) >> hashShift); -#endif - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref Entry TableData(Entry[] table) - { - // points to element 0, which is used for embedded aux data - return ref Unsafe.As(ref Unsafe.As(table).Data); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref byte HashShift(ref Entry tableData) - { - return ref tableData._unmanagedPart.hashShift; - } - - // TableMask is "size - 1" - // we need that more often that we need size - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref int TableMask(ref Entry tableData) - { - return ref tableData._unmanagedPart.tableMask; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref byte VictimCounter(ref Entry tableData) - { - return ref tableData._unmanagedPart.victimCounter; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref Entry Element(ref Entry tableData, int index) - { - // element 0 is used for embedded aux data, skip it - return ref Unsafe.Add(ref tableData, index + 1); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool TryGet(TKey key, out TValue? value) - { - // table is always initialized and is not null. - ref Entry tableData = ref TableData(_table!); - int hash = key!.GetHashCode(); - int index = HashToBucket(ref tableData, hash); - for (int i = 0; i < BUCKET_SIZE;) - { - ref Entry pEntry = ref Element(ref tableData, index); - - // we must read in this order: version -> [entry parts] -> version - // if version is odd or changes, the entry is inconsistent and thus ignored - uint version = Volatile.Read(ref pEntry.Version); - - if (hash == pEntry.Hash && key.Equals(pEntry._key)) - { - // we do ordinary reads of the value and - // Interlocked.ReadMemoryBarrier() before reading the version - value = pEntry._value; - - // make sure the second read of 'version' happens after reading '_value' - Interlocked.ReadMemoryBarrier(); - - // mask the lower version bit to make it even. - // This way we can check if version is odd or changing in just one compare. - version &= unchecked((uint)~1); - if (version != pEntry.Version) - { - // oh, so close, the entry is in inconsistent state. - // it is either changing or has changed while we were reading. - // treat it as a miss. - break; - } - - return true; - } - - if (version == 0) - { - // the rest of the bucket is unclaimed, no point to search further - break; - } - - // quadratic reprobe - i++; - index = (index + i) & TableMask(ref tableData); - } - - value = default; - return false; - } - - // we generally do not OOM in casts, just return null unless throwOnFail is specified. - private Entry[]? CreateCastCache(int size, bool throwOnFail = false) - { - // size must be positive - Debug.Assert(size > 1); - // size must be a power of two - Debug.Assert((size & (size - 1)) == 0); - - Entry[]? table = null; - try - { - table = new Entry[size + 1]; - } - catch (OutOfMemoryException) when (!throwOnFail) - { - } - - if (table == null) - { - size = _initialCacheSize; - try - { - table = new Entry[size + 1]; - } - catch (OutOfMemoryException) - { - } - } - - if (table == null) - { - return table; - } - - ref Entry tableData = ref TableData(table); - - // set the table mask. we need it often, do not want to compute each time. - TableMask(ref tableData) = size - 1; - - // Fibonacci hash reduces the value into desired range by shifting right by the number of leading zeroes in 'size-1' - byte shift = (byte)BitOperations.LeadingZeroCount(size - 1); - HashShift(ref tableData) = shift; - - return table; - } - - internal void TrySet(TKey key, TValue value) - { - int bucket; - int hash = key!.GetHashCode(); - ref Entry tableData = ref Unsafe.NullRef(); - - do - { - tableData = ref TableData(_table); - if (TableMask(ref tableData) == 1) - { - // 2-element table is used as a sentinel. - // we did not allocate a real table yet or have flushed it. - // try replacing the table, but do not insert anything. - MaybeReplaceCacheWithLarger(_lastFlushSize); - return; - } - - bucket = HashToBucket(ref tableData, hash); - int index = bucket; - ref Entry pEntry = ref Element(ref tableData, index); - - for (int i = 0; i < BUCKET_SIZE;) - { - // claim the entry if unused or is more distant than us from its origin. - // Note - someone familiar with Robin Hood hashing will notice that - // we do the opposite - we are "robbing the poor". - // Robin Hood strategy improves average lookup in a lossles dictionary by reducing - // outliers via giving preference to more distant entries. - // What we have here is a lossy cache with outliers bounded by the bucket size. - // We improve average lookup by giving preference to the "richer" entries. - // If we used Robin Hood strategy we could eventually end up with all - // entries in the table being maximally "poor". - - uint version = pEntry.Version; - - // mask the lower version bit to make it even. - // This way we will detect both if version is changing (odd) or has changed (even, but different). - version &= unchecked((uint)~1); - - if ((version & VERSION_NUM_MASK) >= (VERSION_NUM_MASK - 2)) - { - // If exactly VERSION_NUM_MASK updates happens between here and publishing, we may not recognize a race. - // It is extremely unlikely, but to not worry about the possibility, lets not allow version to go this high and just get a new cache. - // This will not happen often. - FlushCurrentCache(); - return; - } - - if (version == 0 || (version >> VERSION_NUM_SIZE) > i) - { - uint newVersion = ((uint)i << VERSION_NUM_SIZE) + (version & VERSION_NUM_MASK) + 1; - uint versionOrig = Interlocked.CompareExchange(ref pEntry.Version, newVersion, version); - if (versionOrig == version) - { - pEntry.Hash = hash; - pEntry._key = key; - pEntry._value = value; - - // entry is in inconsistent state and cannot be read or written to until we - // update the version, which is the last thing we do here - Volatile.Write(ref pEntry.Version, newVersion + 1); - return; - } - // someone snatched the entry. try the next one in the bucket. - } - - if (hash == pEntry.Hash && key.Equals(pEntry._key)) - { - // looks like we already have an entry for this. - // duplicate entries are harmless, but a bit of a waste. - return; - } - - // quadratic reprobe - i++; - index += i; - pEntry = ref Element(ref tableData, index & TableMask(ref tableData)); - } - - // bucket is full. - } while (TryGrow(ref tableData)); - - // reread tableData after TryGrow. - tableData = ref TableData(_table); - - if (TableMask(ref tableData) == 1) - { - // do not insert into a sentinel. - return; - } - - // pick a victim somewhat randomly within a bucket - // NB: ++ is not interlocked. We are ok if we lose counts here. It is just a number that changes. - byte victimDistance = (byte)(VictimCounter(ref tableData)++ & (BUCKET_SIZE - 1)); - // position the victim in a quadratic reprobe bucket - int victim = (victimDistance * victimDistance + victimDistance) / 2; - - { - ref Entry pEntry = ref Element(ref tableData, (bucket + victim) & TableMask(ref tableData)); - - uint version = pEntry.Version; - - // mask the lower version bit to make it even. - // This way we will detect both if version is changing (odd) or has changed (even, but different). - version &= unchecked((uint)~1); - - if ((version & VERSION_NUM_MASK) >= (VERSION_NUM_MASK - 2)) - { - // If exactly VERSION_NUM_MASK updates happens between here and publishing, we may not recognize a race. - // It is extremely unlikely, but to not worry about the possibility, lets not allow version to go this high and just get a new cache. - // This will not happen often. - FlushCurrentCache(); - return; - } - - uint newVersion = (uint)((victimDistance << VERSION_NUM_SIZE) + (version & VERSION_NUM_MASK) + 1); - uint versionOrig = Interlocked.CompareExchange(ref pEntry.Version, newVersion, version); - - if (versionOrig == version) - { - pEntry.Hash = hash; - pEntry._key = key; - pEntry._value = value; - Volatile.Write(ref pEntry.Version, newVersion + 1); - } - } - } - - private static int CacheElementCount(ref Entry tableData) - { - return TableMask(ref tableData) + 1; - } - - private void FlushCurrentCache() - { - ref Entry tableData = ref TableData(_table); - int lastSize = CacheElementCount(ref tableData); - if (lastSize < _initialCacheSize) - lastSize = _initialCacheSize; - - // store the last size to use when creating a new table - // it is just a hint, not needed for correctness, so no synchronization - // with the writing of the table - _lastFlushSize = lastSize; - // flushing is just replacing the table with a sentinel. - _table = s_sentinelTable!; - } - - private bool MaybeReplaceCacheWithLarger(int size) - { - Entry[]? newTable = CreateCastCache(size); - if (newTable == null) - { - return false; - } - - _table = newTable; - return true; - } - - private bool TryGrow(ref Entry tableData) - { - int newSize = CacheElementCount(ref tableData) * 2; - if (newSize <= _maxCacheSize) - { - return MaybeReplaceCacheWithLarger(newSize); - } - - return false; - } - } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs new file mode 100644 index 00000000000000..2279b94fa5ebd0 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs @@ -0,0 +1,415 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Numerics; +using System.Runtime.InteropServices; +using System.Threading; + +namespace System.Runtime.CompilerServices +{ + // Unmanaged part of cache entry + // it is not nested in GenericCache because generic types cannot have explicit layout. + [StructLayout(LayoutKind.Explicit)] + internal struct UnmanagedPart + { + // version has the following structure: + // [ distance:3bit | versionNum:29bit ] + // + // distance is how many iterations the entry is from it ideal position. + // we use that for preemption. + // + // versionNum is a monotonically increasing numerical tag. + // Writer "claims" entry by atomically incrementing the tag. Thus odd number indicates an entry in progress. + // Upon completion of adding an entry the tag is incremented again making it even. Even number indicates a complete entry. + // + // Readers will read the version twice before and after retrieving the entry. + // To have a usable entry both reads must yield the same even version. + // + [FieldOffset(0)] + internal uint _version; + [FieldOffset(sizeof(int))] + internal int _hash; + + // AuxData + [FieldOffset(0)] + internal int tableMask; + [FieldOffset(sizeof(int))] + internal byte hashShift; + [FieldOffset(sizeof(int) + 1)] + internal byte victimCounter; + } + + // TKey may contain references, but we want it to be a struct, + // so that equality is devirtualized. + internal unsafe struct GenericCache + where TKey: struct, IEquatable + { + private const int VERSION_NUM_SIZE = 29; + private const uint VERSION_NUM_MASK = (1 << VERSION_NUM_SIZE) - 1; + private const int BUCKET_SIZE = 8; + + // nothing is ever stored into this, so we can use a static instance. + private static Entry[]? s_sentinelTable; + + // The actual storage. + private Entry[] _table; + + // when flushing, remember the last size. + private int _lastFlushSize; + + private int _initialCacheSize; + private int _maxCacheSize; + + // creates a new cache instance + public GenericCache(int initialCacheSize, int maxCacheSize) + { + _initialCacheSize = initialCacheSize; + _maxCacheSize = maxCacheSize; + + // A trivial 2-elements table used for "flushing" the cache. + // Nothing is ever stored in such a small table and identity of the sentinel is not important. + // It is required that we are able to allocate this, we may need this in OOM cases. + s_sentinelTable ??= CreateCastCache(2, throwOnFail: true); + + _table = +#if !DEBUG + // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. + CreateCastCache(initialCacheSize) ?? +#endif + s_sentinelTable!; + _lastFlushSize = initialCacheSize; + } + + private struct Entry + { + internal UnmanagedPart _unmanagedPart; + internal TKey _key; + internal TValue _value; + + [UnscopedRef] + public ref uint Version => ref _unmanagedPart._version; + [UnscopedRef] + public ref int Hash => ref _unmanagedPart._hash; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int HashToBucket(ref Entry tableData, int hash) + { + byte hashShift = HashShift(ref tableData); +#if TARGET_64BIT + return (int)(((ulong)hash * 11400714819323198485ul) >> hashShift); +#else + return (int)(((uint)hash * 2654435769u) >> hashShift); +#endif + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref Entry TableData(Entry[] table) + { + // points to element 0, which is used for embedded aux data + return ref Unsafe.As(ref Unsafe.As(table).Data); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref byte HashShift(ref Entry tableData) + { + return ref tableData._unmanagedPart.hashShift; + } + + // TableMask is "size - 1" + // we need that more often that we need size + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref int TableMask(ref Entry tableData) + { + return ref tableData._unmanagedPart.tableMask; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref byte VictimCounter(ref Entry tableData) + { + return ref tableData._unmanagedPart.victimCounter; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref Entry Element(ref Entry tableData, int index) + { + // element 0 is used for embedded aux data, skip it + return ref Unsafe.Add(ref tableData, index + 1); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGet(TKey key, out TValue? value) + { + // table is always initialized and is not null. + ref Entry tableData = ref TableData(_table!); + int hash = key!.GetHashCode(); + int index = HashToBucket(ref tableData, hash); + for (int i = 0; i < BUCKET_SIZE;) + { + ref Entry pEntry = ref Element(ref tableData, index); + + // we must read in this order: version -> [entry parts] -> version + // if version is odd or changes, the entry is inconsistent and thus ignored + uint version = Volatile.Read(ref pEntry.Version); + + if (hash == pEntry.Hash && key.Equals(pEntry._key)) + { + // we do ordinary reads of the value and + // Interlocked.ReadMemoryBarrier() before reading the version + value = pEntry._value; + + // make sure the second read of 'version' happens after reading '_value' + Interlocked.ReadMemoryBarrier(); + + // mask the lower version bit to make it even. + // This way we can check if version is odd or changing in just one compare. + version &= unchecked((uint)~1); + if (version != pEntry.Version) + { + // oh, so close, the entry is in inconsistent state. + // it is either changing or has changed while we were reading. + // treat it as a miss. + break; + } + + return true; + } + + if (version == 0) + { + // the rest of the bucket is unclaimed, no point to search further + break; + } + + // quadratic reprobe + i++; + index = (index + i) & TableMask(ref tableData); + } + + value = default; + return false; + } + + // we generally do not OOM in casts, just return null unless throwOnFail is specified. + private Entry[]? CreateCastCache(int size, bool throwOnFail = false) + { + // size must be positive + Debug.Assert(size > 1); + // size must be a power of two + Debug.Assert((size & (size - 1)) == 0); + + Entry[]? table = null; + try + { + table = new Entry[size + 1]; + } + catch (OutOfMemoryException) when (!throwOnFail) + { + } + + if (table == null) + { + size = _initialCacheSize; + try + { + table = new Entry[size + 1]; + } + catch (OutOfMemoryException) + { + } + } + + if (table == null) + { + return table; + } + + ref Entry tableData = ref TableData(table); + + // set the table mask. we need it often, do not want to compute each time. + TableMask(ref tableData) = size - 1; + + // Fibonacci hash reduces the value into desired range by shifting right by the number of leading zeroes in 'size-1' + byte shift = (byte)BitOperations.LeadingZeroCount(size - 1); + HashShift(ref tableData) = shift; + + return table; + } + + internal void TrySet(TKey key, TValue value) + { + int bucket; + int hash = key!.GetHashCode(); + ref Entry tableData = ref Unsafe.NullRef(); + + do + { + tableData = ref TableData(_table); + if (TableMask(ref tableData) == 1) + { + // 2-element table is used as a sentinel. + // we did not allocate a real table yet or have flushed it. + // try replacing the table, but do not insert anything. + MaybeReplaceCacheWithLarger(_lastFlushSize); + return; + } + + bucket = HashToBucket(ref tableData, hash); + int index = bucket; + ref Entry pEntry = ref Element(ref tableData, index); + + for (int i = 0; i < BUCKET_SIZE;) + { + // claim the entry if unused or is more distant than us from its origin. + // Note - someone familiar with Robin Hood hashing will notice that + // we do the opposite - we are "robbing the poor". + // Robin Hood strategy improves average lookup in a lossles dictionary by reducing + // outliers via giving preference to more distant entries. + // What we have here is a lossy cache with outliers bounded by the bucket size. + // We improve average lookup by giving preference to the "richer" entries. + // If we used Robin Hood strategy we could eventually end up with all + // entries in the table being maximally "poor". + + uint version = pEntry.Version; + + // mask the lower version bit to make it even. + // This way we will detect both if version is changing (odd) or has changed (even, but different). + version &= unchecked((uint)~1); + + if ((version & VERSION_NUM_MASK) >= (VERSION_NUM_MASK - 2)) + { + // If exactly VERSION_NUM_MASK updates happens between here and publishing, we may not recognize a race. + // It is extremely unlikely, but to not worry about the possibility, lets not allow version to go this high and just get a new cache. + // This will not happen often. + FlushCurrentCache(); + return; + } + + if (version == 0 || (version >> VERSION_NUM_SIZE) > i) + { + uint newVersion = ((uint)i << VERSION_NUM_SIZE) + (version & VERSION_NUM_MASK) + 1; + uint versionOrig = Interlocked.CompareExchange(ref pEntry.Version, newVersion, version); + if (versionOrig == version) + { + pEntry.Hash = hash; + pEntry._key = key; + pEntry._value = value; + + // entry is in inconsistent state and cannot be read or written to until we + // update the version, which is the last thing we do here + Volatile.Write(ref pEntry.Version, newVersion + 1); + return; + } + // someone snatched the entry. try the next one in the bucket. + } + + if (hash == pEntry.Hash && key.Equals(pEntry._key)) + { + // looks like we already have an entry for this. + // duplicate entries are harmless, but a bit of a waste. + return; + } + + // quadratic reprobe + i++; + index += i; + pEntry = ref Element(ref tableData, index & TableMask(ref tableData)); + } + + // bucket is full. + } while (TryGrow(ref tableData)); + + // reread tableData after TryGrow. + tableData = ref TableData(_table); + + if (TableMask(ref tableData) == 1) + { + // do not insert into a sentinel. + return; + } + + // pick a victim somewhat randomly within a bucket + // NB: ++ is not interlocked. We are ok if we lose counts here. It is just a number that changes. + byte victimDistance = (byte)(VictimCounter(ref tableData)++ & (BUCKET_SIZE - 1)); + // position the victim in a quadratic reprobe bucket + int victim = (victimDistance * victimDistance + victimDistance) / 2; + + { + ref Entry pEntry = ref Element(ref tableData, (bucket + victim) & TableMask(ref tableData)); + + uint version = pEntry.Version; + + // mask the lower version bit to make it even. + // This way we will detect both if version is changing (odd) or has changed (even, but different). + version &= unchecked((uint)~1); + + if ((version & VERSION_NUM_MASK) >= (VERSION_NUM_MASK - 2)) + { + // If exactly VERSION_NUM_MASK updates happens between here and publishing, we may not recognize a race. + // It is extremely unlikely, but to not worry about the possibility, lets not allow version to go this high and just get a new cache. + // This will not happen often. + FlushCurrentCache(); + return; + } + + uint newVersion = (uint)((victimDistance << VERSION_NUM_SIZE) + (version & VERSION_NUM_MASK) + 1); + uint versionOrig = Interlocked.CompareExchange(ref pEntry.Version, newVersion, version); + + if (versionOrig == version) + { + pEntry.Hash = hash; + pEntry._key = key; + pEntry._value = value; + Volatile.Write(ref pEntry.Version, newVersion + 1); + } + } + } + + private static int CacheElementCount(ref Entry tableData) + { + return TableMask(ref tableData) + 1; + } + + private void FlushCurrentCache() + { + ref Entry tableData = ref TableData(_table); + int lastSize = CacheElementCount(ref tableData); + if (lastSize < _initialCacheSize) + lastSize = _initialCacheSize; + + // store the last size to use when creating a new table + // it is just a hint, not needed for correctness, so no synchronization + // with the writing of the table + _lastFlushSize = lastSize; + // flushing is just replacing the table with a sentinel. + _table = s_sentinelTable!; + } + + private bool MaybeReplaceCacheWithLarger(int size) + { + Entry[]? newTable = CreateCastCache(size); + if (newTable == null) + { + return false; + } + + _table = newTable; + return true; + } + + private bool TryGrow(ref Entry tableData) + { + int newSize = CacheElementCount(ref tableData) * 2; + if (newSize <= _maxCacheSize) + { + return MaybeReplaceCacheWithLarger(newSize); + } + + return false; + } + } +} From 1b6a319ebfeba44ecee60691b59586a200c56a82 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Fri, 21 Jul 2023 13:03:07 -0700 Subject: [PATCH 06/13] comments --- .../Runtime/CompilerServices/GenericCache.cs | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs index 2279b94fa5ebd0..1617d4f2a44379 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs @@ -11,15 +11,16 @@ namespace System.Runtime.CompilerServices { - // Unmanaged part of cache entry - // it is not nested in GenericCache because generic types cannot have explicit layout. + // unmanaged part of generic cache entries + // it is a union, so that we could put some extra info in the element #0 of the table. + // the struct is not nested in GenericCache because generic types cannot have explicit layout. [StructLayout(LayoutKind.Explicit)] - internal struct UnmanagedPart + internal struct VersionAndHash { // version has the following structure: // [ distance:3bit | versionNum:29bit ] // - // distance is how many iterations the entry is from it ideal position. + // distance is how many iterations the entry is from its ideal position. // we use that for preemption. // // versionNum is a monotonically increasing numerical tag. @@ -34,7 +35,7 @@ internal struct UnmanagedPart [FieldOffset(sizeof(int))] internal int _hash; - // AuxData + // AuxData (to store some data specific to the table in the element #0 ) [FieldOffset(0)] internal int tableMask; [FieldOffset(sizeof(int))] @@ -48,6 +49,18 @@ internal struct UnmanagedPart internal unsafe struct GenericCache where TKey: struct, IEquatable { + private struct Entry + { + internal VersionAndHash _unmanagedPart; + internal TKey _key; + internal TValue _value; + + [UnscopedRef] + public ref uint Version => ref _unmanagedPart._version; + [UnscopedRef] + public ref int Hash => ref _unmanagedPart._hash; + } + private const int VERSION_NUM_SIZE = 29; private const uint VERSION_NUM_MASK = (1 << VERSION_NUM_SIZE) - 1; private const int BUCKET_SIZE = 8; @@ -73,29 +86,17 @@ public GenericCache(int initialCacheSize, int maxCacheSize) // A trivial 2-elements table used for "flushing" the cache. // Nothing is ever stored in such a small table and identity of the sentinel is not important. // It is required that we are able to allocate this, we may need this in OOM cases. - s_sentinelTable ??= CreateCastCache(2, throwOnFail: true); + s_sentinelTable ??= CreateCacheTable(2, throwOnFail: true); _table = #if !DEBUG // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. - CreateCastCache(initialCacheSize) ?? + CreateCacheTable(initialCacheSize) ?? #endif s_sentinelTable!; _lastFlushSize = initialCacheSize; } - private struct Entry - { - internal UnmanagedPart _unmanagedPart; - internal TKey _key; - internal TValue _value; - - [UnscopedRef] - public ref uint Version => ref _unmanagedPart._version; - [UnscopedRef] - public ref int Hash => ref _unmanagedPart._hash; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int HashToBucket(ref Entry tableData, int hash) { @@ -157,9 +158,9 @@ internal bool TryGet(TKey key, out TValue? value) uint version = Volatile.Read(ref pEntry.Version); if (hash == pEntry.Hash && key.Equals(pEntry._key)) +// if (key.Equals(pEntry._key)) { - // we do ordinary reads of the value and - // Interlocked.ReadMemoryBarrier() before reading the version + // we use ordinary reads to fetch the value value = pEntry._value; // make sure the second read of 'version' happens after reading '_value' @@ -194,8 +195,8 @@ internal bool TryGet(TKey key, out TValue? value) return false; } - // we generally do not OOM in casts, just return null unless throwOnFail is specified. - private Entry[]? CreateCastCache(int size, bool throwOnFail = false) + // we generally do not want OOM in cache lookups, just return null unless throwOnFail is specified. + private Entry[]? CreateCacheTable(int size, bool throwOnFail = false) { // size must be positive Debug.Assert(size > 1); @@ -391,7 +392,7 @@ private void FlushCurrentCache() private bool MaybeReplaceCacheWithLarger(int size) { - Entry[]? newTable = CreateCastCache(size); + Entry[]? newTable = CreateCacheTable(size); if (newTable == null) { return false; From f760cd31e53b26b4ac63f2f341d7917d3d975764 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Fri, 21 Jul 2023 13:22:56 -0700 Subject: [PATCH 07/13] less refs --- .../Runtime/CompilerServices/GenericCache.cs | 71 +++++++++---------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs index 1617d4f2a44379..de077c74b2ea89 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs @@ -98,9 +98,9 @@ public GenericCache(int initialCacheSize, int maxCacheSize) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int HashToBucket(ref Entry tableData, int hash) + private static int HashToBucket(Entry[] table, int hash) { - byte hashShift = HashShift(ref tableData); + byte hashShift = HashShift(table); #if TARGET_64BIT return (int)(((ulong)hash * 11400714819323198485ul) >> hashShift); #else @@ -116,49 +116,49 @@ private static ref Entry TableData(Entry[] table) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref byte HashShift(ref Entry tableData) + private static ref byte HashShift(Entry[] table) { - return ref tableData._unmanagedPart.hashShift; + return ref TableData(table)._unmanagedPart.hashShift; } // TableMask is "size - 1" // we need that more often that we need size [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref int TableMask(ref Entry tableData) + private static int TableMask(Entry[] table) { - return ref tableData._unmanagedPart.tableMask; + return table.Length - 2; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref byte VictimCounter(ref Entry tableData) + private static ref byte VictimCounter(Entry[] table) { - return ref tableData._unmanagedPart.victimCounter; + return ref TableData(table)._unmanagedPart.victimCounter; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref Entry Element(ref Entry tableData, int index) + private static ref Entry Element(Entry[] table, int index) { // element 0 is used for embedded aux data, skip it - return ref Unsafe.Add(ref tableData, index + 1); + return ref Unsafe.Add(ref Unsafe.As(ref Unsafe.As(table).Data), index + 1); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool TryGet(TKey key, out TValue? value) { // table is always initialized and is not null. - ref Entry tableData = ref TableData(_table!); + Entry[] table = _table!; int hash = key!.GetHashCode(); - int index = HashToBucket(ref tableData, hash); + int index = HashToBucket(table, hash); for (int i = 0; i < BUCKET_SIZE;) { - ref Entry pEntry = ref Element(ref tableData, index); + ref Entry pEntry = ref Element(table, index); // we must read in this order: version -> [entry parts] -> version // if version is odd or changes, the entry is inconsistent and thus ignored uint version = Volatile.Read(ref pEntry.Version); - if (hash == pEntry.Hash && key.Equals(pEntry._key)) -// if (key.Equals(pEntry._key)) +// if (hash == pEntry.Hash && key.Equals(pEntry._key)) + if (key.Equals(pEntry._key)) { // we use ordinary reads to fetch the value value = pEntry._value; @@ -188,7 +188,7 @@ internal bool TryGet(TKey key, out TValue? value) // quadratic reprobe i++; - index = (index + i) & TableMask(ref tableData); + index = (index + i) & TableMask(table); } value = default; @@ -231,12 +231,9 @@ internal bool TryGet(TKey key, out TValue? value) ref Entry tableData = ref TableData(table); - // set the table mask. we need it often, do not want to compute each time. - TableMask(ref tableData) = size - 1; - // Fibonacci hash reduces the value into desired range by shifting right by the number of leading zeroes in 'size-1' byte shift = (byte)BitOperations.LeadingZeroCount(size - 1); - HashShift(ref tableData) = shift; + HashShift(table) = shift; return table; } @@ -245,12 +242,12 @@ internal void TrySet(TKey key, TValue value) { int bucket; int hash = key!.GetHashCode(); - ref Entry tableData = ref Unsafe.NullRef(); + Entry[] table; do { - tableData = ref TableData(_table); - if (TableMask(ref tableData) == 1) + table = _table; + if (table.Length == 2) { // 2-element table is used as a sentinel. // we did not allocate a real table yet or have flushed it. @@ -259,9 +256,9 @@ internal void TrySet(TKey key, TValue value) return; } - bucket = HashToBucket(ref tableData, hash); + bucket = HashToBucket(table, hash); int index = bucket; - ref Entry pEntry = ref Element(ref tableData, index); + ref Entry pEntry = ref Element(table, index); for (int i = 0; i < BUCKET_SIZE;) { @@ -318,16 +315,16 @@ internal void TrySet(TKey key, TValue value) // quadratic reprobe i++; index += i; - pEntry = ref Element(ref tableData, index & TableMask(ref tableData)); + pEntry = ref Element(table, index & TableMask(table)); } // bucket is full. - } while (TryGrow(ref tableData)); + } while (TryGrow(table)); // reread tableData after TryGrow. - tableData = ref TableData(_table); + table = _table; - if (TableMask(ref tableData) == 1) + if (table.Length == 2) { // do not insert into a sentinel. return; @@ -335,12 +332,12 @@ internal void TrySet(TKey key, TValue value) // pick a victim somewhat randomly within a bucket // NB: ++ is not interlocked. We are ok if we lose counts here. It is just a number that changes. - byte victimDistance = (byte)(VictimCounter(ref tableData)++ & (BUCKET_SIZE - 1)); + byte victimDistance = (byte)(VictimCounter(table)++ & (BUCKET_SIZE - 1)); // position the victim in a quadratic reprobe bucket int victim = (victimDistance * victimDistance + victimDistance) / 2; { - ref Entry pEntry = ref Element(ref tableData, (bucket + victim) & TableMask(ref tableData)); + ref Entry pEntry = ref Element(table, (bucket + victim) & TableMask(table)); uint version = pEntry.Version; @@ -370,15 +367,15 @@ internal void TrySet(TKey key, TValue value) } } - private static int CacheElementCount(ref Entry tableData) + private static int CacheElementCount(Entry[] table) { - return TableMask(ref tableData) + 1; + return table.Length - 1; } private void FlushCurrentCache() { - ref Entry tableData = ref TableData(_table); - int lastSize = CacheElementCount(ref tableData); + Entry[] table = _table; + int lastSize = CacheElementCount(table); if (lastSize < _initialCacheSize) lastSize = _initialCacheSize; @@ -402,9 +399,9 @@ private bool MaybeReplaceCacheWithLarger(int size) return true; } - private bool TryGrow(ref Entry tableData) + private bool TryGrow(Entry[] table) { - int newSize = CacheElementCount(ref tableData) * 2; + int newSize = CacheElementCount(table) * 2; if (newSize <= _maxCacheSize) { return MaybeReplaceCacheWithLarger(newSize); From 726a0851bce6d57458d5a5e559db2d8dc5362fa5 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:23:30 -0700 Subject: [PATCH 08/13] do not store hash --- .../Runtime/CompilerServices/GenericCache.cs | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs index de077c74b2ea89..324b2fb06848fd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs @@ -11,11 +11,11 @@ namespace System.Runtime.CompilerServices { - // unmanaged part of generic cache entries - // it is a union, so that we could put some extra info in the element #0 of the table. + + // EntryInfo is a union, so that we could put some extra info in the element #0 of the table. // the struct is not nested in GenericCache because generic types cannot have explicit layout. [StructLayout(LayoutKind.Explicit)] - internal struct VersionAndHash + internal struct EntryInfo { // version has the following structure: // [ distance:3bit | versionNum:29bit ] @@ -32,15 +32,11 @@ internal struct VersionAndHash // [FieldOffset(0)] internal uint _version; - [FieldOffset(sizeof(int))] - internal int _hash; // AuxData (to store some data specific to the table in the element #0 ) [FieldOffset(0)] - internal int tableMask; - [FieldOffset(sizeof(int))] internal byte hashShift; - [FieldOffset(sizeof(int) + 1)] + [FieldOffset(1)] internal byte victimCounter; } @@ -51,14 +47,12 @@ internal unsafe struct GenericCache { private struct Entry { - internal VersionAndHash _unmanagedPart; + internal EntryInfo _info; internal TKey _key; internal TValue _value; [UnscopedRef] - public ref uint Version => ref _unmanagedPart._version; - [UnscopedRef] - public ref int Hash => ref _unmanagedPart._hash; + public ref uint Version => ref _info._version; } private const int VERSION_NUM_SIZE = 29; @@ -118,21 +112,20 @@ private static ref Entry TableData(Entry[] table) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static ref byte HashShift(Entry[] table) { - return ref TableData(table)._unmanagedPart.hashShift; + return ref TableData(table)._info.hashShift; } - // TableMask is "size - 1" - // we need that more often that we need size [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int TableMask(Entry[] table) + private static ref byte VictimCounter(Entry[] table) { - return table.Length - 2; + return ref TableData(table)._info.victimCounter; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref byte VictimCounter(Entry[] table) + private static int TableMask(Entry[] table) { - return ref TableData(table)._unmanagedPart.victimCounter; + // element 0 is used for embedded aux data + return table.Length - 2; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -157,7 +150,11 @@ internal bool TryGet(TKey key, out TValue? value) // if version is odd or changes, the entry is inconsistent and thus ignored uint version = Volatile.Read(ref pEntry.Version); -// if (hash == pEntry.Hash && key.Equals(pEntry._key)) + // NOTE: We could store hash as a part of entry info and compare hash before comparing keys. + // Space-wise it would typically be free because of alignment. + // However, hash compare would be advantageous only if it is much cheaper than the key compare. + // That is not the case for current uses of this cache, so for now we do not store + // hash and just do direct comparing of keys. (hash compare can be easily added, if needed) if (key.Equals(pEntry._key)) { // we use ordinary reads to fetch the value @@ -293,7 +290,6 @@ internal void TrySet(TKey key, TValue value) uint versionOrig = Interlocked.CompareExchange(ref pEntry.Version, newVersion, version); if (versionOrig == version) { - pEntry.Hash = hash; pEntry._key = key; pEntry._value = value; @@ -305,7 +301,7 @@ internal void TrySet(TKey key, TValue value) // someone snatched the entry. try the next one in the bucket. } - if (hash == pEntry.Hash && key.Equals(pEntry._key)) + if (key.Equals(pEntry._key)) { // looks like we already have an entry for this. // duplicate entries are harmless, but a bit of a waste. @@ -359,7 +355,6 @@ internal void TrySet(TKey key, TValue value) if (versionOrig == version) { - pEntry.Hash = hash; pEntry._key = key; pEntry._value = value; Volatile.Write(ref pEntry.Version, newVersion + 1); From b179e59dd7e3faed7f740ba365d6e410f212ea54 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Fri, 21 Jul 2023 15:51:42 -0700 Subject: [PATCH 09/13] fix CoreCLR and some more refactoring --- .../Runtime/CompilerServices/CastHelpers.cs | 11 ++++--- .../src/System/Runtime/TypeCast.cs | 10 +++++- .../src/System/Runtime/TypeLoaderExports.cs | 1 - .../Runtime/CompilerServices/CastCache.cs | 4 +++ src/coreclr/vm/corelib.h | 2 +- .../Runtime/CompilerServices/CastCache.cs | 33 ++++++++++--------- .../Runtime/CompilerServices/GenericCache.cs | 9 ++--- 7 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 415387d0b3ed98..cad1136a8f43f7 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -9,15 +9,16 @@ namespace System.Runtime.CompilerServices { - internal static class CastCacheContainer + internal static unsafe class CastHelpers { // In coreclr the table is allocated and written to on the native side. internal static int[]? s_table; - } - internal static unsafe class CastHelpers - { - private static CastCache CastCacheInstance => new CastCache(CastCacheContainer.s_table!); + private static CastCache CastCacheInstance + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => new CastCache(s_table!); + } [MethodImpl(MethodImplOptions.InternalCall)] private static extern object IsInstanceOfAny_NoCacheLookup(void* toTypeHnd, object obj); diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs index 88924ed1d85864..364a248158e733 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs @@ -24,7 +24,15 @@ namespace System.Runtime [EagerStaticClassConstruction] internal static class TypeCast { - private static CastCache s_castCache = new CastCache(); +#if DEBUG + private const int InitialCacheSize = 8; // MUST BE A POWER OF TWO + private const int MaximumCacheSize = 512; // make this lower than release to make it easier to reach this in tests. +#else + private const int InitialCacheSize = 128; // MUST BE A POWER OF TWO + private const int MaximumCacheSize = 4096; // 4096 * sizeof(CastCacheEntry) is 98304 bytes on 64bit. We will rarely need this much though. +#endif // DEBUG + + private static CastCache s_castCache = new CastCache(InitialCacheSize, MaximumCacheSize); [Flags] internal enum AssignmentVariation diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs index 61df0731af5373..0935df4216fa1a 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs @@ -103,7 +103,6 @@ public Value(IntPtr result, IntPtr auxResult) private const int MaximumCacheSize = 128 * 1024; #endif // DEBUG - // Initialize the cache eagerly to avoid null checks. private static GenericCache s_cache; internal static void Initialize() diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index 1fc4aa1240c109..389a8a5d82e1d9 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -10,6 +10,10 @@ internal enum CastResult // trivial implementation of the cast cache internal unsafe struct CastCache { + public CastCache(int initialCacheSize, int maxCacheSize) + { + } + internal CastResult TryGet(nuint source, nuint target) { return CastResult.MaybeCast; diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index d085abd6361e27..96576a85813125 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -1168,7 +1168,7 @@ DEFINE_CLASS(NULLABLE_COMPARER, CollectionsGeneric, NullableComparer`1) DEFINE_CLASS(INATTRIBUTE, Interop, InAttribute) -DEFINE_CLASS(CASTCACHE, CompilerServices, CastCache) +DEFINE_CLASS(CASTCACHE, CompilerServices, CastHelpers) DEFINE_FIELD(CASTCACHE, TABLE, s_table) DEFINE_CLASS(CASTHELPERS, CompilerServices, CastHelpers) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index 61196425a2843c..e89f81e439dce4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -20,14 +20,6 @@ internal enum CastResult internal unsafe struct CastCache { -#if DEBUG - private const int INITIAL_CACHE_SIZE = 8; // MUST BE A POWER OF TWO - private const int MAXIMUM_CACHE_SIZE = 512; // make this lower than release to make it easier to reach this in tests. - #else - private const int INITIAL_CACHE_SIZE = 128; // MUST BE A POWER OF TWO - private const int MAXIMUM_CACHE_SIZE = 4096; // 4096 * sizeof(CastCacheEntry) is 98304 bytes on 64bit. We will rarely need this much though. - #endif // DEBUG - private const int VERSION_NUM_SIZE = 29; private const uint VERSION_NUM_MASK = (1 << VERSION_NUM_SIZE) - 1; private const int BUCKET_SIZE = 8; @@ -41,6 +33,9 @@ internal unsafe struct CastCache // when flushing, remember the last size. private int _lastFlushSize; + private int _initialCacheSize; + private int _maxCacheSize; + // wraps existing table public CastCache(int[] table) { @@ -48,8 +43,14 @@ public CastCache(int[] table) } // creates a new cache instance - public CastCache() + public CastCache(int initialCacheSize, int maxCacheSize) { + Debug.Assert(BitOperations.PopCount((uint)initialCacheSize) == 1 && initialCacheSize > 1); + Debug.Assert(BitOperations.PopCount((uint)maxCacheSize) == 1 && maxCacheSize >= initialCacheSize); + + _initialCacheSize = initialCacheSize; + _maxCacheSize = maxCacheSize; + // A trivial 2-elements table used for "flushing" the cache. // Nothing is ever stored in such a small table and identity of the sentinel is not important. // It is required that we are able to allocate this, we may need this in OOM cases. @@ -58,10 +59,10 @@ public CastCache() _table = #if !DEBUG // Initialize to the sentinel in DEBUG as if just flushed, to ensure the sentinel can be handled in Set. - CreateCastCache(INITIAL_CACHE_SIZE) ?? + CreateCastCache(_initialCacheSize) ?? #endif s_sentinelTable!; - _lastFlushSize = INITIAL_CACHE_SIZE; + _lastFlushSize = _initialCacheSize; } [StructLayout(LayoutKind.Sequential)] @@ -213,7 +214,7 @@ internal CastResult TryGet(nuint source, nuint target) // The following helpers must match native implementations in castcache.h and castcache.cpp // we generally do not OOM in casts, just return null unless throwOnFail is specified. - private static int[]? CreateCastCache(int size, bool throwOnFail = false) + private int[]? CreateCastCache(int size, bool throwOnFail = false) { // size must be positive Debug.Assert(size > 1); @@ -231,7 +232,7 @@ internal CastResult TryGet(nuint source, nuint target) if (table == null) { - size = INITIAL_CACHE_SIZE; + size = _initialCacheSize; try { table = new int[(size + 1) * sizeof(CastCacheEntry) / sizeof(int)]; @@ -391,8 +392,8 @@ private void FlushCurrentCache() { ref int tableData = ref TableData(_table); int lastSize = CacheElementCount(ref tableData); - if (lastSize < INITIAL_CACHE_SIZE) - lastSize = INITIAL_CACHE_SIZE; + if (lastSize < _initialCacheSize) + lastSize = _initialCacheSize; // store the last size to use when creating a new table // it is just a hint, not needed for correctness, so no synchronization @@ -417,7 +418,7 @@ private bool MaybeReplaceCacheWithLarger(int size) private bool TryGrow(ref int tableData) { int newSize = CacheElementCount(ref tableData) * 2; - if (newSize <= MAXIMUM_CACHE_SIZE) + if (newSize <= _maxCacheSize) { return MaybeReplaceCacheWithLarger(newSize); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs index 324b2fb06848fd..712f5ed924645c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/GenericCache.cs @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections; -using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Numerics; @@ -40,8 +38,8 @@ internal struct EntryInfo internal byte victimCounter; } - // TKey may contain references, but we want it to be a struct, - // so that equality is devirtualized. + // NOTE: It is ok if TKey contains references, but we want it to be a struct, + // so that equality is devirtualized. internal unsafe struct GenericCache where TKey: struct, IEquatable { @@ -74,6 +72,9 @@ private struct Entry // creates a new cache instance public GenericCache(int initialCacheSize, int maxCacheSize) { + Debug.Assert(BitOperations.PopCount((uint)initialCacheSize) == 1 && initialCacheSize > 1); + Debug.Assert(BitOperations.PopCount((uint)maxCacheSize) == 1 && maxCacheSize >= initialCacheSize); + _initialCacheSize = initialCacheSize; _maxCacheSize = maxCacheSize; From 3aec31e001fb05c3fb3f3967505595ca45dec50e Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Sun, 23 Jul 2023 16:45:59 -0700 Subject: [PATCH 10/13] PR feedback --- .../Runtime/CompilerServices/CastHelpers.cs | 16 ++-- .../CompilerHelpers/LibraryInitializer.cs | 1 - .../src/System/Runtime/TypeLoaderExports.cs | 88 +++++++++---------- .../System.Private.CoreLib.Shared.projitems | 4 +- .../Runtime/CompilerServices/CastCache.cs | 9 +- 5 files changed, 56 insertions(+), 62 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index cad1136a8f43f7..0292c992a94f3c 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -14,12 +14,6 @@ internal static unsafe class CastHelpers // In coreclr the table is allocated and written to on the native side. internal static int[]? s_table; - private static CastCache CastCacheInstance - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => new CastCache(s_table!); - } - [MethodImpl(MethodImplOptions.InternalCall)] private static extern object IsInstanceOfAny_NoCacheLookup(void* toTypeHnd, object obj); @@ -45,7 +39,7 @@ private static CastCache CastCacheInstance void* mt = RuntimeHelpers.GetMethodTable(obj); if (mt != toTypeHnd) { - CastResult result = CastCacheInstance.TryGet((nuint)mt, (nuint)toTypeHnd); + CastResult result = CastCache.TryGet(s_table!, (nuint)mt, (nuint)toTypeHnd); if (result == CastResult.CanCast) { // do nothing @@ -195,7 +189,7 @@ private static CastCache CastCacheInstance [MethodImpl(MethodImplOptions.NoInlining)] private static object? IsInstance_Helper(void* toTypeHnd, object obj) { - CastResult result = CastCacheInstance.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); + CastResult result = CastCache.TryGet(s_table!, (nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); if (result == CastResult.CanCast) { return obj; @@ -224,7 +218,7 @@ private static CastCache CastCacheInstance void* mt = RuntimeHelpers.GetMethodTable(obj); if (mt != toTypeHnd) { - result = CastCacheInstance.TryGet((nuint)mt, (nuint)toTypeHnd); + result = CastCache.TryGet(s_table!, (nuint)mt, (nuint)toTypeHnd); if (result != CastResult.CanCast) { goto slowPath; @@ -248,7 +242,7 @@ private static CastCache CastCacheInstance [MethodImpl(MethodImplOptions.NoInlining)] private static object? ChkCast_Helper(void* toTypeHnd, object obj) { - CastResult result = CastCacheInstance.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); + CastResult result = CastCache.TryGet(s_table!, (nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); if (result == CastResult.CanCast) { return obj; @@ -465,7 +459,7 @@ private static void StelemRef(Array array, nint index, object? obj) [MethodImpl(MethodImplOptions.NoInlining)] private static void StelemRef_Helper(ref object? element, void* elementType, object obj) { - CastResult result = CastCacheInstance.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType); + CastResult result = CastCache.TryGet(s_table!, (nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType); if (result == CastResult.CanCast) { WriteBarrier(ref element, obj); diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/LibraryInitializer.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/LibraryInitializer.cs index 720f0c3180b207..bf3c60daa8099c 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/LibraryInitializer.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/LibraryInitializer.cs @@ -19,7 +19,6 @@ public static void InitializeLibrary() { PreallocatedOutOfMemoryException.Initialize(); ClassConstructorRunner.Initialize(); - TypeLoaderExports.Initialize(); } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs index 0935df4216fa1a..415c05276974c8 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs @@ -11,43 +11,26 @@ namespace System.Runtime { + // Initialize the cache eagerly to avoid null checks. + [EagerStaticClassConstruction] public static class TypeLoaderExports { - public static unsafe void ActivatorCreateInstanceAny(ref object ptrToData, IntPtr pEETypePtr) - { - EETypePtr pEEType = new EETypePtr(pEETypePtr); - - if (pEEType.IsValueType) - { - // Nothing else to do for value types. - return; - } - - // For reference types, we need to: - // 1- Allocate the new object - // 2- Call its default ctor - // 3- Update ptrToData to point to that newly allocated object - ptrToData = RuntimeImports.RhNewObject(pEEType); - - if (!TryGetFromCache(pEETypePtr, pEETypePtr, out var v)) - { - v = CacheMiss(pEETypePtr, pEETypePtr, - (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => - { - IntPtr result = RuntimeAugments.TypeLoaderCallbacks.TryGetDefaultConstructorForType(new RuntimeTypeHandle(new EETypePtr(context))); - if (result == IntPtr.Zero) - result = RuntimeAugments.GetFallbackDefaultConstructor(); - return result; - }); - } - - RawCalliHelper.Call(v._result, ptrToData); - } - // // Generic lookup cache // +#if DEBUG + // use smaller numbers to hit resizing/preempting logic in debug + private const int InitialCacheSize = 8; // MUST BE A POWER OF TWO + private const int MaximumCacheSize = 512; +#else + private const int InitialCacheSize = 128; // MUST BE A POWER OF TWO + private const int MaximumCacheSize = 128 * 1024; +#endif // DEBUG + + private static GenericCache s_cache = + new GenericCache(InitialCacheSize, MaximumCacheSize); + private struct Key : IEquatable { public IntPtr _context; @@ -90,24 +73,35 @@ public Value(IntPtr result, IntPtr auxResult) } } - // - // Parameters and state used by generic lookup cache resizing algorithm - // + public static unsafe void ActivatorCreateInstanceAny(ref object ptrToData, IntPtr pEETypePtr) + { + EETypePtr pEEType = new EETypePtr(pEETypePtr); -#if DEBUG - // use smaller numbers to hit resizing/preempting logic in debug - private const int InitialCacheSize = 8; // MUST BE A POWER OF TWO - private const int MaximumCacheSize = 512; -#else - private const int InitialCacheSize = 128; // MUST BE A POWER OF TWO - private const int MaximumCacheSize = 128 * 1024; -#endif // DEBUG + if (pEEType.IsValueType) + { + // Nothing else to do for value types. + return; + } - // Initialize the cache eagerly to avoid null checks. - private static GenericCache s_cache; - internal static void Initialize() - { - s_cache = new GenericCache(InitialCacheSize, MaximumCacheSize); + // For reference types, we need to: + // 1- Allocate the new object + // 2- Call its default ctor + // 3- Update ptrToData to point to that newly allocated object + ptrToData = RuntimeImports.RhNewObject(pEEType); + + if (!TryGetFromCache(pEETypePtr, pEETypePtr, out var v)) + { + v = CacheMiss(pEETypePtr, pEETypePtr, + (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => + { + IntPtr result = RuntimeAugments.TypeLoaderCallbacks.TryGetDefaultConstructorForType(new RuntimeTypeHandle(new EETypePtr(context))); + if (result == IntPtr.Zero) + result = RuntimeAugments.GetFallbackDefaultConstructor(); + return result; + }); + } + + RawCalliHelper.Call(v._result, ptrToData); } private static Value LookupOrAdd(IntPtr context, IntPtr signature) diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index c0993c7572fa0d..67248c45820da8 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -774,8 +774,8 @@ - - + + diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index e89f81e439dce4..dbb69790dc22de 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -150,7 +150,14 @@ private static ref CastCacheEntry Element(ref int tableData, int index) internal CastResult TryGet(nuint source, nuint target) { // table is always initialized and is not null. - ref int tableData = ref TableData(_table!); + return TryGet(_table!, source, target); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static CastResult TryGet(int[] table, nuint source, nuint target) + { + // table is always initialized and is not null. + ref int tableData = ref TableData(table); int index = KeyToBucket(ref tableData, source, target); for (int i = 0; i < BUCKET_SIZE;) From c87db36d25ea5d2b127d7526ff9af30720577df7 Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Sun, 23 Jul 2023 16:49:09 -0700 Subject: [PATCH 11/13] remove no longer needed CastCache wrapping constructor --- .../src/System/Runtime/CompilerServices/CastCache.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index dbb69790dc22de..a93da167302cc3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -36,13 +36,6 @@ internal unsafe struct CastCache private int _initialCacheSize; private int _maxCacheSize; - // wraps existing table - public CastCache(int[] table) - { - _table = table; - } - - // creates a new cache instance public CastCache(int initialCacheSize, int maxCacheSize) { Debug.Assert(BitOperations.PopCount((uint)initialCacheSize) == 1 && initialCacheSize > 1); From 26113372bd0f9d5e8b3a1d8db1044190c013f54b Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Sun, 23 Jul 2023 16:57:30 -0700 Subject: [PATCH 12/13] remove auto-inserted unused usings. --- .../src/System/Runtime/CompilerServices/CastCache.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs index a93da167302cc3..7653749d5d5f28 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastCache.cs @@ -1,10 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections; -using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Numerics; using System.Runtime.InteropServices; using System.Threading; From 01866465663cce61fd8b85fceaf101c243148aab Mon Sep 17 00:00:00 2001 From: vsadov <8218165+VSadov@users.noreply.github.com> Date: Sun, 23 Jul 2023 21:50:38 -0700 Subject: [PATCH 13/13] remove unused ActivatorCreateInstanceAny --- .../src/System/Runtime/TypeLoaderExports.cs | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs index 415c05276974c8..c49220b95672da 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/TypeLoaderExports.cs @@ -73,37 +73,6 @@ public Value(IntPtr result, IntPtr auxResult) } } - public static unsafe void ActivatorCreateInstanceAny(ref object ptrToData, IntPtr pEETypePtr) - { - EETypePtr pEEType = new EETypePtr(pEETypePtr); - - if (pEEType.IsValueType) - { - // Nothing else to do for value types. - return; - } - - // For reference types, we need to: - // 1- Allocate the new object - // 2- Call its default ctor - // 3- Update ptrToData to point to that newly allocated object - ptrToData = RuntimeImports.RhNewObject(pEEType); - - if (!TryGetFromCache(pEETypePtr, pEETypePtr, out var v)) - { - v = CacheMiss(pEETypePtr, pEETypePtr, - (IntPtr context, IntPtr signature, object contextObject, ref IntPtr auxResult) => - { - IntPtr result = RuntimeAugments.TypeLoaderCallbacks.TryGetDefaultConstructorForType(new RuntimeTypeHandle(new EETypePtr(context))); - if (result == IntPtr.Zero) - result = RuntimeAugments.GetFallbackDefaultConstructor(); - return result; - }); - } - - RawCalliHelper.Call(v._result, ptrToData); - } - private static Value LookupOrAdd(IntPtr context, IntPtr signature) { if (!TryGetFromCache(context, signature, out var v))