Bix/Ryujinx
Bix/Ryujinx
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move solution and projects to src

Browse source
This commit is contained in:
TSR Berry 2023-04-08 01:22:00 +02:00 • committed by Mary
parent cd124bda58
commit cee7121058
3466 changed files with 55 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/ARMeilleure/Translation/Cache/CacheEntry.cs Normal file
View file

@ -0,0 +1,26 @@
using ARMeilleure.CodeGen.Unwinding;
using System;
using System.Diagnostics.CodeAnalysis;
namespace ARMeilleure.Translation.Cache
{
readonly struct CacheEntry : IComparable<CacheEntry>
{
public int Offset { get; }
public int Size { get; }
public UnwindInfo UnwindInfo { get; }
public CacheEntry(int offset, int size, UnwindInfo unwindInfo)
{
Offset = offset;
Size = size;
UnwindInfo = unwindInfo;
}
public int CompareTo([AllowNull] CacheEntry other)
{
return Offset.CompareTo(other.Offset);
}
}
}

96
src/ARMeilleure/Translation/Cache/CacheMemoryAllocator.cs Normal file
View file

@ -0,0 +1,96 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
namespace ARMeilleure.Translation.Cache
{
class CacheMemoryAllocator
{
private readonly struct MemoryBlock : IComparable<MemoryBlock>
{
public int Offset { get; }
public int Size { get; }
public MemoryBlock(int offset, int size)
{
Offset = offset;
Size = size;
}
public int CompareTo([AllowNull] MemoryBlock other)
{
return Offset.CompareTo(other.Offset);
}
}
private readonly List<MemoryBlock> _blocks = new List<MemoryBlock>();
public CacheMemoryAllocator(int capacity)
{
_blocks.Add(new MemoryBlock(0, capacity));
}
public int Allocate(int size)
{
for (int i = 0; i < _blocks.Count; i++)
{
MemoryBlock block = _blocks[i];
if (block.Size > size)
{
_blocks[i] = new MemoryBlock(block.Offset + size, block.Size - size);
return block.Offset;
}
else if (block.Size == size)
{
_blocks.RemoveAt(i);
return block.Offset;
}
}
// We don't have enough free memory to perform the allocation.
return -1;
}
public void Free(int offset, int size)
{
Insert(new MemoryBlock(offset, size));
}
private void Insert(MemoryBlock block)
{
int index = _blocks.BinarySearch(block);
if (index < 0)
{
index = ~index;
}
if (index < _blocks.Count)
{
MemoryBlock next = _blocks[index];
int endOffs = block.Offset + block.Size;
if (next.Offset == endOffs)
{
block = new MemoryBlock(block.Offset, block.Size + next.Size);
_blocks.RemoveAt(index);
}
}
if (index > 0)
{
MemoryBlock prev = _blocks[index - 1];
if (prev.Offset + prev.Size == block.Offset)
{
block = new MemoryBlock(block.Offset - prev.Size, block.Size + prev.Size);
_blocks.RemoveAt(--index);
}
}
_blocks.Insert(index, block);
}
}
}

198
src/ARMeilleure/Translation/Cache/JitCache.cs Normal file
View file

@ -0,0 +1,198 @@
using ARMeilleure.CodeGen;
using ARMeilleure.CodeGen.Unwinding;
using ARMeilleure.Memory;
using ARMeilleure.Native;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices;
namespace ARMeilleure.Translation.Cache
{
static class JitCache
{
private const int PageSize = 4 * 1024;
private const int PageMask = PageSize - 1;
private const int CodeAlignment = 4; // Bytes.
private const int CacheSize = 2047 * 1024 * 1024;
private static ReservedRegion _jitRegion;
private static JitCacheInvalidation _jitCacheInvalidator;
private static CacheMemoryAllocator _cacheAllocator;
private static readonly List<CacheEntry> _cacheEntries = new List<CacheEntry>();
private static readonly object _lock = new object();
private static bool _initialized;
public static void Initialize(IJitMemoryAllocator allocator)
{
if (_initialized) return;
lock (_lock)
{
if (_initialized) return;
_jitRegion = new ReservedRegion(allocator, CacheSize);
_jitCacheInvalidator = new JitCacheInvalidation(allocator);
_cacheAllocator = new CacheMemoryAllocator(CacheSize);
if (OperatingSystem.IsWindows())
{
JitUnwindWindows.InstallFunctionTableHandler(_jitRegion.Pointer, CacheSize, _jitRegion.Pointer + Allocate(PageSize));
}
_initialized = true;
}
}
public static IntPtr Map(CompiledFunction func)
{
byte[] code = func.Code;
lock (_lock)
{
Debug.Assert(_initialized);
int funcOffset = Allocate(code.Length);
IntPtr funcPtr = _jitRegion.Pointer + funcOffset;
if (OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
unsafe
{
fixed (byte *codePtr = code)
{
JitSupportDarwin.Copy(funcPtr, (IntPtr)codePtr, (ulong)code.Length);
}
}
}
else
{
ReprotectAsWritable(funcOffset, code.Length);
Marshal.Copy(code, 0, funcPtr, code.Length);
ReprotectAsExecutable(funcOffset, code.Length);
_jitCacheInvalidator.Invalidate(funcPtr, (ulong)code.Length);
}
Add(funcOffset, code.Length, func.UnwindInfo);
return funcPtr;
}
}
public static void Unmap(IntPtr pointer)
{
lock (_lock)
{
Debug.Assert(_initialized);
int funcOffset = (int)(pointer.ToInt64() - _jitRegion.Pointer.ToInt64());
bool result = TryFind(funcOffset, out CacheEntry entry);
Debug.Assert(result);
_cacheAllocator.Free(funcOffset, AlignCodeSize(entry.Size));
Remove(funcOffset);
}
}
private static void ReprotectAsWritable(int offset, int size)
{
int endOffs = offset + size;
int regionStart = offset & ~PageMask;
int regionEnd = (endOffs + PageMask) & ~PageMask;
_jitRegion.Block.MapAsRwx((ulong)regionStart, (ulong)(regionEnd - regionStart));
}
private static void ReprotectAsExecutable(int offset, int size)
{
int endOffs = offset + size;
int regionStart = offset & ~PageMask;
int regionEnd = (endOffs + PageMask) & ~PageMask;
_jitRegion.Block.MapAsRx((ulong)regionStart, (ulong)(regionEnd - regionStart));
}
private static int Allocate(int codeSize)
{
codeSize = AlignCodeSize(codeSize);
int allocOffset = _cacheAllocator.Allocate(codeSize);
if (allocOffset < 0)
{
throw new OutOfMemoryException("JIT Cache exhausted.");
}
_jitRegion.ExpandIfNeeded((ulong)allocOffset + (ulong)codeSize);
return allocOffset;
}
private static int AlignCodeSize(int codeSize)
{
return checked(codeSize + (CodeAlignment - 1)) & ~(CodeAlignment - 1);
}
private static void Add(int offset, int size, UnwindInfo unwindInfo)
{
CacheEntry entry = new CacheEntry(offset, size, unwindInfo);
int index = _cacheEntries.BinarySearch(entry);
if (index < 0)
{
index = ~index;
}
_cacheEntries.Insert(index, entry);
}
private static void Remove(int offset)
{
int index = _cacheEntries.BinarySearch(new CacheEntry(offset, 0, default));
if (index < 0)
{
index = ~index - 1;
}
if (index >= 0)
{
_cacheEntries.RemoveAt(index);
}
}
public static bool TryFind(int offset, out CacheEntry entry)
{
lock (_lock)
{
int index = _cacheEntries.BinarySearch(new CacheEntry(offset, 0, default));
if (index < 0)
{
index = ~index - 1;
}
if (index >= 0)
{
entry = _cacheEntries[index];
return true;
}
}
entry = default;
return false;
}
}
}

79
src/ARMeilleure/Translation/Cache/JitCacheInvalidation.cs Normal file
View file

@ -0,0 +1,79 @@
using ARMeilleure.Memory;
using System;
using System.Runtime.InteropServices;
namespace ARMeilleure.Translation.Cache
{
class JitCacheInvalidation
{
private static int[] _invalidationCode = new int[]
{
unchecked((int)0xd53b0022), // mrs x2, ctr_el0
unchecked((int)0xd3504c44), // ubfx x4, x2, #16, #4
unchecked((int)0x52800083), // mov w3, #0x4
unchecked((int)0x12000c45), // and w5, w2, #0xf
unchecked((int)0x1ac42064), // lsl w4, w3, w4
unchecked((int)0x51000482), // sub w2, w4, #0x1
unchecked((int)0x8a220002), // bic x2, x0, x2
unchecked((int)0x1ac52063), // lsl w3, w3, w5
unchecked((int)0xeb01005f), // cmp x2, x1
unchecked((int)0x93407c84), // sxtw x4, w4
unchecked((int)0x540000a2), // b.cs 3c <do_ic_clear>
unchecked((int)0xd50b7b22), // dc cvau, x2
unchecked((int)0x8b040042), // add x2, x2, x4
unchecked((int)0xeb02003f), // cmp x1, x2
unchecked((int)0x54ffffa8), // b.hi 2c <dc_clear_loop>
unchecked((int)0xd5033b9f), // dsb ish
unchecked((int)0x51000462), // sub w2, w3, #0x1
unchecked((int)0x93407c63), // sxtw x3, w3
unchecked((int)0x8a220000), // bic x0, x0, x2
unchecked((int)0xeb00003f), // cmp x1, x0
unchecked((int)0x540000a9), // b.ls 64 <exit>
unchecked((int)0xd50b7520), // ic ivau, x0
unchecked((int)0x8b030000), // add x0, x0, x3
unchecked((int)0xeb00003f), // cmp x1, x0
unchecked((int)0x54ffffa8), // b.hi 54 <ic_clear_loop>
unchecked((int)0xd5033b9f), // dsb ish
unchecked((int)0xd5033fdf), // isb
unchecked((int)0xd65f03c0), // ret
};
private delegate void InvalidateCache(ulong start, ulong end);
private InvalidateCache _invalidateCache;
private ReservedRegion _invalidateCacheCodeRegion;
private readonly bool _needsInvalidation;
public JitCacheInvalidation(IJitMemoryAllocator allocator)
{
// On macOS, a different path is used to write to the JIT cache, which does the invalidation.
if (!OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
ulong size = (ulong)_invalidationCode.Length * sizeof(int);
ulong mask = (ulong)ReservedRegion.DefaultGranularity - 1;
size = (size + mask) & ~mask;
_invalidateCacheCodeRegion = new ReservedRegion(allocator, size);
_invalidateCacheCodeRegion.ExpandIfNeeded(size);
Marshal.Copy(_invalidationCode, 0, _invalidateCacheCodeRegion.Pointer, _invalidationCode.Length);
_invalidateCacheCodeRegion.Block.MapAsRx(0, size);
_invalidateCache = Marshal.GetDelegateForFunctionPointer<InvalidateCache>(_invalidateCacheCodeRegion.Pointer);
_needsInvalidation = true;
}
}
public void Invalidate(IntPtr basePointer, ulong size)
{
if (_needsInvalidation)
{
_invalidateCache((ulong)basePointer, (ulong)basePointer + size);
}
}
}
}

189
src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs Normal file
View file

@ -0,0 +1,189 @@
// https://github.com/MicrosoftDocs/cpp-docs/blob/master/docs/build/exception-handling-x64.md
using ARMeilleure.CodeGen.Unwinding;
using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
namespace ARMeilleure.Translation.Cache
{
static partial class JitUnwindWindows
{
private const int MaxUnwindCodesArraySize = 32; // Must be an even value.
private struct RuntimeFunction
{
public uint BeginAddress;
public uint EndAddress;
public uint UnwindData;
}
private struct UnwindInfo
{
public byte VersionAndFlags;
public byte SizeOfProlog;
public byte CountOfUnwindCodes;
public byte FrameRegister;
public unsafe fixed ushort UnwindCodes[MaxUnwindCodesArraySize];
}
private enum UnwindOp
{
PushNonvol = 0,
AllocLarge = 1,
AllocSmall = 2,
SetFpreg = 3,
SaveNonvol = 4,
SaveNonvolFar = 5,
SaveXmm128 = 8,
SaveXmm128Far = 9,
PushMachframe = 10
}
private unsafe delegate RuntimeFunction* GetRuntimeFunctionCallback(ulong controlPc, IntPtr context);
[LibraryImport("kernel32.dll")]
[return: MarshalAs(UnmanagedType.Bool)]
private static unsafe partial bool RtlInstallFunctionTableCallback(
ulong tableIdentifier,
ulong baseAddress,
uint length,
GetRuntimeFunctionCallback callback,
IntPtr context,
[MarshalAs(UnmanagedType.LPWStr)] string outOfProcessCallbackDll);
private static GetRuntimeFunctionCallback _getRuntimeFunctionCallback;
private static int _sizeOfRuntimeFunction;
private unsafe static RuntimeFunction* _runtimeFunction;
private unsafe static UnwindInfo* _unwindInfo;
public static void InstallFunctionTableHandler(IntPtr codeCachePointer, uint codeCacheLength, IntPtr workBufferPtr)
{
ulong codeCachePtr = (ulong)codeCachePointer.ToInt64();
_sizeOfRuntimeFunction = Marshal.SizeOf<RuntimeFunction>();
bool result;
unsafe
{
_runtimeFunction = (RuntimeFunction*)workBufferPtr;
_unwindInfo = (UnwindInfo*)(workBufferPtr + _sizeOfRuntimeFunction);
_getRuntimeFunctionCallback = new GetRuntimeFunctionCallback(FunctionTableHandler);
result = RtlInstallFunctionTableCallback(
codeCachePtr | 3,
codeCachePtr,
codeCacheLength,
_getRuntimeFunctionCallback,
codeCachePointer,
null);
}
if (!result)
{
throw new InvalidOperationException("Failure installing function table callback.");
}
}
private static unsafe RuntimeFunction* FunctionTableHandler(ulong controlPc, IntPtr context)
{
int offset = (int)((long)controlPc - context.ToInt64());
if (!JitCache.TryFind(offset, out CacheEntry funcEntry))
{
return null; // Not found.
}
var unwindInfo = funcEntry.UnwindInfo;
int codeIndex = 0;
for (int index = unwindInfo.PushEntries.Length - 1; index >= 0; index--)
{
var entry = unwindInfo.PushEntries[index];
switch (entry.PseudoOp)
{
case UnwindPseudoOp.SaveXmm128:
{
int stackOffset = entry.StackOffsetOrAllocSize;
Debug.Assert(stackOffset % 16 == 0);
if (stackOffset <= 0xFFFF0)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128, entry.PrologOffset, entry.RegIndex);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset / 16);
}
else
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128Far, entry.PrologOffset, entry.RegIndex);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 16);
}
break;
}
case UnwindPseudoOp.AllocStack:
{
int allocSize = entry.StackOffsetOrAllocSize;
Debug.Assert(allocSize % 8 == 0);
if (allocSize <= 128)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocSmall, entry.PrologOffset, (allocSize / 8) - 1);
}
else if (allocSize <= 0x7FFF8)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize / 8);
}
else
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 1);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 16);
}
break;
}
case UnwindPseudoOp.PushReg:
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.PushNonvol, entry.PrologOffset, entry.RegIndex);
break;
}
default: throw new NotImplementedException($"({nameof(entry.PseudoOp)} = {entry.PseudoOp})");
}
}
Debug.Assert(codeIndex <= MaxUnwindCodesArraySize);
_unwindInfo->VersionAndFlags = 1; // Flags: The function has no handler.
_unwindInfo->SizeOfProlog = (byte)unwindInfo.PrologSize;
_unwindInfo->CountOfUnwindCodes = (byte)codeIndex;
_unwindInfo->FrameRegister = 0;
_runtimeFunction->BeginAddress = (uint)funcEntry.Offset;
_runtimeFunction->EndAddress = (uint)(funcEntry.Offset + funcEntry.Size);
_runtimeFunction->UnwindData = (uint)_sizeOfRuntimeFunction;
return _runtimeFunction;
}
private static ushort PackUnwindOp(UnwindOp op, int prologOffset, int opInfo)
{
return (ushort)(prologOffset | ((int)op << 8) | (opInfo << 12));
}
}
}