mirrors/Ryubing
1
0
Fork 0
mirror of https://git.ryujinx.app/ryubing/ryujinx.git synced 2025-09-10 03:45:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Memory changes 2.2 (ryubing/ryujinx!143)

Browse Source 
See merge request ryubing/ryujinx!143
This commit is contained in:
LotP
2025-09-06 11:10:55 -05:00
parent 4c9b48b754
commit a60b2a0ba3
14 changed files with 242 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/Ryujinx.Graphics.Device/DeviceState.cs
View File

@@ -97,7 +97,7 @@ namespace Ryujinx.Graphics.Device
uint alignedOffset = index * RegisterSize; uint alignedOffset = index * RegisterSize;
DebugWrite(alignedOffset, data); DebugWrite(alignedOffset, data);
GetRefIntAlignedUncheck(index) = data; SetIntAlignedUncheck(index, data);
_writeCallbacks[index]?.Invoke(data); _writeCallbacks[index]?.Invoke(data);
} }
@@ -112,9 +112,7 @@ namespace Ryujinx.Graphics.Device
uint alignedOffset = index * RegisterSize; uint alignedOffset = index * RegisterSize;
DebugWrite(alignedOffset, data); DebugWrite(alignedOffset, data);
ref int storage = ref GetRefIntAlignedUncheck(index); changed = SetIntAlignedUncheckChanged(index, data);
changed = storage != data;
storage = data;
_writeCallbacks[index]?.Invoke(data); _writeCallbacks[index]?.Invoke(data);
} }
@@ -154,5 +152,24 @@ namespace Ryujinx.Graphics.Device
{ {
return ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index); return ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index);
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void SetIntAlignedUncheck(ulong index, int data)
{
Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index) = data;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool SetIntAlignedUncheckChanged(ulong index, int data)
{
ref int val = ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index);
if (val == data)
{
return false;
}
val = data;
return true;
}
} }
} }

12
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View File

@@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong size, ulong size,
BufferStage stage, BufferStage stage,
bool sparseCompatible, bool sparseCompatible,
List<Buffer> baseBuffers) RangeItem<Buffer>[] baseBuffers)
{ {
_context = context; _context = context;
_physicalMemory = physicalMemory; _physicalMemory = physicalMemory;
@@ -128,18 +128,18 @@ namespace Ryujinx.Graphics.Gpu.Memory
List<IRegionHandle> baseHandles = null; List<IRegionHandle> baseHandles = null;
if (baseBuffers.Count != 0) if (baseBuffers.Length != 0)
{ {
baseHandles = new List<IRegionHandle>(); baseHandles = new List<IRegionHandle>();
foreach (Buffer buffer in baseBuffers) foreach (RangeItem<Buffer> item in baseBuffers)
{ {
if (buffer._useGranular) if (item.Value._useGranular)
{ {
baseHandles.AddRange((buffer._memoryTrackingGranular.GetHandles())); baseHandles.AddRange((item.Value._memoryTrackingGranular.GetHandles()));
} }
else else
{ {
baseHandles.Add(buffer._memoryTracking); baseHandles.Add(item.Value._memoryTracking);
} }
} }
} }

11
src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs
View File

@@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="parent">Parent buffer</param> /// <param name="parent">Parent buffer</param>
/// <param name="stage">Initial buffer stage</param> /// <param name="stage">Initial buffer stage</param>
/// <param name="baseBuffers">Buffers to inherit state from</param> /// <param name="baseBuffers">Buffers to inherit state from</param>
public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, List<Buffer> baseBuffers) public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, RangeItem<Buffer>[] baseBuffers)
{ {
_size = (int)parent.Size; _size = (int)parent.Size;
_systemMemoryType = context.Capabilities.MemoryType; _systemMemoryType = context.Capabilities.MemoryType;
@@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferStage storageFlags = stage & BufferStage.StorageMask; BufferStage storageFlags = stage & BufferStage.StorageMask;
if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Count == 0) if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Length == 0)
{ {
_desiredType = BufferBackingType.DeviceMemory; _desiredType = BufferBackingType.DeviceMemory;
} }
@@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// TODO: Might be nice to force atomic access to be device local for any stage. // TODO: Might be nice to force atomic access to be device local for any stage.
} }
if (baseBuffers.Count != 0) if (baseBuffers.Length != 0)
{ {
foreach (Buffer buffer in baseBuffers) foreach (RangeItem<Buffer> item in baseBuffers)
{ {
CombineState(buffer.BackingState); CombineState(item.Value.BackingState);
} }
} }
} }

95
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
View File

@@ -81,13 +81,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
MemoryRange subRange = range.GetSubRange(index); MemoryRange subRange = range.GetSubRange(index);
_buffers.Lock.EnterReadLock(); _buffers.Lock.EnterReadLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(subRange.Address, subRange.Size); Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(subRange.Address, subRange.Size);
RangeItem<Buffer> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
current.Value.Unmapped(subRange.Address, subRange.Size); overlaps[i].Value.Unmapped(subRange.Address, subRange.Size);
current = current.Next;
} }
_buffers.Lock.ExitReadLock(); _buffers.Lock.ExitReadLock();
@@ -490,9 +488,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) private void CreateBufferAligned(ulong address, ulong size, BufferStage stage)
{ {
_buffers.Lock.EnterWriteLock(); _buffers.Lock.EnterWriteLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(address, size); Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(address, size);
if (first is not null) if (overlaps.Length > 0)
{ {
// The buffer already exists. We can just return the existing buffer // The buffer already exists. We can just return the existing buffer
// if the buffer we need is fully contained inside the overlapping buffer. // if the buffer we need is fully contained inside the overlapping buffer.
@@ -502,7 +500,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size; ulong endAddress = address + size;
if (first.Address > address || first.EndAddress < endAddress) if (overlaps[0].Address > address || overlaps[0].EndAddress < endAddress)
{ {
bool anySparseCompatible = false; bool anySparseCompatible = false;
@@ -515,39 +513,40 @@ namespace Ryujinx.Graphics.Gpu.Memory
// sequential memory. // sequential memory.
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the // Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages. // range crosses a page, and after alignment, ends having a size of 2 pages.
if (first == last && if (overlaps.Length == 1 &&
address >= first.Address && address >= overlaps[0].Address &&
endAddress - first.EndAddress <= BufferAlignmentSize * 2) endAddress - overlaps[0].EndAddress <= BufferAlignmentSize * 2)
{ {
// Try to grow the buffer by 1.5x of its current size. // Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts. // This improves performance in the cases where the buffer is resized often by small amounts.
ulong existingSize = first.Value.Size; ulong existingSize = overlaps[0].Value.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize); size = Math.Max(size, growthSize);
endAddress = address + size; endAddress = address + size;
(first, last) = _buffers.FindOverlaps(address, size); overlaps = _buffers.FindOverlapsAsSpan(address, size);
} }
address = Math.Min(address, first.Address); address = Math.Min(address, overlaps[0].Address);
endAddress = Math.Max(endAddress, last.EndAddress); endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
List<Buffer> overlaps = []; RangeItem<Buffer>[] overlapsArray = overlaps.ToArray();
RangeItem<Buffer> current = first; for (int i = 0; i < overlaps.Length; i++)
while (current != last.Next)
{ {
anySparseCompatible |= current.Value.SparseCompatible; anySparseCompatible |= overlaps[i].Value.SparseCompatible;
overlaps.Add(current.Value);
_buffers.Remove(current.Value);
current = current.Next;
} }
_buffers.RemoveRange(overlaps[0], overlaps[^1]);
_buffers.Lock.ExitWriteLock();
ulong newSize = endAddress - address; ulong newSize = endAddress - address;
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlapsArray);
_buffers.Lock.EnterWriteLock();
_buffers.Add(newBuffer); _buffers.Add(newBuffer);
} }
@@ -577,19 +576,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
bool sparseAligned = alignment >= SparseBufferAlignmentSize; bool sparseAligned = alignment >= SparseBufferAlignmentSize;
_buffers.Lock.EnterWriteLock(); _buffers.Lock.EnterWriteLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(address, size); Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(address, size);
if (first is not null) if (overlaps.Length > 0)
{ {
// If the buffer already exists, make sure if covers the entire range, // If the buffer already exists, make sure if covers the entire range,
// and make sure it is properly aligned, otherwise sparse mapping may fail. // and make sure it is properly aligned, otherwise sparse mapping may fail.
ulong endAddress = address + size; ulong endAddress = address + size;
if (first.Address > address || if (overlaps[0].Address > address ||
first.EndAddress < endAddress || overlaps[0].EndAddress < endAddress ||
(first.Address & (alignment - 1)) != 0 || (overlaps[0].Address & (alignment - 1)) != 0 ||
(!first.Value.SparseCompatible && sparseAligned)) (!overlaps[0].Value.SparseCompatible && sparseAligned))
{ {
// We need to make sure the new buffer is properly aligned. // We need to make sure the new buffer is properly aligned.
// However, after the range is aligned, it is possible that it // However, after the range is aligned, it is possible that it
@@ -597,33 +596,30 @@ namespace Ryujinx.Graphics.Gpu.Memory
// and ensure we cover all overlaps. // and ensure we cover all overlaps.
RangeItem<Buffer> oldFirst; RangeItem<Buffer> oldFirst;
endAddress = Math.Max(endAddress, last.EndAddress); endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
do do
{ {
address = Math.Min(address, first.Address); address = Math.Min(address, overlaps[0].Address);
address &= ~(alignment - 1); address &= ~(alignment - 1);
oldFirst = first; oldFirst = overlaps[0];
(first, last) = _buffers.FindOverlaps(address, endAddress - address); overlaps = _buffers.FindOverlapsAsSpan(address, endAddress - address);
} }
while (oldFirst != first); while (oldFirst != overlaps[0]);
ulong newSize = endAddress - address; ulong newSize = endAddress - address;
List<Buffer> overlaps = []; RangeItem<Buffer>[] overlapsArray = overlaps.ToArray();
RangeItem<Buffer> current = first; _buffers.RemoveRange(overlaps[0], overlaps[^1]);
while (current != last.Next)
{
overlaps.Add(current.Value);
_buffers.Remove(current.Value);
current = current.Next;
}
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); _buffers.Lock.ExitWriteLock();
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlapsArray);
_buffers.Lock.EnterWriteLock();
_buffers.Add(newBuffer); _buffers.Add(newBuffer);
} }
@@ -635,6 +631,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_buffers.Add(buffer); _buffers.Add(buffer);
} }
_buffers.Lock.ExitWriteLock(); _buffers.Lock.ExitWriteLock();
} }
@@ -648,13 +645,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="stage">The type of usage that created the buffer</param> /// <param name="stage">The type of usage that created the buffer</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param> /// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="overlaps">Buffers overlapping the range</param> /// <param name="overlaps">Buffers overlapping the range</param>
private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, List<Buffer> overlaps) private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, RangeItem<Buffer>[] overlaps)
{ {
Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps);
for (int index = 0; index < overlaps.Count; index++) for (int index = 0; index < overlaps.Length; index++)
{ {
Buffer buffer = overlaps[index]; Buffer buffer = overlaps[index].Value;
int dstOffset = (int)(buffer.Address - newBuffer.Address); int dstOffset = (int)(buffer.Address - newBuffer.Address);
@@ -930,7 +927,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (size != 0) if (size != 0)
{ {
buffer = _buffers.FindOverlapFast(address, size).Value; buffer = _buffers.FindOverlap(address, size).Value;
buffer.CopyFromDependantVirtualBuffers(); buffer.CopyFromDependantVirtualBuffers();
buffer.SynchronizeMemory(address, size); buffer.SynchronizeMemory(address, size);

60
src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
View File

@@ -80,8 +80,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
private BufferMigration _source; private BufferMigration _source;
private BufferModifiedRangeList _migrationTarget; private BufferModifiedRangeList _migrationTarget;
private List<RangeItem<BufferModifiedRange>> _overlaps;
/// <summary> /// <summary>
/// Whether the modified range list has any entries or not. /// Whether the modified range list has any entries or not.
@@ -108,7 +106,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
_context = context; _context = context;
_parent = parent; _parent = parent;
_flushAction = flushAction; _flushAction = flushAction;
_overlaps = [];
} }
/// <summary> /// <summary>
@@ -122,12 +119,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Slices a given region using the modified regions in the list. Calls the action for the new slices. // Slices a given region using the modified regions in the list. Calls the action for the new slices.
Lock.EnterReadLock(); Lock.EnterReadLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); Span<RangeItem<BufferModifiedRange>> overlaps = FindOverlapsAsSpan(address, size);
RangeItem<BufferModifiedRange> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
BufferModifiedRange overlap = current.Value; BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.Address > address) if (overlap.Address > address)
{ {
@@ -138,7 +134,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Remaining region is after this overlap. // Remaining region is after this overlap.
size -= overlap.EndAddress - address; size -= overlap.EndAddress - address;
address = overlap.EndAddress; address = overlap.EndAddress;
current = current.Next;
} }
Lock.ExitReadLock(); Lock.ExitReadLock();
@@ -158,12 +153,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size of the modified region in bytes</param> /// <param name="size">Size of the modified region in bytes</param>
public void SignalModified(ulong address, ulong size) public void SignalModified(ulong address, ulong size)
{ {
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
ulong endAddress = address + size; ulong endAddress = address + size;
ulong syncNumber = _context.SyncNumber; ulong syncNumber = _context.SyncNumber;
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlapsAsNodes(address, size);
if (first is null) if (first is null)
{ {
@@ -172,8 +166,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
return; return;
} }
if (first == last) if (first == last)
{ {
if (first.Address == address && first.EndAddress == endAddress) if (first.Address == address && first.EndAddress == endAddress)
@@ -260,19 +252,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction) public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction)
{ {
Lock.EnterReadLock(); Lock.EnterReadLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); Span<RangeItem<BufferModifiedRange>> overlaps = FindOverlapsAsSpan(address, size);
RangeItem<BufferModifiedRange> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
BufferModifiedRange overlap = current.Value; BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.SyncNumber == syncNumber) if (overlap.SyncNumber == syncNumber)
{ {
rangeAction(overlap.Address, overlap.Size); rangeAction(overlap.Address, overlap.Size);
} }
current = current.Next;
} }
Lock.ExitReadLock(); Lock.ExitReadLock();
@@ -288,22 +277,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
// We use the non-span method here because keeping the lock will cause a deadlock. // We use the non-span method here because keeping the lock will cause a deadlock.
Lock.EnterReadLock(); Lock.EnterReadLock();
RangeItem<BufferModifiedRange>[] overlaps = FindOverlapsAsArray(address, size);
_overlaps.Clear();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
{
_overlaps.Add(current);
current = current.Next;
}
Lock.ExitReadLock(); Lock.ExitReadLock();
for (int i = 0; i < _overlaps.Count; i++) for (int i = 0; i < overlaps.Length; i++)
{ {
BufferModifiedRange overlap = _overlaps[i].Value; BufferModifiedRange overlap = overlaps[i].Value;
rangeAction(overlap.Address, overlap.Size); rangeAction(overlap.Address, overlap.Size);
} }
} }
@@ -404,8 +383,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size; ulong endAddress = address + size;
ulong currentSync = _context.SyncNumber; ulong currentSync = _context.SyncNumber;
List<RangeItem<BufferModifiedRange>> overlaps = [];
// Range list must be consistent for this operation // Range list must be consistent for this operation
if (_migrationTarget != null) if (_migrationTarget != null)
{ {
@@ -416,16 +393,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
Lock.EnterWriteLock(); Lock.EnterWriteLock();
// We use the non-span method here because the array is partially modified by the code, which would invalidate a span. // We use the non-span method here because the array is partially modified by the code, which would invalidate a span.
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); RangeItem<BufferModifiedRange>[] overlaps = FindOverlapsAsArray(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
{
overlaps.Add(current);
current = current.Next;
}
int rangeCount = overlaps.Count; int rangeCount = overlaps.Length;
if (rangeCount == 0) if (rangeCount == 0)
{ {
@@ -582,7 +552,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
ulong endAddress = address + size; ulong endAddress = address + size;
Lock.EnterWriteLock(); Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); (RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlapsAsNodes(address, size);
if (first is null) if (first is null)
{ {

2
src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs
View File

@@ -122,7 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong originalVa = gpuVa; ulong originalVa = gpuVa;
_virtualRanges.Lock.EnterWriteLock(); _virtualRanges.Lock.EnterWriteLock();
(RangeItem<VirtualRange> first, RangeItem<VirtualRange> last) = _virtualRanges.FindOverlaps(gpuVa, size); (RangeItem<VirtualRange> first, RangeItem<VirtualRange> last) = _virtualRanges.FindOverlapsAsNodes(gpuVa, size);
if (first is not null) if (first is not null)
{ {

4
src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs
View File

@@ -865,7 +865,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
{ {
lock (_threadingLock) lock (_threadingLock)
{ {
thread.ProcessListNode = _threads.AddLast(thread); _threads.AddLast(thread.ProcessListNode);
} }
} }
@@ -1227,7 +1227,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
{ {
thread.Suspend(ThreadSchedState.ThreadPauseFlag); thread.Suspend(ThreadSchedState.ThreadPauseFlag);
thread.Context.RequestInterrupt(); thread.Context.RequestInterrupt();
if (!thread.DebugHalt.WaitOne(TimeSpan.FromMilliseconds(50))) if (!thread.DebugHalt.Wait(TimeSpan.FromMilliseconds(50)))
{ {
Logger.Warning?.Print(LogClass.Kernel, $"Failed to suspend thread {thread.ThreadUid} in time."); Logger.Warning?.Print(LogClass.Kernel, $"Failed to suspend thread {thread.ThreadUid} in time.");
} }

14
src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs
View File

@@ -13,16 +13,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
Monitor.Exit(mutex); Monitor.Exit(mutex);
currentThread.Withholder = threadList;
currentThread.Reschedule(ThreadSchedState.Paused);
currentThread.WithholderNode = threadList.AddLast(currentThread);
if (currentThread.TerminationRequested) if (currentThread.TerminationRequested)
{ {
threadList.Remove(currentThread.WithholderNode);
currentThread.Reschedule(ThreadSchedState.Running); currentThread.Reschedule(ThreadSchedState.Running);
currentThread.Withholder = null; currentThread.Withholder = null;
@@ -31,6 +23,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
} }
else else
{ {
currentThread.Withholder = threadList;
currentThread.Reschedule(ThreadSchedState.Paused);
threadList.AddLast(currentThread.WithholderNode);
if (timeout > 0) if (timeout > 0)
{ {
context.TimeManager.ScheduleFutureInvocation(currentThread, timeout); context.TimeManager.ScheduleFutureInvocation(currentThread, timeout);

2
src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs
View File

@@ -50,7 +50,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
// even if they are not scheduled on guest cores. // even if they are not scheduled on guest cores.
if (currentThread != null && !currentThread.IsSchedulable && currentThread.Context.Running) if (currentThread != null && !currentThread.IsSchedulable && currentThread.Context.Running)
{ {
currentThread.SchedulerWaitEvent.WaitOne(); currentThread.SchedulerWaitEvent.Wait();
} }
} }
} }

8
src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs
View File

@@ -194,7 +194,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return; return;
} }
thread.SiblingsPerCore[core] = SuggestedQueue(prio, core).AddFirst(thread); SuggestedQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]);
_suggestedPrioritiesPerCore[core] |= 1L << prio; _suggestedPrioritiesPerCore[core] |= 1L << prio;
} }
@@ -223,7 +223,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return; return;
} }
thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddLast(thread); ScheduledQueue(prio, core).AddLast(thread.SiblingsPerCore[core]);
_scheduledPrioritiesPerCore[core] |= 1L << prio; _scheduledPrioritiesPerCore[core] |= 1L << prio;
} }
@@ -235,7 +235,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return; return;
} }
thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddFirst(thread); ScheduledQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]);
_scheduledPrioritiesPerCore[core] |= 1L << prio; _scheduledPrioritiesPerCore[core] |= 1L << prio;
} }
@@ -251,7 +251,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
queue.Remove(thread.SiblingsPerCore[core]); queue.Remove(thread.SiblingsPerCore[core]);
thread.SiblingsPerCore[core] = queue.AddLast(thread); queue.AddLast(thread.SiblingsPerCore[core]);
return queue.First.Value; return queue.First.Value;
} }

5
src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs
View File

@@ -318,11 +318,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
if (nextThread == null) if (nextThread == null)
{ {
ActivateIdleThread(); ActivateIdleThread();
currentThread.SchedulerWaitEvent.WaitOne(); currentThread.SchedulerWaitEvent.Wait();
} }
else else
{ {
WaitHandle.SignalAndWait(nextThread.SchedulerWaitEvent, currentThread.SchedulerWaitEvent); nextThread.SchedulerWaitEvent.Set();
currentThread.SchedulerWaitEvent.Wait();
} }
} }
else else

25
src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
View File

@@ -39,9 +39,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
public const int MaxWaitSyncObjects = 64; public const int MaxWaitSyncObjects = 64;
private ManualResetEvent _schedulerWaitEvent; private ManualResetEventSlim _schedulerWaitEvent;
public ManualResetEvent SchedulerWaitEvent => _schedulerWaitEvent; public ManualResetEventSlim SchedulerWaitEvent => _schedulerWaitEvent;
public Thread HostThread { get; private set; } public Thread HostThread { get; private set; }
@@ -93,6 +93,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private LinkedListNode<KThread> _mutexWaiterNode; private LinkedListNode<KThread> _mutexWaiterNode;
private readonly LinkedList<KThread> _pinnedWaiters; private readonly LinkedList<KThread> _pinnedWaiters;
private LinkedListNode<KThread> _pinnedWaiterNode;
public KThread MutexOwner { get; private set; } public KThread MutexOwner { get; private set; }
@@ -135,7 +136,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private readonly Lock _activityOperationLock = new(); private readonly Lock _activityOperationLock = new();
internal readonly ManualResetEvent DebugHalt = new(false); internal readonly ManualResetEventSlim DebugHalt = new(false);
public KThread(KernelContext context) : base(context) public KThread(KernelContext context) : base(context)
{ {
@@ -144,8 +145,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount]; SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
for (int i = 0; i < SiblingsPerCore.Length; i++)
{
SiblingsPerCore[i] = new LinkedListNode<KThread>(this);
}
_mutexWaiters = []; _mutexWaiters = [];
_pinnedWaiters = []; _pinnedWaiters = [];
WithholderNode = new LinkedListNode<KThread>(this);
ProcessListNode = new LinkedListNode<KThread>(this);
_mutexWaiterNode = new LinkedListNode<KThread>(this);
_pinnedWaiterNode = new LinkedListNode<KThread>(this);
} }
public Result Initialize( public Result Initialize(
@@ -631,7 +642,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
break; break;
} }
_pinnedWaiters.AddLast(currentThread); _pinnedWaiters.AddLast(_pinnedWaiterNode);
currentThread.Reschedule(ThreadSchedState.Paused); currentThread.Reschedule(ThreadSchedState.Paused);
} }
@@ -848,7 +859,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return KernelResult.ThreadTerminating; return KernelResult.ThreadTerminating;
} }
_pinnedWaiters.AddLast(currentThread); _pinnedWaiters.AddLast(_pinnedWaiterNode);
currentThread.Reschedule(ThreadSchedState.Paused); currentThread.Reschedule(ThreadSchedState.Paused);
} }
@@ -1262,7 +1273,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
{ {
if (_schedulerWaitEvent == null) if (_schedulerWaitEvent == null)
{ {
ManualResetEvent schedulerWaitEvent = new(false); ManualResetEventSlim schedulerWaitEvent = new(false);
if (Interlocked.Exchange(ref _schedulerWaitEvent, schedulerWaitEvent) == null) if (Interlocked.Exchange(ref _schedulerWaitEvent, schedulerWaitEvent) == null)
{ {
@@ -1277,7 +1288,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private void ThreadStart() private void ThreadStart()
{ {
_schedulerWaitEvent.WaitOne(); _schedulerWaitEvent.Wait();
DebugHalt.Reset(); DebugHalt.Reset();
KernelStatic.SetKernelContext(KernelContext, this); KernelStatic.SetKernelContext(KernelContext, this);

184
src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs
View File

@@ -12,8 +12,8 @@ namespace Ryujinx.Memory.Range
/// <typeparam name="T">Type of the range.</typeparam> /// <typeparam name="T">Type of the range.</typeparam>
public unsafe class NonOverlappingRangeList<T> : RangeListBase<T> where T : class, INonOverlappingRange public unsafe class NonOverlappingRangeList<T> : RangeListBase<T> where T : class, INonOverlappingRange
{ {
private readonly Dictionary<ulong, RangeItem<T>> _quickAccess = new(AddressEqualityComparer.Comparer); // private readonly Dictionary<ulong, RangeItem<T>> _quickAccess = new(AddressEqualityComparer.Comparer);
private readonly Dictionary<ulong, RangeItem<T>> _fastQuickAccess = new(AddressEqualityComparer.Comparer); // private readonly Dictionary<ulong, RangeItem<T>> _fastQuickAccess = new(AddressEqualityComparer.Comparer);
public readonly ReaderWriterLockSlim Lock = new(); public readonly ReaderWriterLockSlim Lock = new();
@@ -45,7 +45,7 @@ namespace Ryujinx.Memory.Range
Insert(index, rangeItem); Insert(index, rangeItem);
_quickAccess.Add(item.Address, rangeItem); // _quickAccess.Add(item.Address, rangeItem);
} }
/// <summary> /// <summary>
@@ -71,15 +71,15 @@ namespace Ryujinx.Memory.Range
Items[index + 1].Previous = rangeItem; Items[index + 1].Previous = rangeItem;
} }
foreach (ulong addr in Items[index].QuickAccessAddresses) // foreach (ulong addr in Items[index].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
Items[index] = rangeItem; Items[index] = rangeItem;
_quickAccess[item.Address] = rangeItem; // _quickAccess[item.Address] = rangeItem;
return true; return true;
} }
@@ -108,18 +108,18 @@ namespace Ryujinx.Memory.Range
Items[index + 1].Previous = rangeItem; Items[index + 1].Previous = rangeItem;
} }
foreach (ulong addr in item.QuickAccessAddresses) // foreach (ulong addr in item.QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
Items[index] = rangeItem; Items[index] = rangeItem;
if (item.Address != rangeItem.Address) // if (item.Address != rangeItem.Address)
_quickAccess.Remove(item.Address); // _quickAccess.Remove(item.Address);
//
_quickAccess[rangeItem.Address] = rangeItem; // _quickAccess[rangeItem.Address] = rangeItem;
return true; return true;
} }
@@ -196,13 +196,13 @@ namespace Ryujinx.Memory.Range
if (index >= 0 && Items[index].Value.Equals(item)) if (index >= 0 && Items[index].Value.Equals(item))
{ {
_quickAccess.Remove(item.Address); // _quickAccess.Remove(item.Address);
//
foreach (ulong addr in Items[index].QuickAccessAddresses) // foreach (ulong addr in Items[index].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
RemoveAt(index); RemoveAt(index);
@@ -232,15 +232,15 @@ namespace Ryujinx.Memory.Range
(int startIndex, int endIndex) = BinarySearchEdges(startItem.Address, endItem.EndAddress); (int startIndex, int endIndex) = BinarySearchEdges(startItem.Address, endItem.EndAddress);
for (int i = startIndex; i < endIndex; i++) // for (int i = startIndex; i < endIndex; i++)
{ // {
_quickAccess.Remove(Items[i].Address); // _quickAccess.Remove(Items[i].Address);
foreach (ulong addr in Items[i].QuickAccessAddresses) // foreach (ulong addr in Items[i].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
} // }
if (endIndex < Count) if (endIndex < Count)
{ {
@@ -279,12 +279,12 @@ namespace Ryujinx.Memory.Range
while (Items[endIndex] is not null && Items[endIndex].Address < address + size) while (Items[endIndex] is not null && Items[endIndex].Address < address + size)
{ {
_quickAccess.Remove(Items[endIndex].Address); // _quickAccess.Remove(Items[endIndex].Address);
foreach (ulong addr in Items[endIndex].QuickAccessAddresses) // foreach (ulong addr in Items[endIndex].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
if (endIndex == Count - 1) if (endIndex == Count - 1)
{ {
@@ -321,9 +321,9 @@ namespace Ryujinx.Memory.Range
{ {
Lock.EnterWriteLock(); Lock.EnterWriteLock();
Count = 0; Count = 0;
_quickAccess.Clear();
_fastQuickAccess.Clear();
Lock.ExitWriteLock(); Lock.ExitWriteLock();
// _quickAccess.Clear();
// _fastQuickAccess.Clear();
} }
/// <summary> /// <summary>
@@ -344,7 +344,7 @@ namespace Ryujinx.Memory.Range
// So we need to return both the split 0-1 and 1-2 ranges. // So we need to return both the split 0-1 and 1-2 ranges.
Lock.EnterWriteLock(); Lock.EnterWriteLock();
(RangeItem<T> first, RangeItem<T> last) = FindOverlaps(address, size); (RangeItem<T> first, RangeItem<T> last) = FindOverlapsAsNodes(address, size);
list = new List<T>(); list = new List<T>();
if (first is null) if (first is null)
@@ -436,10 +436,10 @@ namespace Ryujinx.Memory.Range
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public override RangeItem<T> FindOverlap(ulong address, ulong size) public override RangeItem<T> FindOverlap(ulong address, ulong size)
{ {
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap)) // if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap))
{ // {
return overlap; // return overlap;
} // }
int index = BinarySearchLeftEdge(address, address + size); int index = BinarySearchLeftEdge(address, address + size);
@@ -448,11 +448,11 @@ namespace Ryujinx.Memory.Range
return null; return null;
} }
if (Items[index].Address < address) // if (Items[index].Address < address)
{ // {
_quickAccess.TryAdd(address, Items[index]); // _quickAccess.TryAdd(address, Items[index]);
Items[index].QuickAccessAddresses.Add(address); // Items[index].QuickAccessAddresses.Add(address);
} // }
return Items[index]; return Items[index];
} }
@@ -466,10 +466,10 @@ namespace Ryujinx.Memory.Range
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public override RangeItem<T> FindOverlapFast(ulong address, ulong size) public override RangeItem<T> FindOverlapFast(ulong address, ulong size)
{ {
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) // if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap) || _fastQuickAccess.TryGetValue(address, out overlap))
{ // {
return overlap; // return overlap;
} // }
int index = BinarySearch(address, address + size); int index = BinarySearch(address, address + size);
@@ -478,16 +478,16 @@ namespace Ryujinx.Memory.Range
return null; return null;
} }
if (Items[index].Address < address) // if (Items[index].Address < address)
{ // {
_quickAccess.TryAdd(address, Items[index]); // _quickAccess.TryAdd(address, Items[index]);
} // }
else // else
{ // {
_fastQuickAccess.TryAdd(address, Items[index]); // _fastQuickAccess.TryAdd(address, Items[index]);
} // }
//
Items[index].QuickAccessAddresses.Add(address); // Items[index].QuickAccessAddresses.Add(address);
return Items[index]; return Items[index];
} }
@@ -499,18 +499,8 @@ namespace Ryujinx.Memory.Range
/// <param name="size">Size in bytes of the range</param> /// <param name="size">Size in bytes of the range</param>
/// <returns>The first and last overlapping items, or null if none are found</returns> /// <returns>The first and last overlapping items, or null if none are found</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public (RangeItem<T>, RangeItem<T>) FindOverlaps(ulong address, ulong size) public (RangeItem<T>, RangeItem<T>) FindOverlapsAsNodes(ulong address, ulong size)
{ {
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap))
{
if (overlap.Next is null || overlap.Next.Address >= address + size)
{
return (overlap, overlap);
}
return (overlap, Items[BinarySearchRightEdge(address, address + size)]);
}
(int index, int endIndex) = BinarySearchEdges(address, address + size); (int index, int endIndex) = BinarySearchEdges(address, address + size);
if (index < 0) if (index < 0)
@@ -518,13 +508,45 @@ namespace Ryujinx.Memory.Range
return (null, null); return (null, null);
} }
if (Items[index].Address < address) return (Items[index], Items[endIndex - 1]);
}
public RangeItem<T>[] FindOverlapsAsArray(ulong address, ulong size)
{
(int index, int endIndex) = BinarySearchEdges(address, address + size);
RangeItem<T>[] result;
if (index < 0)
{ {
_quickAccess.TryAdd(address, Items[index]); result = [];
Items[index].QuickAccessAddresses.Add(address); }
else
{
result = new RangeItem<T>[endIndex - index];
Array.Copy(Items, index, result, 0, endIndex - index);
} }
return (Items[index], Items[endIndex - 1]); return result;
}
public Span<RangeItem<T>> FindOverlapsAsSpan(ulong address, ulong size)
{
(int index, int endIndex) = BinarySearchEdges(address, address + size);
Span<RangeItem<T>> result;
if (index < 0)
{
result = [];
}
else
{
result = Items.AsSpan().Slice(index, endIndex - index);
}
return result;
} }
public override IEnumerator<T> GetEnumerator() public override IEnumerator<T> GetEnumerator()

34
src/Ryujinx.Memory/Tracking/MemoryTracking.cs
View File

@@ -1,4 +1,5 @@
using Ryujinx.Memory.Range; using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic; using System.Collections.Generic;
namespace Ryujinx.Memory.Tracking namespace Ryujinx.Memory.Tracking
@@ -79,12 +80,10 @@ namespace Ryujinx.Memory.Tracking
{ {
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions; NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock(); regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(va, size); Span<RangeItem<VirtualRegion>> overlaps = regions.FindOverlapsAsSpan(va, size);
for (int i = 0; i < overlaps.Length; i++)
RangeItem<VirtualRegion> current = first;
while (last != null && current != last.Next)
{ {
VirtualRegion region = current.Value; VirtualRegion region = overlaps[i].Value;
// If the region has been fully remapped, signal that it has been mapped again. // If the region has been fully remapped, signal that it has been mapped again.
bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size);
@@ -94,7 +93,6 @@ namespace Ryujinx.Memory.Tracking
} }
region.UpdateProtection(); region.UpdateProtection();
current = current.Next;
} }
regions.Lock.ExitReadLock(); regions.Lock.ExitReadLock();
} }
@@ -118,15 +116,11 @@ namespace Ryujinx.Memory.Tracking
{ {
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions; NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock(); regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(va, size); Span<RangeItem<VirtualRegion>> overlaps = regions.FindOverlapsAsSpan(va, size);
RangeItem<VirtualRegion> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
VirtualRegion region = current.Value; overlaps[i].Value.SignalMappingChanged(false);
region.SignalMappingChanged(false);
current = current.Next;
} }
regions.Lock.ExitReadLock(); regions.Lock.ExitReadLock();
} }
@@ -303,21 +297,13 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock) lock (TrackingLock)
{ {
NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions; NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions;
List<RangeItem<VirtualRegion>> overlaps = [];
// We use the non-span method here because keeping the lock will cause a deadlock. // We use the non-span method here because keeping the lock will cause a deadlock.
regions.Lock.EnterReadLock(); regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(address, size); RangeItem<VirtualRegion>[] overlaps = regions.FindOverlapsAsArray(address, size);
RangeItem<VirtualRegion> current = first;
while (last != null && current != last.Next)
{
overlaps.Add(current);
current = current.Next;
}
regions.Lock.ExitReadLock(); regions.Lock.ExitReadLock();
if (first is null && !precise) if (overlaps.Length == 0 && !precise)
{ {
if (_memoryManager.IsRangeMapped(address, size)) if (_memoryManager.IsRangeMapped(address, size))
{ {
@@ -338,7 +324,7 @@ namespace Ryujinx.Memory.Tracking
size += (ulong)_pageSize; size += (ulong)_pageSize;
} }
for (int i = 0; i < overlaps.Count; i++) for (int i = 0; i < overlaps.Length; i++)
{ {
VirtualRegion region = overlaps[i].Value; VirtualRegion region = overlaps[i].Value;