From a60b2a0ba366f4b22b942d47fdf5de4110d3e533 Mon Sep 17 00:00:00 2001 From: LotP <22-lotp@users.noreply.git.ryujinx.app> Date: Sat, 6 Sep 2025 11:10:55 -0500 Subject: [PATCH] Memory changes 2.2 (ryubing/ryujinx!143) See merge request ryubing/ryujinx!143 --- src/Ryujinx.Graphics.Device/DeviceState.cs | 25 ++- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 12 +- .../Memory/BufferBackingState.cs | 11 +- .../Memory/BufferCache.cs | 95 +++++---- .../Memory/BufferModifiedRangeList.cs | 60 ++---- .../Memory/VirtualRangeCache.cs | 2 +- .../HOS/Kernel/Process/KProcess.cs | 4 +- .../Kernel/Threading/KConditionVariable.cs | 14 +- .../HOS/Kernel/Threading/KCriticalSection.cs | 2 +- .../HOS/Kernel/Threading/KPriorityQueue.cs | 8 +- .../HOS/Kernel/Threading/KScheduler.cs | 5 +- .../HOS/Kernel/Threading/KThread.cs | 25 ++- .../Range/NonOverlappingRangeList.cs | 184 ++++++++++-------- src/Ryujinx.Memory/Tracking/MemoryTracking.cs | 34 +--- 14 files changed, 242 insertions(+), 239 deletions(-) diff --git a/src/Ryujinx.Graphics.Device/DeviceState.cs b/src/Ryujinx.Graphics.Device/DeviceState.cs index f0b9da894..98a0b7072 100644 --- a/src/Ryujinx.Graphics.Device/DeviceState.cs +++ b/src/Ryujinx.Graphics.Device/DeviceState.cs @@ -97,7 +97,7 @@ namespace Ryujinx.Graphics.Device uint alignedOffset = index * RegisterSize; DebugWrite(alignedOffset, data); - GetRefIntAlignedUncheck(index) = data; + SetIntAlignedUncheck(index, data); _writeCallbacks[index]?.Invoke(data); } @@ -112,9 +112,7 @@ namespace Ryujinx.Graphics.Device uint alignedOffset = index * RegisterSize; DebugWrite(alignedOffset, data); - ref int storage = ref GetRefIntAlignedUncheck(index); - changed = storage != data; - storage = data; + changed = SetIntAlignedUncheckChanged(index, data); _writeCallbacks[index]?.Invoke(data); } @@ -154,5 +152,24 @@ namespace Ryujinx.Graphics.Device { return ref Unsafe.Add(ref Unsafe.As(ref State), (nint)index); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetIntAlignedUncheck(ulong index, int data) + { + Unsafe.Add(ref Unsafe.As(ref State), (nint)index) = data; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool SetIntAlignedUncheckChanged(ulong index, int data) + { + ref int val = ref Unsafe.Add(ref Unsafe.As(ref State), (nint)index); + if (val == data) + { + return false; + } + val = data; + + return true; + } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index 293f7fd2c..c4b848035 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong size, BufferStage stage, bool sparseCompatible, - List baseBuffers) + RangeItem[] baseBuffers) { _context = context; _physicalMemory = physicalMemory; @@ -128,18 +128,18 @@ namespace Ryujinx.Graphics.Gpu.Memory List baseHandles = null; - if (baseBuffers.Count != 0) + if (baseBuffers.Length != 0) { baseHandles = new List(); - foreach (Buffer buffer in baseBuffers) + foreach (RangeItem item in baseBuffers) { - if (buffer._useGranular) + if (item.Value._useGranular) { - baseHandles.AddRange((buffer._memoryTrackingGranular.GetHandles())); + baseHandles.AddRange((item.Value._memoryTrackingGranular.GetHandles())); } else { - baseHandles.Add(buffer._memoryTracking); + baseHandles.Add(item.Value._memoryTracking); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs index df130bb1d..186bea77b 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs @@ -1,4 +1,5 @@ using Ryujinx.Graphics.GAL; +using Ryujinx.Memory.Range; using System; using System.Collections.Generic; @@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Parent buffer /// Initial buffer stage /// Buffers to inherit state from - public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, List baseBuffers) + public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, RangeItem[] baseBuffers) { _size = (int)parent.Size; _systemMemoryType = context.Capabilities.MemoryType; @@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory BufferStage storageFlags = stage & BufferStage.StorageMask; - if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Count == 0) + if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Length == 0) { _desiredType = BufferBackingType.DeviceMemory; } @@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory // TODO: Might be nice to force atomic access to be device local for any stage. } - if (baseBuffers.Count != 0) + if (baseBuffers.Length != 0) { - foreach (Buffer buffer in baseBuffers) + foreach (RangeItem item in baseBuffers) { - CombineState(buffer.BackingState); + CombineState(item.Value.BackingState); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index 8d0bf9833..b55de8962 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -81,13 +81,11 @@ namespace Ryujinx.Graphics.Gpu.Memory MemoryRange subRange = range.GetSubRange(index); _buffers.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(subRange.Address, subRange.Size); + Span> overlaps = _buffers.FindOverlapsAsSpan(subRange.Address, subRange.Size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - current.Value.Unmapped(subRange.Address, subRange.Size); - current = current.Next; + overlaps[i].Value.Unmapped(subRange.Address, subRange.Size); } _buffers.Lock.ExitReadLock(); @@ -490,9 +488,9 @@ namespace Ryujinx.Graphics.Gpu.Memory private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) { _buffers.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); + Span> overlaps = _buffers.FindOverlapsAsSpan(address, size); - if (first is not null) + if (overlaps.Length > 0) { // The buffer already exists. We can just return the existing buffer // if the buffer we need is fully contained inside the overlapping buffer. @@ -502,7 +500,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong endAddress = address + size; - if (first.Address > address || first.EndAddress < endAddress) + if (overlaps[0].Address > address || overlaps[0].EndAddress < endAddress) { bool anySparseCompatible = false; @@ -515,39 +513,40 @@ namespace Ryujinx.Graphics.Gpu.Memory // sequential memory. // Allowing for 2 pages (rather than just one) is necessary to catch cases where the // range crosses a page, and after alignment, ends having a size of 2 pages. - if (first == last && - address >= first.Address && - endAddress - first.EndAddress <= BufferAlignmentSize * 2) + if (overlaps.Length == 1 && + address >= overlaps[0].Address && + endAddress - overlaps[0].EndAddress <= BufferAlignmentSize * 2) { // Try to grow the buffer by 1.5x of its current size. // This improves performance in the cases where the buffer is resized often by small amounts. - ulong existingSize = first.Value.Size; + ulong existingSize = overlaps[0].Value.Size; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; size = Math.Max(size, growthSize); endAddress = address + size; - (first, last) = _buffers.FindOverlaps(address, size); + overlaps = _buffers.FindOverlapsAsSpan(address, size); } - address = Math.Min(address, first.Address); - endAddress = Math.Max(endAddress, last.EndAddress); + address = Math.Min(address, overlaps[0].Address); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); - List overlaps = []; + RangeItem[] overlapsArray = overlaps.ToArray(); - RangeItem current = first; - while (current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - anySparseCompatible |= current.Value.SparseCompatible; - overlaps.Add(current.Value); - _buffers.Remove(current.Value); - - current = current.Next; + anySparseCompatible |= overlaps[i].Value.SparseCompatible; } + _buffers.RemoveRange(overlaps[0], overlaps[^1]); + + _buffers.Lock.ExitWriteLock(); + ulong newSize = endAddress - address; - Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlapsArray); + + _buffers.Lock.EnterWriteLock(); _buffers.Add(newBuffer); } @@ -577,19 +576,19 @@ namespace Ryujinx.Graphics.Gpu.Memory bool sparseAligned = alignment >= SparseBufferAlignmentSize; _buffers.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); + Span> overlaps = _buffers.FindOverlapsAsSpan(address, size); - if (first is not null) + if (overlaps.Length > 0) { // If the buffer already exists, make sure if covers the entire range, // and make sure it is properly aligned, otherwise sparse mapping may fail. ulong endAddress = address + size; - if (first.Address > address || - first.EndAddress < endAddress || - (first.Address & (alignment - 1)) != 0 || - (!first.Value.SparseCompatible && sparseAligned)) + if (overlaps[0].Address > address || + overlaps[0].EndAddress < endAddress || + (overlaps[0].Address & (alignment - 1)) != 0 || + (!overlaps[0].Value.SparseCompatible && sparseAligned)) { // We need to make sure the new buffer is properly aligned. // However, after the range is aligned, it is possible that it @@ -597,33 +596,30 @@ namespace Ryujinx.Graphics.Gpu.Memory // and ensure we cover all overlaps. RangeItem oldFirst; - endAddress = Math.Max(endAddress, last.EndAddress); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); do { - address = Math.Min(address, first.Address); + address = Math.Min(address, overlaps[0].Address); address &= ~(alignment - 1); - oldFirst = first; - (first, last) = _buffers.FindOverlaps(address, endAddress - address); + oldFirst = overlaps[0]; + overlaps = _buffers.FindOverlapsAsSpan(address, endAddress - address); } - while (oldFirst != first); + while (oldFirst != overlaps[0]); ulong newSize = endAddress - address; - List overlaps = []; + RangeItem[] overlapsArray = overlaps.ToArray(); - RangeItem current = first; - while (current != last.Next) - { - overlaps.Add(current.Value); - _buffers.Remove(current.Value); - - current = current.Next; - } + _buffers.RemoveRange(overlaps[0], overlaps[^1]); - Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); + _buffers.Lock.ExitWriteLock(); + + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlapsArray); + + _buffers.Lock.EnterWriteLock(); _buffers.Add(newBuffer); } @@ -635,6 +631,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _buffers.Add(buffer); } + _buffers.Lock.ExitWriteLock(); } @@ -648,13 +645,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers overlapping the range - private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, List overlaps) + private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, RangeItem[] overlaps) { Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); - for (int index = 0; index < overlaps.Count; index++) + for (int index = 0; index < overlaps.Length; index++) { - Buffer buffer = overlaps[index]; + Buffer buffer = overlaps[index].Value; int dstOffset = (int)(buffer.Address - newBuffer.Address); @@ -930,7 +927,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (size != 0) { - buffer = _buffers.FindOverlapFast(address, size).Value; + buffer = _buffers.FindOverlap(address, size).Value; buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index 1c7adf2fc..46a76ed7c 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -80,8 +80,6 @@ namespace Ryujinx.Graphics.Gpu.Memory private BufferMigration _source; private BufferModifiedRangeList _migrationTarget; - - private List> _overlaps; /// /// Whether the modified range list has any entries or not. @@ -108,7 +106,6 @@ namespace Ryujinx.Graphics.Gpu.Memory _context = context; _parent = parent; _flushAction = flushAction; - _overlaps = []; } /// @@ -122,12 +119,11 @@ namespace Ryujinx.Graphics.Gpu.Memory // Slices a given region using the modified regions in the list. Calls the action for the new slices. Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + Span> overlaps = FindOverlapsAsSpan(address, size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - BufferModifiedRange overlap = current.Value; + BufferModifiedRange overlap = overlaps[i].Value; if (overlap.Address > address) { @@ -138,7 +134,6 @@ namespace Ryujinx.Graphics.Gpu.Memory // Remaining region is after this overlap. size -= overlap.EndAddress - address; address = overlap.EndAddress; - current = current.Next; } Lock.ExitReadLock(); @@ -158,12 +153,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size of the modified region in bytes public void SignalModified(ulong address, ulong size) { - // We may overlap with some existing modified regions. They must be cut into by the new entry. - Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - ulong endAddress = address + size; ulong syncNumber = _context.SyncNumber; + // We may overlap with some existing modified regions. They must be cut into by the new entry. + Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = FindOverlapsAsNodes(address, size); if (first is null) { @@ -172,8 +166,6 @@ namespace Ryujinx.Graphics.Gpu.Memory return; } - - if (first == last) { if (first.Address == address && first.EndAddress == endAddress) @@ -260,19 +252,16 @@ namespace Ryujinx.Graphics.Gpu.Memory public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction) { Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + Span> overlaps = FindOverlapsAsSpan(address, size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - BufferModifiedRange overlap = current.Value; + BufferModifiedRange overlap = overlaps[i].Value; if (overlap.SyncNumber == syncNumber) { rangeAction(overlap.Address, overlap.Size); } - - current = current.Next; } Lock.ExitReadLock(); @@ -288,22 +277,12 @@ namespace Ryujinx.Graphics.Gpu.Memory { // We use the non-span method here because keeping the lock will cause a deadlock. Lock.EnterReadLock(); - - _overlaps.Clear(); - - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - - RangeItem current = first; - while (last != null && current != last.Next) - { - _overlaps.Add(current); - current = current.Next; - } + RangeItem[] overlaps = FindOverlapsAsArray(address, size); Lock.ExitReadLock(); - for (int i = 0; i < _overlaps.Count; i++) + for (int i = 0; i < overlaps.Length; i++) { - BufferModifiedRange overlap = _overlaps[i].Value; + BufferModifiedRange overlap = overlaps[i].Value; rangeAction(overlap.Address, overlap.Size); } } @@ -404,8 +383,6 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong endAddress = address + size; ulong currentSync = _context.SyncNumber; - List> overlaps = []; - // Range list must be consistent for this operation if (_migrationTarget != null) { @@ -416,16 +393,9 @@ namespace Ryujinx.Graphics.Gpu.Memory Lock.EnterWriteLock(); // We use the non-span method here because the array is partially modified by the code, which would invalidate a span. - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - - RangeItem current = first; - while (last != null && current != last.Next) - { - overlaps.Add(current); - current = current.Next; - } + RangeItem[] overlaps = FindOverlapsAsArray(address, size); - int rangeCount = overlaps.Count; + int rangeCount = overlaps.Length; if (rangeCount == 0) { @@ -582,7 +552,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { ulong endAddress = address + size; Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + (RangeItem first, RangeItem last) = FindOverlapsAsNodes(address, size); if (first is null) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index 06253cb30..1d44ee65f 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -122,7 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong originalVa = gpuVa; _virtualRanges.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _virtualRanges.FindOverlaps(gpuVa, size); + (RangeItem first, RangeItem last) = _virtualRanges.FindOverlapsAsNodes(gpuVa, size); if (first is not null) { diff --git a/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs b/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs index 0a57f5bc6..54311f229 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs @@ -865,7 +865,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process { lock (_threadingLock) { - thread.ProcessListNode = _threads.AddLast(thread); + _threads.AddLast(thread.ProcessListNode); } } @@ -1227,7 +1227,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process { thread.Suspend(ThreadSchedState.ThreadPauseFlag); thread.Context.RequestInterrupt(); - if (!thread.DebugHalt.WaitOne(TimeSpan.FromMilliseconds(50))) + if (!thread.DebugHalt.Wait(TimeSpan.FromMilliseconds(50))) { Logger.Warning?.Print(LogClass.Kernel, $"Failed to suspend thread {thread.ThreadUid} in time."); } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs index c6aa984c2..cd2e9b543 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs @@ -13,16 +13,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading Monitor.Exit(mutex); - currentThread.Withholder = threadList; - - currentThread.Reschedule(ThreadSchedState.Paused); - - currentThread.WithholderNode = threadList.AddLast(currentThread); - if (currentThread.TerminationRequested) { - threadList.Remove(currentThread.WithholderNode); - currentThread.Reschedule(ThreadSchedState.Running); currentThread.Withholder = null; @@ -31,6 +23,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading } else { + currentThread.Withholder = threadList; + + currentThread.Reschedule(ThreadSchedState.Paused); + + threadList.AddLast(currentThread.WithholderNode); + if (timeout > 0) { context.TimeManager.ScheduleFutureInvocation(currentThread, timeout); diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs index 172f67cca..f5e5367d1 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs @@ -50,7 +50,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading // even if they are not scheduled on guest cores. if (currentThread != null && !currentThread.IsSchedulable && currentThread.Context.Running) { - currentThread.SchedulerWaitEvent.WaitOne(); + currentThread.SchedulerWaitEvent.Wait(); } } } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs index e4b4a370a..e49efca14 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs @@ -194,7 +194,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return; } - thread.SiblingsPerCore[core] = SuggestedQueue(prio, core).AddFirst(thread); + SuggestedQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]); _suggestedPrioritiesPerCore[core] |= 1L << prio; } @@ -223,7 +223,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return; } - thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddLast(thread); + ScheduledQueue(prio, core).AddLast(thread.SiblingsPerCore[core]); _scheduledPrioritiesPerCore[core] |= 1L << prio; } @@ -235,7 +235,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return; } - thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddFirst(thread); + ScheduledQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]); _scheduledPrioritiesPerCore[core] |= 1L << prio; } @@ -251,7 +251,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading queue.Remove(thread.SiblingsPerCore[core]); - thread.SiblingsPerCore[core] = queue.AddLast(thread); + queue.AddLast(thread.SiblingsPerCore[core]); return queue.First.Value; } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs index b8e57619c..93bf7e00a 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs @@ -318,11 +318,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading if (nextThread == null) { ActivateIdleThread(); - currentThread.SchedulerWaitEvent.WaitOne(); + currentThread.SchedulerWaitEvent.Wait(); } else { - WaitHandle.SignalAndWait(nextThread.SchedulerWaitEvent, currentThread.SchedulerWaitEvent); + nextThread.SchedulerWaitEvent.Set(); + currentThread.SchedulerWaitEvent.Wait(); } } else diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs index c799b5e7e..6e0dd906f 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs @@ -39,9 +39,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading public const int MaxWaitSyncObjects = 64; - private ManualResetEvent _schedulerWaitEvent; + private ManualResetEventSlim _schedulerWaitEvent; - public ManualResetEvent SchedulerWaitEvent => _schedulerWaitEvent; + public ManualResetEventSlim SchedulerWaitEvent => _schedulerWaitEvent; public Thread HostThread { get; private set; } @@ -93,6 +93,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading private LinkedListNode _mutexWaiterNode; private readonly LinkedList _pinnedWaiters; + private LinkedListNode _pinnedWaiterNode; public KThread MutexOwner { get; private set; } @@ -135,7 +136,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading private readonly Lock _activityOperationLock = new(); - internal readonly ManualResetEvent DebugHalt = new(false); + internal readonly ManualResetEventSlim DebugHalt = new(false); public KThread(KernelContext context) : base(context) { @@ -144,8 +145,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading SiblingsPerCore = new LinkedListNode[KScheduler.CpuCoresCount]; + for (int i = 0; i < SiblingsPerCore.Length; i++) + { + SiblingsPerCore[i] = new LinkedListNode(this); + } + _mutexWaiters = []; _pinnedWaiters = []; + + WithholderNode = new LinkedListNode(this); + ProcessListNode = new LinkedListNode(this); + _mutexWaiterNode = new LinkedListNode(this); + _pinnedWaiterNode = new LinkedListNode(this); } public Result Initialize( @@ -631,7 +642,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading break; } - _pinnedWaiters.AddLast(currentThread); + _pinnedWaiters.AddLast(_pinnedWaiterNode); currentThread.Reschedule(ThreadSchedState.Paused); } @@ -848,7 +859,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return KernelResult.ThreadTerminating; } - _pinnedWaiters.AddLast(currentThread); + _pinnedWaiters.AddLast(_pinnedWaiterNode); currentThread.Reschedule(ThreadSchedState.Paused); } @@ -1262,7 +1273,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading { if (_schedulerWaitEvent == null) { - ManualResetEvent schedulerWaitEvent = new(false); + ManualResetEventSlim schedulerWaitEvent = new(false); if (Interlocked.Exchange(ref _schedulerWaitEvent, schedulerWaitEvent) == null) { @@ -1277,7 +1288,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading private void ThreadStart() { - _schedulerWaitEvent.WaitOne(); + _schedulerWaitEvent.Wait(); DebugHalt.Reset(); KernelStatic.SetKernelContext(KernelContext, this); diff --git a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs index 76513c9ef..2d0f36c72 100644 --- a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs +++ b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs @@ -12,8 +12,8 @@ namespace Ryujinx.Memory.Range /// Type of the range. public unsafe class NonOverlappingRangeList : RangeListBase where T : class, INonOverlappingRange { - private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); - private readonly Dictionary> _fastQuickAccess = new(AddressEqualityComparer.Comparer); + // private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); + // private readonly Dictionary> _fastQuickAccess = new(AddressEqualityComparer.Comparer); public readonly ReaderWriterLockSlim Lock = new(); @@ -45,7 +45,7 @@ namespace Ryujinx.Memory.Range Insert(index, rangeItem); - _quickAccess.Add(item.Address, rangeItem); + // _quickAccess.Add(item.Address, rangeItem); } /// @@ -71,15 +71,15 @@ namespace Ryujinx.Memory.Range Items[index + 1].Previous = rangeItem; } - foreach (ulong addr in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // foreach (ulong addr in Items[index].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } Items[index] = rangeItem; - _quickAccess[item.Address] = rangeItem; + // _quickAccess[item.Address] = rangeItem; return true; } @@ -108,18 +108,18 @@ namespace Ryujinx.Memory.Range Items[index + 1].Previous = rangeItem; } - foreach (ulong addr in item.QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // foreach (ulong addr in item.QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } Items[index] = rangeItem; - if (item.Address != rangeItem.Address) - _quickAccess.Remove(item.Address); - - _quickAccess[rangeItem.Address] = rangeItem; + // if (item.Address != rangeItem.Address) + // _quickAccess.Remove(item.Address); + // + // _quickAccess[rangeItem.Address] = rangeItem; return true; } @@ -196,13 +196,13 @@ namespace Ryujinx.Memory.Range if (index >= 0 && Items[index].Value.Equals(item)) { - _quickAccess.Remove(item.Address); - - foreach (ulong addr in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // _quickAccess.Remove(item.Address); + // + // foreach (ulong addr in Items[index].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } RemoveAt(index); @@ -232,15 +232,15 @@ namespace Ryujinx.Memory.Range (int startIndex, int endIndex) = BinarySearchEdges(startItem.Address, endItem.EndAddress); - for (int i = startIndex; i < endIndex; i++) - { - _quickAccess.Remove(Items[i].Address); - foreach (ulong addr in Items[i].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } - } + // for (int i = startIndex; i < endIndex; i++) + // { + // _quickAccess.Remove(Items[i].Address); + // foreach (ulong addr in Items[i].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } + // } if (endIndex < Count) { @@ -279,12 +279,12 @@ namespace Ryujinx.Memory.Range while (Items[endIndex] is not null && Items[endIndex].Address < address + size) { - _quickAccess.Remove(Items[endIndex].Address); - foreach (ulong addr in Items[endIndex].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // _quickAccess.Remove(Items[endIndex].Address); + // foreach (ulong addr in Items[endIndex].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } if (endIndex == Count - 1) { @@ -321,9 +321,9 @@ namespace Ryujinx.Memory.Range { Lock.EnterWriteLock(); Count = 0; - _quickAccess.Clear(); - _fastQuickAccess.Clear(); Lock.ExitWriteLock(); + // _quickAccess.Clear(); + // _fastQuickAccess.Clear(); } /// @@ -344,7 +344,7 @@ namespace Ryujinx.Memory.Range // So we need to return both the split 0-1 and 1-2 ranges. Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + (RangeItem first, RangeItem last) = FindOverlapsAsNodes(address, size); list = new List(); if (first is null) @@ -436,10 +436,10 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] public override RangeItem FindOverlap(ulong address, ulong size) { - if (_quickAccess.TryGetValue(address, out RangeItem overlap)) - { - return overlap; - } + // if (_quickAccess.TryGetValue(address, out RangeItem overlap)) + // { + // return overlap; + // } int index = BinarySearchLeftEdge(address, address + size); @@ -448,11 +448,11 @@ namespace Ryujinx.Memory.Range return null; } - if (Items[index].Address < address) - { - _quickAccess.TryAdd(address, Items[index]); - Items[index].QuickAccessAddresses.Add(address); - } + // if (Items[index].Address < address) + // { + // _quickAccess.TryAdd(address, Items[index]); + // Items[index].QuickAccessAddresses.Add(address); + // } return Items[index]; } @@ -466,10 +466,10 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] public override RangeItem FindOverlapFast(ulong address, ulong size) { - if (_quickAccess.TryGetValue(address, out RangeItem overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) - { - return overlap; - } + // if (_quickAccess.TryGetValue(address, out RangeItem overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) + // { + // return overlap; + // } int index = BinarySearch(address, address + size); @@ -478,16 +478,16 @@ namespace Ryujinx.Memory.Range return null; } - if (Items[index].Address < address) - { - _quickAccess.TryAdd(address, Items[index]); - } - else - { - _fastQuickAccess.TryAdd(address, Items[index]); - } - - Items[index].QuickAccessAddresses.Add(address); + // if (Items[index].Address < address) + // { + // _quickAccess.TryAdd(address, Items[index]); + // } + // else + // { + // _fastQuickAccess.TryAdd(address, Items[index]); + // } + // + // Items[index].QuickAccessAddresses.Add(address); return Items[index]; } @@ -499,18 +499,8 @@ namespace Ryujinx.Memory.Range /// Size in bytes of the range /// The first and last overlapping items, or null if none are found [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (RangeItem, RangeItem) FindOverlaps(ulong address, ulong size) + public (RangeItem, RangeItem) FindOverlapsAsNodes(ulong address, ulong size) { - if (_quickAccess.TryGetValue(address, out RangeItem overlap)) - { - if (overlap.Next is null || overlap.Next.Address >= address + size) - { - return (overlap, overlap); - } - - return (overlap, Items[BinarySearchRightEdge(address, address + size)]); - } - (int index, int endIndex) = BinarySearchEdges(address, address + size); if (index < 0) @@ -518,13 +508,45 @@ namespace Ryujinx.Memory.Range return (null, null); } - if (Items[index].Address < address) + return (Items[index], Items[endIndex - 1]); + } + + public RangeItem[] FindOverlapsAsArray(ulong address, ulong size) + { + (int index, int endIndex) = BinarySearchEdges(address, address + size); + + RangeItem[] result; + + if (index < 0) { - _quickAccess.TryAdd(address, Items[index]); - Items[index].QuickAccessAddresses.Add(address); + result = []; + } + else + { + result = new RangeItem[endIndex - index]; + + Array.Copy(Items, index, result, 0, endIndex - index); } - return (Items[index], Items[endIndex - 1]); + return result; + } + + public Span> FindOverlapsAsSpan(ulong address, ulong size) + { + (int index, int endIndex) = BinarySearchEdges(address, address + size); + + Span> result; + + if (index < 0) + { + result = []; + } + else + { + result = Items.AsSpan().Slice(index, endIndex - index); + } + + return result; } public override IEnumerator GetEnumerator() diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index feee36ead..cd97e6fdf 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -1,4 +1,5 @@ using Ryujinx.Memory.Range; +using System; using System.Collections.Generic; namespace Ryujinx.Memory.Tracking @@ -79,12 +80,10 @@ namespace Ryujinx.Memory.Tracking { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); - - RangeItem current = first; - while (last != null && current != last.Next) + Span> overlaps = regions.FindOverlapsAsSpan(va, size); + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = current.Value; + VirtualRegion region = overlaps[i].Value; // If the region has been fully remapped, signal that it has been mapped again. bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); @@ -94,7 +93,6 @@ namespace Ryujinx.Memory.Tracking } region.UpdateProtection(); - current = current.Next; } regions.Lock.ExitReadLock(); } @@ -118,15 +116,11 @@ namespace Ryujinx.Memory.Tracking { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); + Span> overlaps = regions.FindOverlapsAsSpan(va, size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = current.Value; - - region.SignalMappingChanged(false); - current = current.Next; + overlaps[i].Value.SignalMappingChanged(false); } regions.Lock.ExitReadLock(); } @@ -303,21 +297,13 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; - List> overlaps = []; // We use the non-span method here because keeping the lock will cause a deadlock. regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(address, size); - - RangeItem current = first; - while (last != null && current != last.Next) - { - overlaps.Add(current); - current = current.Next; - } + RangeItem[] overlaps = regions.FindOverlapsAsArray(address, size); regions.Lock.ExitReadLock(); - if (first is null && !precise) + if (overlaps.Length == 0 && !precise) { if (_memoryManager.IsRangeMapped(address, size)) { @@ -338,7 +324,7 @@ namespace Ryujinx.Memory.Tracking size += (ulong)_pageSize; } - for (int i = 0; i < overlaps.Count; i++) + for (int i = 0; i < overlaps.Length; i++) { VirtualRegion region = overlaps[i].Value;