|
|
|
@ -30,28 +30,26 @@ static void IncrementScheduledCount(Kernel::KThread* thread) { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
KScheduler::KScheduler(KernelCore& kernel) : m_kernel{kernel} { |
|
|
|
m_switch_fiber = std::make_shared<Common::Fiber>([this] { |
|
|
|
KScheduler::KScheduler(KernelCore& kernel) { |
|
|
|
m_switch_fiber = std::make_shared<Common::Fiber>([this, &kernel] { |
|
|
|
while (true) { |
|
|
|
ScheduleImplFiber(); |
|
|
|
ScheduleImplFiber(kernel); |
|
|
|
} |
|
|
|
}); |
|
|
|
|
|
|
|
m_state.needs_scheduling = true; |
|
|
|
} |
|
|
|
|
|
|
|
KScheduler::~KScheduler() = default; |
|
|
|
|
|
|
|
void KScheduler::SetInterruptTaskRunnable() { |
|
|
|
void KScheduler::SetInterruptTaskRunnable(KernelCore& kernel) { |
|
|
|
m_state.interrupt_task_runnable = true; |
|
|
|
m_state.needs_scheduling = true; |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::RequestScheduleOnInterrupt() { |
|
|
|
void KScheduler::RequestScheduleOnInterrupt(KernelCore& kernel) { |
|
|
|
m_state.needs_scheduling = true; |
|
|
|
|
|
|
|
if (CanSchedule(m_kernel)) { |
|
|
|
ScheduleOnInterrupt(); |
|
|
|
if (CanSchedule(kernel)) { |
|
|
|
ScheduleOnInterrupt(kernel); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@ -71,12 +69,12 @@ void KScheduler::EnableScheduling(KernelCore& kernel, u64 cores_needing_scheduli |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
scheduler->RescheduleOtherCores(cores_needing_scheduling); |
|
|
|
scheduler->RescheduleOtherCores(kernel, cores_needing_scheduling); |
|
|
|
|
|
|
|
if (GetCurrentThread(kernel).GetDisableDispatchCount() > 1) { |
|
|
|
GetCurrentThread(kernel).EnableDispatch(kernel); |
|
|
|
} else { |
|
|
|
scheduler->RescheduleCurrentCore(); |
|
|
|
scheduler->RescheduleCurrentCore(kernel); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@ -99,55 +97,55 @@ u64 KScheduler::UpdateHighestPriorityThreads(KernelCore& kernel) { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::Schedule() { |
|
|
|
ASSERT(GetCurrentThread(m_kernel).GetDisableDispatchCount() == 1); |
|
|
|
ASSERT(m_core_id == GetCurrentCoreId(m_kernel)); |
|
|
|
void KScheduler::Schedule(KernelCore& kernel) { |
|
|
|
ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() == 1); |
|
|
|
ASSERT(m_core_id == GetCurrentCoreId(kernel)); |
|
|
|
|
|
|
|
ScheduleImpl(); |
|
|
|
ScheduleImpl(kernel); |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::ScheduleOnInterrupt() { |
|
|
|
GetCurrentThread(m_kernel).DisableDispatch(m_kernel); |
|
|
|
Schedule(); |
|
|
|
GetCurrentThread(m_kernel).EnableDispatch(m_kernel); |
|
|
|
void KScheduler::ScheduleOnInterrupt(KernelCore& kernel) { |
|
|
|
GetCurrentThread(kernel).DisableDispatch(kernel); |
|
|
|
Schedule(kernel); |
|
|
|
GetCurrentThread(kernel).EnableDispatch(kernel); |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::PreemptSingleCore() { |
|
|
|
GetCurrentThread(m_kernel).DisableDispatch(m_kernel); |
|
|
|
void KScheduler::PreemptSingleCore(KernelCore& kernel) { |
|
|
|
GetCurrentThread(kernel).DisableDispatch(kernel); |
|
|
|
|
|
|
|
auto* thread = GetCurrentThreadPointer(m_kernel); |
|
|
|
auto& previous_scheduler = m_kernel.Scheduler(thread->GetCurrentCore()); |
|
|
|
previous_scheduler.Unload(thread); |
|
|
|
auto* thread = GetCurrentThreadPointer(kernel); |
|
|
|
auto& previous_scheduler = kernel.Scheduler(thread->GetCurrentCore()); |
|
|
|
previous_scheduler.Unload(kernel, thread); |
|
|
|
|
|
|
|
Common::Fiber::YieldTo(thread->GetHostContext(), *m_switch_fiber); |
|
|
|
|
|
|
|
GetCurrentThread(m_kernel).EnableDispatch(m_kernel); |
|
|
|
GetCurrentThread(kernel).EnableDispatch(kernel); |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::RescheduleCurrentCore() { |
|
|
|
ASSERT(!m_kernel.IsPhantomModeForSingleCore()); |
|
|
|
ASSERT(GetCurrentThread(m_kernel).GetDisableDispatchCount() == 1); |
|
|
|
void KScheduler::RescheduleCurrentCore(KernelCore& kernel) { |
|
|
|
ASSERT(!kernel.IsPhantomModeForSingleCore()); |
|
|
|
ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() == 1); |
|
|
|
|
|
|
|
GetCurrentThread(m_kernel).EnableDispatch(m_kernel); |
|
|
|
GetCurrentThread(kernel).EnableDispatch(kernel); |
|
|
|
|
|
|
|
if (m_state.needs_scheduling.load()) { |
|
|
|
// Disable interrupts, and then check again if rescheduling is needed.
|
|
|
|
// KScopedInterruptDisable intr_disable;
|
|
|
|
|
|
|
|
m_kernel.CurrentScheduler()->RescheduleCurrentCoreImpl(); |
|
|
|
kernel.CurrentScheduler()->RescheduleCurrentCoreImpl(kernel); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::RescheduleCurrentCoreImpl() { |
|
|
|
void KScheduler::RescheduleCurrentCoreImpl(KernelCore& kernel) { |
|
|
|
// Check that scheduling is needed.
|
|
|
|
if (m_state.needs_scheduling.load()) [[likely]] { |
|
|
|
GetCurrentThread(m_kernel).DisableDispatch(m_kernel); |
|
|
|
Schedule(); |
|
|
|
GetCurrentThread(m_kernel).EnableDispatch(m_kernel); |
|
|
|
GetCurrentThread(kernel).DisableDispatch(kernel); |
|
|
|
Schedule(kernel); |
|
|
|
GetCurrentThread(kernel).EnableDispatch(kernel); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::Initialize(KThread* main_thread, KThread* idle_thread, s32 core_id) { |
|
|
|
void KScheduler::Initialize(KernelCore& kernel, KThread* main_thread, KThread* idle_thread, s32 core_id) { |
|
|
|
// Set core ID/idle thread/interrupt task manager.
|
|
|
|
m_core_id = core_id; |
|
|
|
m_idle_thread = idle_thread; |
|
|
|
@ -156,39 +154,39 @@ void KScheduler::Initialize(KThread* main_thread, KThread* idle_thread, s32 core |
|
|
|
|
|
|
|
// Insert the main thread into the priority queue.
|
|
|
|
// {
|
|
|
|
// KScopedSchedulerLock lk{m_kernel};
|
|
|
|
// GetPriorityQueue(m_kernel).PushBack(GetCurrentThreadPointer(m_kernel));
|
|
|
|
// SetSchedulerUpdateNeeded(m_kernel);
|
|
|
|
// KScopedSchedulerLock lk{kernel};
|
|
|
|
// GetPriorityQueue(kernel).PushBack(GetCurrentThreadPointer(kernel));
|
|
|
|
// SetSchedulerUpdateNeeded(kernel);
|
|
|
|
// }
|
|
|
|
|
|
|
|
// Bind interrupt handler.
|
|
|
|
// kernel.GetInterruptManager().BindHandler(
|
|
|
|
// GetSchedulerInterruptHandler(m_kernel), KInterruptName::Scheduler, m_core_id,
|
|
|
|
// GetSchedulerInterruptHandler(kernel), KInterruptName::Scheduler, m_core_id,
|
|
|
|
// KInterruptController::PriorityLevel::Scheduler, false, false);
|
|
|
|
|
|
|
|
// Set the current thread.
|
|
|
|
m_current_thread = main_thread; |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::Activate() { |
|
|
|
ASSERT(GetCurrentThread(m_kernel).GetDisableDispatchCount() == 1); |
|
|
|
void KScheduler::Activate(KernelCore& kernel) { |
|
|
|
ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() == 1); |
|
|
|
|
|
|
|
// m_state.should_count_idle = KTargetSystem::IsDebugMode();
|
|
|
|
m_is_active = true; |
|
|
|
RescheduleCurrentCore(); |
|
|
|
RescheduleCurrentCore(kernel); |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::OnThreadStart() { |
|
|
|
GetCurrentThread(m_kernel).EnableDispatch(m_kernel); |
|
|
|
void KScheduler::OnThreadStart(KernelCore& kernel) { |
|
|
|
GetCurrentThread(kernel).EnableDispatch(kernel); |
|
|
|
} |
|
|
|
|
|
|
|
u64 KScheduler::UpdateHighestPriorityThread(KThread* highest_thread) { |
|
|
|
u64 KScheduler::UpdateHighestPriorityThread(KernelCore& kernel, KThread* highest_thread) { |
|
|
|
if (KThread* prev_highest_thread = m_state.highest_priority_thread; |
|
|
|
prev_highest_thread != highest_thread) [[likely]] { |
|
|
|
if (prev_highest_thread != nullptr) [[likely]] { |
|
|
|
IncrementScheduledCount(prev_highest_thread); |
|
|
|
prev_highest_thread->SetLastScheduledTick( |
|
|
|
m_kernel.System().CoreTiming().GetClockTicks()); |
|
|
|
kernel.System().CoreTiming().GetClockTicks()); |
|
|
|
} |
|
|
|
if (m_state.should_count_idle) { |
|
|
|
if (highest_thread != nullptr) [[likely]] { |
|
|
|
@ -245,8 +243,7 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) { |
|
|
|
} |
|
|
|
|
|
|
|
top_threads[core_id] = top_thread; |
|
|
|
cores_needing_scheduling |= |
|
|
|
kernel.Scheduler(core_id).UpdateHighestPriorityThread(top_threads[core_id]); |
|
|
|
cores_needing_scheduling |= kernel.Scheduler(core_id).UpdateHighestPriorityThread(kernel, top_threads[core_id]); |
|
|
|
} |
|
|
|
|
|
|
|
// Idle cores are bad. We're going to try to migrate threads to each idle core in turn.
|
|
|
|
@ -274,8 +271,7 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) { |
|
|
|
suggested->SetActiveCore(core_id); |
|
|
|
priority_queue.ChangeCore(suggested_core, suggested); |
|
|
|
top_threads[core_id] = suggested; |
|
|
|
cores_needing_scheduling |= |
|
|
|
kernel.Scheduler(core_id).UpdateHighestPriorityThread(top_threads[core_id]); |
|
|
|
cores_needing_scheduling |= kernel.Scheduler(core_id).UpdateHighestPriorityThread(kernel, top_threads[core_id]); |
|
|
|
break; |
|
|
|
} |
|
|
|
|
|
|
|
@ -298,17 +294,13 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) { |
|
|
|
// The candidate core can run some other thread! We'll migrate its current
|
|
|
|
// top thread to us.
|
|
|
|
top_threads[candidate_core] = next_on_candidate_core; |
|
|
|
cores_needing_scheduling |= |
|
|
|
kernel.Scheduler(candidate_core) |
|
|
|
.UpdateHighestPriorityThread(top_threads[candidate_core]); |
|
|
|
cores_needing_scheduling |= kernel.Scheduler(candidate_core).UpdateHighestPriorityThread(kernel, top_threads[candidate_core]); |
|
|
|
|
|
|
|
// Perform the migration.
|
|
|
|
suggested->SetActiveCore(core_id); |
|
|
|
priority_queue.ChangeCore(candidate_core, suggested); |
|
|
|
top_threads[core_id] = suggested; |
|
|
|
cores_needing_scheduling |= |
|
|
|
kernel.Scheduler(core_id).UpdateHighestPriorityThread( |
|
|
|
top_threads[core_id]); |
|
|
|
cores_needing_scheduling |= kernel.Scheduler(core_id).UpdateHighestPriorityThread(kernel, top_threads[core_id]); |
|
|
|
break; |
|
|
|
} |
|
|
|
} |
|
|
|
@ -331,9 +323,9 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) { |
|
|
|
return cores_needing_scheduling; |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::SwitchThread(KThread* next_thread) { |
|
|
|
KProcess* const cur_process = GetCurrentProcessPointer(m_kernel); |
|
|
|
KThread* const cur_thread = GetCurrentThreadPointer(m_kernel); |
|
|
|
void KScheduler::SwitchThread(KernelCore& kernel, KThread* next_thread) { |
|
|
|
KProcess* const cur_process = GetCurrentProcessPointer(kernel); |
|
|
|
KThread* const cur_thread = GetCurrentThreadPointer(kernel); |
|
|
|
|
|
|
|
// We never want to schedule a null thread, so use the idle thread if we don't have a next.
|
|
|
|
if (next_thread == nullptr) { |
|
|
|
@ -355,7 +347,7 @@ void KScheduler::SwitchThread(KThread* next_thread) { |
|
|
|
|
|
|
|
// Update the CPU time tracking variables.
|
|
|
|
const s64 prev_tick = m_last_context_switch_time; |
|
|
|
const s64 cur_tick = m_kernel.System().CoreTiming().GetClockTicks(); |
|
|
|
const s64 cur_tick = kernel.System().CoreTiming().GetClockTicks(); |
|
|
|
const s64 tick_diff = cur_tick - prev_tick; |
|
|
|
cur_thread->AddCpuTime(m_core_id, tick_diff); |
|
|
|
if (cur_process != nullptr) { |
|
|
|
@ -379,23 +371,23 @@ void KScheduler::SwitchThread(KThread* next_thread) { |
|
|
|
// }
|
|
|
|
|
|
|
|
// Set the new thread.
|
|
|
|
SetCurrentThread(m_kernel, next_thread); |
|
|
|
SetCurrentThread(kernel, next_thread); |
|
|
|
m_current_thread = next_thread; |
|
|
|
|
|
|
|
// Set the new Thread Local region.
|
|
|
|
// cpu::SwitchThreadLocalRegion(GetInteger(next_thread->GetThreadLocalRegionAddress()));
|
|
|
|
|
|
|
|
// Update the thread's cpu time differential in TLS, if relevant.
|
|
|
|
next_thread->UpdateTlsThreadCpuTime(m_kernel, cur_tick); |
|
|
|
next_thread->UpdateTlsThreadCpuTime(kernel, cur_tick); |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::ScheduleImpl() { |
|
|
|
void KScheduler::ScheduleImpl(KernelCore& kernel) { |
|
|
|
// First, clear the needs scheduling bool.
|
|
|
|
m_state.needs_scheduling.store(false, std::memory_order_relaxed); |
|
|
|
std::atomic_thread_fence(std::memory_order_seq_cst); |
|
|
|
|
|
|
|
// Load the appropriate thread pointers for scheduling.
|
|
|
|
KThread* const cur_thread{GetCurrentThreadPointer(m_kernel)}; |
|
|
|
KThread* const cur_thread{GetCurrentThreadPointer(kernel)}; |
|
|
|
KThread* highest_priority_thread{m_state.highest_priority_thread}; |
|
|
|
|
|
|
|
// Check whether there are runnable interrupt tasks.
|
|
|
|
@ -423,7 +415,7 @@ void KScheduler::ScheduleImpl() { |
|
|
|
// Returning from ScheduleImpl occurs after this thread has been scheduled again.
|
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::ScheduleImplFiber() { |
|
|
|
void KScheduler::ScheduleImplFiber(KernelCore& kernel) { |
|
|
|
KThread* const cur_thread{m_switch_cur_thread}; |
|
|
|
KThread* highest_priority_thread{m_switch_highest_priority_thread}; |
|
|
|
|
|
|
|
@ -437,7 +429,7 @@ void KScheduler::ScheduleImplFiber() { |
|
|
|
m_switch_from_schedule = false; |
|
|
|
|
|
|
|
// Save the original thread context.
|
|
|
|
Unload(cur_thread); |
|
|
|
Unload(kernel, cur_thread); |
|
|
|
|
|
|
|
// The current thread's context has been entirely taken care of.
|
|
|
|
// Now we want to loop until we successfully switch the thread context.
|
|
|
|
@ -468,7 +460,7 @@ void KScheduler::ScheduleImplFiber() { |
|
|
|
|
|
|
|
// It's time to switch the thread.
|
|
|
|
// Switch to the highest priority thread.
|
|
|
|
SwitchThread(highest_priority_thread); |
|
|
|
SwitchThread(kernel, highest_priority_thread); |
|
|
|
|
|
|
|
// Check if we need scheduling. If we do, then we can't complete the switch and should
|
|
|
|
// retry.
|
|
|
|
@ -493,14 +485,14 @@ void KScheduler::ScheduleImplFiber() { |
|
|
|
} |
|
|
|
|
|
|
|
// Reload the guest thread context.
|
|
|
|
Reload(highest_priority_thread); |
|
|
|
Reload(kernel, highest_priority_thread); |
|
|
|
|
|
|
|
// Reload the host thread.
|
|
|
|
Common::Fiber::YieldTo(m_switch_fiber, *highest_priority_thread->m_host_context); |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::Unload(KThread* thread) { |
|
|
|
m_kernel.PhysicalCore(m_core_id).SaveContext(thread); |
|
|
|
void KScheduler::Unload(KernelCore& kernel, KThread* thread) { |
|
|
|
kernel.PhysicalCore(m_core_id).SaveContext(thread); |
|
|
|
|
|
|
|
// Check if the thread is terminated by checking the DPC flags.
|
|
|
|
if ((thread->GetStackParameters().dpc_flags & static_cast<u32>(DpcFlag::Terminated)) == 0) { |
|
|
|
@ -509,8 +501,8 @@ void KScheduler::Unload(KThread* thread) { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::Reload(KThread* thread) { |
|
|
|
m_kernel.PhysicalCore(m_core_id).LoadContext(thread); |
|
|
|
void KScheduler::Reload(KernelCore& kernel, KThread* thread) { |
|
|
|
kernel.PhysicalCore(m_core_id).LoadContext(thread); |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::ClearPreviousThread(KernelCore& kernel, KThread* thread) { |
|
|
|
@ -877,9 +869,9 @@ void KScheduler::YieldToAnyThread(KernelCore& kernel) { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
void KScheduler::RescheduleOtherCores(u64 cores_needing_scheduling) { |
|
|
|
void KScheduler::RescheduleOtherCores(KernelCore& kernel, u64 cores_needing_scheduling) { |
|
|
|
if (const u64 core_mask = cores_needing_scheduling & ~(1ULL << m_core_id); core_mask != 0) { |
|
|
|
RescheduleCores(m_kernel, core_mask); |
|
|
|
RescheduleCores(kernel, core_mask); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|