github
/
yuzu
mirror of https://github.com/yuzu-emu/yuzu
1
Fork 0
Code Releases Activity
yuzu/src/core/hle/kernel/kernel.cpp

1413 lines
55 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <array>
#include <atomic>
#include <bitset>
#include <functional>
#include <memory>
#include <thread>
#include <unordered_set>
#include <utility>
#include "common/assert.h"
#include "common/logging/log.h"
#include "common/microprofile.h"
#include "common/scope_exit.h"
#include "common/thread.h"
#include "common/thread_worker.h"
#include "core/arm/arm_interface.h"
#include "core/arm/exclusive_monitor.h"
#include "core/core.h"
#include "core/core_timing.h"
#include "core/cpu_manager.h"
#include "core/hardware_properties.h"
#include "core/hle/kernel/init/init_slab_setup.h"
#include "core/hle/kernel/k_client_port.h"
#include "core/hle/kernel/k_dynamic_resource_manager.h"
#include "core/hle/kernel/k_handle_table.h"
#include "core/hle/kernel/k_hardware_timer.h"
#include "core/hle/kernel/k_memory_layout.h"
#include "core/hle/kernel/k_memory_manager.h"
#include "core/hle/kernel/k_object_name.h"
#include "core/hle/kernel/k_page_buffer.h"
#include "core/hle/kernel/k_process.h"
#include "core/hle/kernel/k_resource_limit.h"
#include "core/hle/kernel/k_scheduler.h"
#include "core/hle/kernel/k_scoped_resource_reservation.h"
#include "core/hle/kernel/k_shared_memory.h"
#include "core/hle/kernel/k_system_resource.h"
#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/k_worker_task_manager.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/kernel/physical_core.h"
#include "core/hle/result.h"
#include "core/hle/service/server_manager.h"
#include "core/hle/service/sm/sm.h"
#include "core/memory.h"
MICROPROFILE_DEFINE(Kernel_SVC, "Kernel", "SVC", MP_RGB(70, 200, 70));
namespace Kernel {
struct KernelCore::Impl {
static constexpr size_t ApplicationMemoryBlockSlabHeapSize = 20000;
static constexpr size_t SystemMemoryBlockSlabHeapSize = 10000;
static constexpr size_t BlockInfoSlabHeapSize = 4000;
static constexpr size_t ReservedDynamicPageCount = 64;
explicit Impl(Core::System& system_, KernelCore& kernel_) : system{system_} {}
void SetMulticore(bool is_multi) {
is_multicore = is_multi;
}
void Initialize(KernelCore& kernel) {
hardware_timer = std::make_unique<Kernel::KHardwareTimer>(kernel);
hardware_timer->Initialize();
global_object_list_container = std::make_unique<KAutoObjectWithListContainer>(kernel);
global_scheduler_context = std::make_unique<Kernel::GlobalSchedulerContext>(kernel);
global_handle_table = std::make_unique<Kernel::KHandleTable>(kernel);
global_handle_table->Initialize(KHandleTable::MaxTableSize);
is_phantom_mode_for_singlecore = false;
// Derive the initial memory layout from the emulated board
Init::InitializeSlabResourceCounts(kernel);
DeriveInitialMemoryLayout();
Init::InitializeSlabHeaps(system, *memory_layout);
// Initialize kernel memory and resources.
InitializeSystemResourceLimit(kernel, system.CoreTiming());
InitializeMemoryLayout();
InitializeShutdownThreads();
InitializePhysicalCores();
InitializePreemption(kernel);
InitializeGlobalData(kernel);
// Initialize the Dynamic Slab Heaps.
{
const auto& pt_heap_region = memory_layout->GetPageTableHeapRegion();
ASSERT(pt_heap_region.GetEndAddress() != 0);
InitializeResourceManagers(kernel, pt_heap_region.GetAddress(),
pt_heap_region.GetSize());
}
InitializeHackSharedMemory();
RegisterHostThread(nullptr);
}
void InitializeCores() {
for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) {
cores[core_id]->Initialize((*application_process).Is64BitProcess());
system.ApplicationMemory().SetCurrentPageTable(*application_process, core_id);
}
}
void CloseApplicationProcess() {
KProcess* old_process = application_process.exchange(nullptr);
if (old_process == nullptr) {
return;
}
// old_process->Close();
// TODO: The process should be destroyed based on accurate ref counting after
// calling Close(). Adding a manual Destroy() call instead to avoid a memory leak.
old_process->Finalize();
old_process->Destroy();
}
void Shutdown() {
is_shutting_down.store(true, std::memory_order_relaxed);
SCOPE_EXIT({ is_shutting_down.store(false, std::memory_order_relaxed); });
process_list.clear();
CloseServices();
next_object_id = 0;
next_kernel_process_id = KProcess::InitialKIPIDMin;
next_user_process_id = KProcess::ProcessIDMin;
next_thread_id = 1;
global_handle_table->Finalize();
global_handle_table.reset();
preemption_event = nullptr;
exclusive_monitor.reset();
// Cleanup persistent kernel objects
auto CleanupObject = [](KAutoObject* obj) {
if (obj) {
obj->Close();
obj = nullptr;
}
};
CleanupObject(hid_shared_mem);
CleanupObject(font_shared_mem);
CleanupObject(irs_shared_mem);
CleanupObject(time_shared_mem);
CleanupObject(hidbus_shared_mem);
CleanupObject(system_resource_limit);
for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) {
if (shutdown_threads[core_id]) {
shutdown_threads[core_id]->Close();
shutdown_threads[core_id] = nullptr;
}
schedulers[core_id].reset();
}
// Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others
next_host_thread_id = Core::Hardware::NUM_CPU_CORES;
// Close kernel objects that were not freed on shutdown
{
std::scoped_lock lk{registered_in_use_objects_lock};
if (registered_in_use_objects.size()) {
for (auto& object : registered_in_use_objects) {
object->Close();
}
registered_in_use_objects.clear();
}
}
CloseApplicationProcess();
// Track kernel objects that were not freed on shutdown
{
std::scoped_lock lk{registered_objects_lock};
if (registered_objects.size()) {
LOG_DEBUG(Kernel, "{} kernel objects were dangling on shutdown!",
registered_objects.size());
registered_objects.clear();
}
}
object_name_global_data.reset();
// Ensure that the object list container is finalized and properly shutdown.
global_object_list_container->Finalize();
global_object_list_container.reset();
hardware_timer->Finalize();
hardware_timer.reset();
}
void CloseServices() {
// Ensures all servers gracefully shutdown.
std::scoped_lock lk{server_lock};
server_managers.clear();
}
void InitializePhysicalCores() {
exclusive_monitor =
Core::MakeExclusiveMonitor(system.ApplicationMemory(), Core::Hardware::NUM_CPU_CORES);
for (u32 i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
const s32 core{static_cast<s32>(i)};
schedulers[i] = std::make_unique<Kernel::KScheduler>(system.Kernel());
cores[i] = std::make_unique<Kernel::PhysicalCore>(i, system, *schedulers[i]);
auto* main_thread{Kernel::KThread::Create(system.Kernel())};
main_thread->SetCurrentCore(core);
ASSERT(Kernel::KThread::InitializeMainThread(system, main_thread, core).IsSuccess());
auto* idle_thread{Kernel::KThread::Create(system.Kernel())};
idle_thread->SetCurrentCore(core);
ASSERT(Kernel::KThread::InitializeIdleThread(system, idle_thread, core).IsSuccess());
schedulers[i]->Initialize(main_thread, idle_thread, core);
}
}
// Creates the default system resource limit
void InitializeSystemResourceLimit(KernelCore& kernel,
const Core::Timing::CoreTiming& core_timing) {
system_resource_limit = KResourceLimit::Create(system.Kernel());
system_resource_limit->Initialize(&core_timing);
const auto sizes{memory_layout->GetTotalAndKernelMemorySizes()};
const auto total_size{sizes.first};
const auto kernel_size{sizes.second};
// If setting the default system values fails, then something seriously wrong has occurred.
ASSERT(
system_resource_limit->SetLimitValue(LimitableResource::PhysicalMemoryMax, total_size)
.IsSuccess());
ASSERT(system_resource_limit->SetLimitValue(LimitableResource::ThreadCountMax, 800)
.IsSuccess());
ASSERT(system_resource_limit->SetLimitValue(LimitableResource::EventCountMax, 900)
.IsSuccess());
ASSERT(system_resource_limit->SetLimitValue(LimitableResource::TransferMemoryCountMax, 200)
.IsSuccess());
ASSERT(system_resource_limit->SetLimitValue(LimitableResource::SessionCountMax, 1133)
.IsSuccess());
system_resource_limit->Reserve(LimitableResource::PhysicalMemoryMax, kernel_size);
// Reserve secure applet memory, introduced in firmware 5.0.0
constexpr u64 secure_applet_memory_size{4_MiB};
ASSERT(system_resource_limit->Reserve(LimitableResource::PhysicalMemoryMax,
secure_applet_memory_size));
}
void InitializePreemption(KernelCore& kernel) {
preemption_event = Core::Timing::CreateEvent(
"PreemptionCallback",
[this, &kernel](std::uintptr_t, s64 time,
std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> {
{
KScopedSchedulerLock lock(kernel);
global_scheduler_context->PreemptThreads();
}
return std::nullopt;
});
const auto time_interval = std::chrono::nanoseconds{std::chrono::milliseconds(10)};
system.CoreTiming().ScheduleLoopingEvent(time_interval, time_interval, preemption_event);
}
void InitializeResourceManagers(KernelCore& kernel, KVirtualAddress address, size_t size) {
// Ensure that the buffer is suitable for our use.
ASSERT(Common::IsAligned(GetInteger(address), PageSize));
ASSERT(Common::IsAligned(size, PageSize));
// Ensure that we have space for our reference counts.
const size_t rc_size =
Common::AlignUp(KPageTableSlabHeap::CalculateReferenceCountSize(size), PageSize);
ASSERT(rc_size < size);
size -= rc_size;
// Initialize the resource managers' shared page manager.
resource_manager_page_manager = std::make_unique<KDynamicPageManager>();
resource_manager_page_manager->Initialize(
address, size, std::max<size_t>(PageSize, KPageBufferSlabHeap::BufferSize));
// Initialize the KPageBuffer slab heap.
page_buffer_slab_heap.Initialize(system);
// Initialize the fixed-size slabheaps.
app_memory_block_heap = std::make_unique<KMemoryBlockSlabHeap>();
sys_memory_block_heap = std::make_unique<KMemoryBlockSlabHeap>();
block_info_heap = std::make_unique<KBlockInfoSlabHeap>();
app_memory_block_heap->Initialize(resource_manager_page_manager.get(),
ApplicationMemoryBlockSlabHeapSize);
sys_memory_block_heap->Initialize(resource_manager_page_manager.get(),
SystemMemoryBlockSlabHeapSize);
block_info_heap->Initialize(resource_manager_page_manager.get(), BlockInfoSlabHeapSize);
// Reserve all but a fixed number of remaining pages for the page table heap.
const size_t num_pt_pages = resource_manager_page_manager->GetCount() -
resource_manager_page_manager->GetUsed() -
ReservedDynamicPageCount;
page_table_heap = std::make_unique<KPageTableSlabHeap>();
// TODO(bunnei): Pass in address once we support kernel virtual memory allocations.
page_table_heap->Initialize(
resource_manager_page_manager.get(), num_pt_pages,
/*GetPointer<KPageTableManager::RefCount>(address + size)*/ nullptr);
// Setup the slab managers.
KDynamicPageManager* const app_dynamic_page_manager = nullptr;
KDynamicPageManager* const sys_dynamic_page_manager =
/*KTargetSystem::IsDynamicResourceLimitsEnabled()*/ true
? resource_manager_page_manager.get()
: nullptr;
app_memory_block_manager = std::make_unique<KMemoryBlockSlabManager>();
sys_memory_block_manager = std::make_unique<KMemoryBlockSlabManager>();
app_block_info_manager = std::make_unique<KBlockInfoManager>();
sys_block_info_manager = std::make_unique<KBlockInfoManager>();
app_page_table_manager = std::make_unique<KPageTableManager>();
sys_page_table_manager = std::make_unique<KPageTableManager>();
app_memory_block_manager->Initialize(app_dynamic_page_manager, app_memory_block_heap.get());
sys_memory_block_manager->Initialize(sys_dynamic_page_manager, sys_memory_block_heap.get());
app_block_info_manager->Initialize(app_dynamic_page_manager, block_info_heap.get());
sys_block_info_manager->Initialize(sys_dynamic_page_manager, block_info_heap.get());
app_page_table_manager->Initialize(app_dynamic_page_manager, page_table_heap.get());
sys_page_table_manager->Initialize(sys_dynamic_page_manager, page_table_heap.get());
// Check that we have the correct number of dynamic pages available.
ASSERT(resource_manager_page_manager->GetCount() -
resource_manager_page_manager->GetUsed() ==
ReservedDynamicPageCount);
// Create the system page table managers.
app_system_resource = std::make_unique<KSystemResource>(kernel);
sys_system_resource = std::make_unique<KSystemResource>(kernel);
// Set the managers for the system resources.
app_system_resource->SetManagers(*app_memory_block_manager, *app_block_info_manager,
*app_page_table_manager);
sys_system_resource->SetManagers(*sys_memory_block_manager, *sys_block_info_manager,
*sys_page_table_manager);
}
void InitializeShutdownThreads() {
for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) {
shutdown_threads[core_id] = KThread::Create(system.Kernel());
ASSERT(KThread::InitializeHighPriorityThread(system, shutdown_threads[core_id], {}, {},
core_id)
.IsSuccess());
}
}
void InitializeGlobalData(KernelCore& kernel) {
object_name_global_data = std::make_unique<KObjectNameGlobalData>(kernel);
}
void MakeApplicationProcess(KProcess* process) {
application_process = process;
}
static inline thread_local u8 host_thread_id = UINT8_MAX;
/// Sets the host thread ID for the caller.
u32 SetHostThreadId(std::size_t core_id) {
// This should only be called during core init.
ASSERT(host_thread_id == UINT8_MAX);
// The first four slots are reserved for CPU core threads
ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
host_thread_id = static_cast<u8>(core_id);
return host_thread_id;
}
/// Gets the host thread ID for the caller
u32 GetHostThreadId() const {
return host_thread_id;
}
// Gets the dummy KThread for the caller, allocating a new one if this is the first time
KThread* GetHostDummyThread(KThread* existing_thread) {
const auto initialize{[](KThread* thread) {
ASSERT(KThread::InitializeDummyThread(thread, nullptr).IsSuccess());
return thread;
}};
thread_local KThread raw_thread{system.Kernel()};
thread_local KThread* thread = existing_thread ? existing_thread : initialize(&raw_thread);
return thread;
}
/// Registers a CPU core thread by allocating a host thread ID for it
void RegisterCoreThread(std::size_t core_id) {
ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
const auto this_id = SetHostThreadId(core_id);
if (!is_multicore) {
single_core_thread_id = this_id;
}
}
/// Registers a new host thread by allocating a host thread ID for it
void RegisterHostThread(KThread* existing_thread) {
[[maybe_unused]] const auto dummy_thread = GetHostDummyThread(existing_thread);
}
[[nodiscard]] u32 GetCurrentHostThreadID() {
const auto this_id = GetHostThreadId();
if (!is_multicore && single_core_thread_id == this_id) {
return static_cast<u32>(system.GetCpuManager().CurrentCore());
}
return this_id;
}
static inline thread_local bool is_phantom_mode_for_singlecore{false};
bool IsPhantomModeForSingleCore() const {
return is_phantom_mode_for_singlecore;
}
void SetIsPhantomModeForSingleCore(bool value) {
ASSERT(!is_multicore);
is_phantom_mode_for_singlecore = value;
}
bool IsShuttingDown() const {
return is_shutting_down.load(std::memory_order_relaxed);
}
static inline thread_local KThread* current_thread{nullptr};
KThread* GetCurrentEmuThread() {
if (!current_thread) {
current_thread = GetHostDummyThread(nullptr);
}
return current_thread;
}
void SetCurrentEmuThread(KThread* thread) {
current_thread = thread;
}
void DeriveInitialMemoryLayout() {
memory_layout = std::make_unique<KMemoryLayout>();
// Insert the root region for the virtual memory tree, from which all other regions will
// derive.
memory_layout->GetVirtualMemoryRegionTree().InsertDirectly(
KernelVirtualAddressSpaceBase,
KernelVirtualAddressSpaceBase + KernelVirtualAddressSpaceSize - 1);
// Insert the root region for the physical memory tree, from which all other regions will
// derive.
memory_layout->GetPhysicalMemoryRegionTree().InsertDirectly(
KernelPhysicalAddressSpaceBase,
KernelPhysicalAddressSpaceBase + KernelPhysicalAddressSpaceSize - 1);
// Save start and end for ease of use.
constexpr KVirtualAddress code_start_virt_addr = KernelVirtualAddressCodeBase;
constexpr KVirtualAddress code_end_virt_addr = KernelVirtualAddressCodeEnd;
// Setup the containing kernel region.
constexpr size_t KernelRegionSize = 1_GiB;
constexpr size_t KernelRegionAlign = 1_GiB;
constexpr KVirtualAddress kernel_region_start =
Common::AlignDown(GetInteger(code_start_virt_addr), KernelRegionAlign);
size_t kernel_region_size = KernelRegionSize;
if (!(kernel_region_start + KernelRegionSize - 1 <= KernelVirtualAddressSpaceLast)) {
kernel_region_size = KernelVirtualAddressSpaceEnd - GetInteger(kernel_region_start);
}
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(kernel_region_start), kernel_region_size, KMemoryRegionType_Kernel));
// Setup the code region.
constexpr size_t CodeRegionAlign = PageSize;
constexpr KVirtualAddress code_region_start =
Common::AlignDown(GetInteger(code_start_virt_addr), CodeRegionAlign);
constexpr KVirtualAddress code_region_end =
Common::AlignUp(GetInteger(code_end_virt_addr), CodeRegionAlign);
constexpr size_t code_region_size = code_region_end - code_region_start;
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(code_region_start), code_region_size, KMemoryRegionType_KernelCode));
// Setup board-specific device physical regions.
Init::SetupDevicePhysicalMemoryRegions(*memory_layout);
// Determine the amount of space needed for the misc region.
size_t misc_region_needed_size;
{
// Each core has a one page stack for all three stack types (Main, Idle, Exception).
misc_region_needed_size = Core::Hardware::NUM_CPU_CORES * (3 * (PageSize + PageSize));
// Account for each auto-map device.
for (const auto& region : memory_layout->GetPhysicalMemoryRegionTree()) {
if (region.HasTypeAttribute(KMemoryRegionAttr_ShouldKernelMap)) {
// Check that the region is valid.
ASSERT(region.GetEndAddress() != 0);
// Account for the region.
misc_region_needed_size +=
PageSize + (Common::AlignUp(region.GetLastAddress(), PageSize) -
Common::AlignDown(region.GetAddress(), PageSize));
}
}
// Multiply the needed size by three, to account for the need for guard space.
misc_region_needed_size *= 3;
}
// Decide on the actual size for the misc region.
constexpr size_t MiscRegionAlign = KernelAslrAlignment;
constexpr size_t MiscRegionMinimumSize = 32_MiB;
const size_t misc_region_size = Common::AlignUp(
std::max(misc_region_needed_size, MiscRegionMinimumSize), MiscRegionAlign);
ASSERT(misc_region_size > 0);
// Setup the misc region.
const KVirtualAddress misc_region_start =
memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion(
misc_region_size, MiscRegionAlign, KMemoryRegionType_Kernel);
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(misc_region_start), misc_region_size, KMemoryRegionType_KernelMisc));
// Determine if we'll use extra thread resources.
const bool use_extra_resources = KSystemControl::Init::ShouldIncreaseThreadResourceLimit();
// Setup the stack region.
constexpr size_t StackRegionSize = 14_MiB;
constexpr size_t StackRegionAlign = KernelAslrAlignment;
const KVirtualAddress stack_region_start =
memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion(
StackRegionSize, StackRegionAlign, KMemoryRegionType_Kernel);
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(stack_region_start), StackRegionSize, KMemoryRegionType_KernelStack));
// Determine the size of the resource region.
const size_t resource_region_size =
memory_layout->GetResourceRegionSizeForInit(use_extra_resources);
// Determine the size of the slab region.
const size_t slab_region_size =
Common::AlignUp(Init::CalculateTotalSlabHeapSize(system.Kernel()), PageSize);
ASSERT(slab_region_size <= resource_region_size);
// Setup the slab region.
const KPhysicalAddress code_start_phys_addr = KernelPhysicalAddressCodeBase;
const KPhysicalAddress code_end_phys_addr = code_start_phys_addr + code_region_size;
const KPhysicalAddress slab_start_phys_addr = code_end_phys_addr;
const KPhysicalAddress slab_end_phys_addr = slab_start_phys_addr + slab_region_size;
constexpr size_t SlabRegionAlign = KernelAslrAlignment;
const size_t slab_region_needed_size =
Common::AlignUp(GetInteger(code_end_phys_addr) + slab_region_size, SlabRegionAlign) -
Common::AlignDown(GetInteger(code_end_phys_addr), SlabRegionAlign);
const KVirtualAddress slab_region_start =
memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion(
slab_region_needed_size, SlabRegionAlign, KMemoryRegionType_Kernel) +
(GetInteger(code_end_phys_addr) % SlabRegionAlign);
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(slab_region_start), slab_region_size, KMemoryRegionType_KernelSlab));
// Setup the temp region.
constexpr size_t TempRegionSize = 128_MiB;
constexpr size_t TempRegionAlign = KernelAslrAlignment;
const KVirtualAddress temp_region_start =
memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion(
TempRegionSize, TempRegionAlign, KMemoryRegionType_Kernel);
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(temp_region_start), TempRegionSize, KMemoryRegionType_KernelTemp));
// Automatically map in devices that have auto-map attributes.
for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) {
// We only care about kernel regions.
if (!region.IsDerivedFrom(KMemoryRegionType_Kernel)) {
continue;
}
// Check whether we should map the region.
if (!region.HasTypeAttribute(KMemoryRegionAttr_ShouldKernelMap)) {
continue;
}
// If this region has already been mapped, no need to consider it.
if (region.HasTypeAttribute(KMemoryRegionAttr_DidKernelMap)) {
continue;
}
// Check that the region is valid.
ASSERT(region.GetEndAddress() != 0);
// Set the attribute to note we've mapped this region.
region.SetTypeAttribute(KMemoryRegionAttr_DidKernelMap);
// Create a virtual pair region and insert it into the tree.
const KPhysicalAddress map_phys_addr = Common::AlignDown(region.GetAddress(), PageSize);
const size_t map_size =
Common::AlignUp(region.GetEndAddress(), PageSize) - GetInteger(map_phys_addr);
const KVirtualAddress map_virt_addr =
memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard(
map_size, PageSize, KMemoryRegionType_KernelMisc, PageSize);
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(map_virt_addr), map_size, KMemoryRegionType_KernelMiscMappedDevice));
region.SetPairAddress(GetInteger(map_virt_addr) + region.GetAddress() -
GetInteger(map_phys_addr));
}
Init::SetupDramPhysicalMemoryRegions(*memory_layout);
// Insert a physical region for the kernel code region.
ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert(
GetInteger(code_start_phys_addr), code_region_size, KMemoryRegionType_DramKernelCode));
// Insert a physical region for the kernel slab region.
ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert(
GetInteger(slab_start_phys_addr), slab_region_size, KMemoryRegionType_DramKernelSlab));
// Determine size available for kernel page table heaps, requiring > 8 MB.
const KPhysicalAddress resource_end_phys_addr = slab_start_phys_addr + resource_region_size;
const size_t page_table_heap_size = resource_end_phys_addr - slab_end_phys_addr;
ASSERT(page_table_heap_size / 4_MiB > 2);
// Insert a physical region for the kernel page table heap region
ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert(
GetInteger(slab_end_phys_addr), page_table_heap_size,
KMemoryRegionType_DramKernelPtHeap));
// All DRAM regions that we haven't tagged by this point will be mapped under the linear
// mapping. Tag them.
for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) {
if (region.GetType() == KMemoryRegionType_Dram) {
// Check that the region is valid.
ASSERT(region.GetEndAddress() != 0);
// Set the linear map attribute.
region.SetTypeAttribute(KMemoryRegionAttr_LinearMapped);
}
}
// Get the linear region extents.
const auto linear_extents =
memory_layout->GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(
KMemoryRegionAttr_LinearMapped);
ASSERT(linear_extents.GetEndAddress() != 0);
// Setup the linear mapping region.
constexpr size_t LinearRegionAlign = 1_GiB;
const KPhysicalAddress aligned_linear_phys_start =
Common::AlignDown(linear_extents.GetAddress(), LinearRegionAlign);
const size_t linear_region_size =
Common::AlignUp(linear_extents.GetEndAddress(), LinearRegionAlign) -
GetInteger(aligned_linear_phys_start);
const KVirtualAddress linear_region_start =
memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard(
linear_region_size, LinearRegionAlign, KMemoryRegionType_None, LinearRegionAlign);
const u64 linear_region_phys_to_virt_diff =
GetInteger(linear_region_start) - GetInteger(aligned_linear_phys_start);
// Map and create regions for all the linearly-mapped data.
{
KPhysicalAddress cur_phys_addr = 0;
u64 cur_size = 0;
for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) {
if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) {
continue;
}
ASSERT(region.GetEndAddress() != 0);
if (cur_size == 0) {
cur_phys_addr = region.GetAddress();
cur_size = region.GetSize();
} else if (cur_phys_addr + cur_size == region.GetAddress()) {
cur_size += region.GetSize();
} else {
cur_phys_addr = region.GetAddress();
cur_size = region.GetSize();
}
const KVirtualAddress region_virt_addr =
region.GetAddress() + linear_region_phys_to_virt_diff;
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(region_virt_addr), region.GetSize(),
GetTypeForVirtualLinearMapping(region.GetType())));
region.SetPairAddress(GetInteger(region_virt_addr));
KMemoryRegion* virt_region =
memory_layout->GetVirtualMemoryRegionTree().FindModifiable(
GetInteger(region_virt_addr));
ASSERT(virt_region != nullptr);
virt_region->SetPairAddress(region.GetAddress());
}
}
// Insert regions for the initial page table region.
ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert(
GetInteger(resource_end_phys_addr), KernelPageTableHeapSize,
KMemoryRegionType_DramKernelInitPt));
ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(
GetInteger(resource_end_phys_addr) + linear_region_phys_to_virt_diff,
KernelPageTableHeapSize, KMemoryRegionType_VirtualDramKernelInitPt));
// All linear-mapped DRAM regions that we haven't tagged by this point will be allocated to
// some pool partition. Tag them.
for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) {
if (region.GetType() == (KMemoryRegionType_Dram | KMemoryRegionAttr_LinearMapped)) {
region.SetType(KMemoryRegionType_DramPoolPartition);
}
}
// Setup all other memory regions needed to arrange the pool partitions.
Init::SetupPoolPartitionMemoryRegions(*memory_layout);
// Cache all linear regions in their own trees for faster access, later.
memory_layout->InitializeLinearMemoryRegionTrees(aligned_linear_phys_start,
linear_region_start);
}
void InitializeMemoryLayout() {
// Initialize the memory manager.
memory_manager = std::make_unique<KMemoryManager>(system);
const auto& management_region = memory_layout->GetPoolManagementRegion();
ASSERT(management_region.GetEndAddress() != 0);
memory_manager->Initialize(management_region.GetAddress(), management_region.GetSize());
}
void InitializeHackSharedMemory() {
// Setup memory regions for emulated processes
// TODO(bunnei): These should not be hardcoded regions initialized within the kernel
constexpr std::size_t hid_size{0x40000};
constexpr std::size_t font_size{0x1100000};
constexpr std::size_t irs_size{0x8000};
constexpr std::size_t time_size{0x1000};
constexpr std::size_t hidbus_size{0x1000};
hid_shared_mem = KSharedMemory::Create(system.Kernel());
font_shared_mem = KSharedMemory::Create(system.Kernel());
irs_shared_mem = KSharedMemory::Create(system.Kernel());
time_shared_mem = KSharedMemory::Create(system.Kernel());
hidbus_shared_mem = KSharedMemory::Create(system.Kernel());
hid_shared_mem->Initialize(system.DeviceMemory(), nullptr, Svc::MemoryPermission::None,
Svc::MemoryPermission::Read, hid_size);
font_shared_mem->Initialize(system.DeviceMemory(), nullptr, Svc::MemoryPermission::None,
Svc::MemoryPermission::Read, font_size);
irs_shared_mem->Initialize(system.DeviceMemory(), nullptr, Svc::MemoryPermission::None,
Svc::MemoryPermission::Read, irs_size);
time_shared_mem->Initialize(system.DeviceMemory(), nullptr, Svc::MemoryPermission::None,
Svc::MemoryPermission::Read, time_size);
hidbus_shared_mem->Initialize(system.DeviceMemory(), nullptr, Svc::MemoryPermission::None,
Svc::MemoryPermission::Read, hidbus_size);
}
std::mutex registered_objects_lock;
std::mutex registered_in_use_objects_lock;
std::atomic<u32> next_object_id{0};
std::atomic<u64> next_kernel_process_id{KProcess::InitialKIPIDMin};
std::atomic<u64> next_user_process_id{KProcess::ProcessIDMin};
std::atomic<u64> next_thread_id{1};
// Lists all processes that exist in the current session.
std::vector<KProcess*> process_list;
std::atomic<KProcess*> application_process{};
std::unique_ptr<Kernel::GlobalSchedulerContext> global_scheduler_context;
std::unique_ptr<Kernel::KHardwareTimer> hardware_timer;
Init::KSlabResourceCounts slab_resource_counts{};
KResourceLimit* system_resource_limit{};
KPageBufferSlabHeap page_buffer_slab_heap;
std::shared_ptr<Core::Timing::EventType> preemption_event;
// This is the kernel's handle table or supervisor handle table which
// stores all the objects in place.
std::unique_ptr<KHandleTable> global_handle_table;
std::unique_ptr<KAutoObjectWithListContainer> global_object_list_container;
std::unique_ptr<KObjectNameGlobalData> object_name_global_data;
std::unordered_set<KAutoObject*> registered_objects;
std::unordered_set<KAutoObject*> registered_in_use_objects;
std::mutex server_lock;
std::vector<std::unique_ptr<Service::ServerManager>> server_managers;
std::unique_ptr<Core::ExclusiveMonitor> exclusive_monitor;
std::array<std::unique_ptr<Kernel::PhysicalCore>, Core::Hardware::NUM_CPU_CORES> cores;
// Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others
std::atomic<u32> next_host_thread_id{Core::Hardware::NUM_CPU_CORES};
// Kernel memory management
std::unique_ptr<KMemoryManager> memory_manager;
// Resource managers
std::unique_ptr<KDynamicPageManager> resource_manager_page_manager;
std::unique_ptr<KPageTableSlabHeap> page_table_heap;
std::unique_ptr<KMemoryBlockSlabHeap> app_memory_block_heap;
std::unique_ptr<KMemoryBlockSlabHeap> sys_memory_block_heap;
std::unique_ptr<KBlockInfoSlabHeap> block_info_heap;
std::unique_ptr<KPageTableManager> app_page_table_manager;
std::unique_ptr<KPageTableManager> sys_page_table_manager;
std::unique_ptr<KMemoryBlockSlabManager> app_memory_block_manager;
std::unique_ptr<KMemoryBlockSlabManager> sys_memory_block_manager;
std::unique_ptr<KBlockInfoManager> app_block_info_manager;
std::unique_ptr<KBlockInfoManager> sys_block_info_manager;
std::unique_ptr<KSystemResource> app_system_resource;
std::unique_ptr<KSystemResource> sys_system_resource;
// Shared memory for services
Kernel::KSharedMemory* hid_shared_mem{};
Kernel::KSharedMemory* font_shared_mem{};
Kernel::KSharedMemory* irs_shared_mem{};
Kernel::KSharedMemory* time_shared_mem{};
Kernel::KSharedMemory* hidbus_shared_mem{};
// Memory layout
std::unique_ptr<KMemoryLayout> memory_layout;
std::array<KThread*, Core::Hardware::NUM_CPU_CORES> shutdown_threads{};
std::array<std::unique_ptr<Kernel::KScheduler>, Core::Hardware::NUM_CPU_CORES> schedulers{};
bool is_multicore{};
std::atomic_bool is_shutting_down{};
u32 single_core_thread_id{};
std::array<u64, Core::Hardware::NUM_CPU_CORES> svc_ticks{};
KWorkerTaskManager worker_task_manager;
// System context
Core::System& system;
};
KernelCore::KernelCore(Core::System& system) : impl{std::make_unique<Impl>(system, *this)} {}
KernelCore::~KernelCore() = default;
void KernelCore::SetMulticore(bool is_multicore) {
impl->SetMulticore(is_multicore);
}
void KernelCore::Initialize() {
slab_heap_container = std::make_unique<SlabHeapContainer>();
impl->Initialize(*this);
}
void KernelCore::InitializeCores() {
impl->InitializeCores();
}
void KernelCore::Shutdown() {
impl->Shutdown();
}
void KernelCore::CloseServices() {
impl->CloseServices();
}
const KResourceLimit* KernelCore::GetSystemResourceLimit() const {
return impl->system_resource_limit;
}
KResourceLimit* KernelCore::GetSystemResourceLimit() {
return impl->system_resource_limit;
}
KScopedAutoObject<KThread> KernelCore::RetrieveThreadFromGlobalHandleTable(Handle handle) const {
return impl->global_handle_table->GetObject<KThread>(handle);
}
void KernelCore::AppendNewProcess(KProcess* process) {
impl->process_list.push_back(process);
}
void KernelCore::MakeApplicationProcess(KProcess* process) {
impl->MakeApplicationProcess(process);
}
KProcess* KernelCore::ApplicationProcess() {
return impl->application_process;
}
const KProcess* KernelCore::ApplicationProcess() const {
return impl->application_process;
}
void KernelCore::CloseApplicationProcess() {
impl->CloseApplicationProcess();
}
const std::vector<KProcess*>& KernelCore::GetProcessList() const {
return impl->process_list;
}
Kernel::GlobalSchedulerContext& KernelCore::GlobalSchedulerContext() {
return *impl->global_scheduler_context;
}
const Kernel::GlobalSchedulerContext& KernelCore::GlobalSchedulerContext() const {
return *impl->global_scheduler_context;
}
Kernel::KScheduler& KernelCore::Scheduler(std::size_t id) {
return *impl->schedulers[id];
}
const Kernel::KScheduler& KernelCore::Scheduler(std::size_t id) const {
return *impl->schedulers[id];
}
Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) {
return *impl->cores[id];
}
const Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) const {
return *impl->cores[id];
}
size_t KernelCore::CurrentPhysicalCoreIndex() const {
const u32 core_id = impl->GetCurrentHostThreadID();
if (core_id >= Core::Hardware::NUM_CPU_CORES) {
return Core::Hardware::NUM_CPU_CORES - 1;
}
return core_id;
}
Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() {
return *impl->cores[CurrentPhysicalCoreIndex()];
}
const Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() const {
return *impl->cores[CurrentPhysicalCoreIndex()];
}
Kernel::KScheduler* KernelCore::CurrentScheduler() {
const u32 core_id = impl->GetCurrentHostThreadID();
if (core_id >= Core::Hardware::NUM_CPU_CORES) {
// This is expected when called from not a guest thread
return {};
}
return impl->schedulers[core_id].get();
}
Kernel::KHardwareTimer& KernelCore::HardwareTimer() {
return *impl->hardware_timer;
}
Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() {
return *impl->exclusive_monitor;
}
const Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() const {
return *impl->exclusive_monitor;
}
KAutoObjectWithListContainer& KernelCore::ObjectListContainer() {
return *impl->global_object_list_container;
}
const KAutoObjectWithListContainer& KernelCore::ObjectListContainer() const {
return *impl->global_object_list_container;
}
void KernelCore::InvalidateAllInstructionCaches() {
for (auto& physical_core : impl->cores) {
physical_core->ArmInterface().ClearInstructionCache();
}
}
void KernelCore::InvalidateCpuInstructionCacheRange(KProcessAddress addr, std::size_t size) {
for (auto& physical_core : impl->cores) {
if (!physical_core->IsInitialized()) {
continue;
}
physical_core->ArmInterface().InvalidateCacheRange(GetInteger(addr), size);
}
}
void KernelCore::PrepareReschedule(std::size_t id) {
// TODO: Reimplement, this
}
void KernelCore::RegisterKernelObject(KAutoObject* object) {
std::scoped_lock lk{impl->registered_objects_lock};
impl->registered_objects.insert(object);
}
void KernelCore::UnregisterKernelObject(KAutoObject* object) {
std::scoped_lock lk{impl->registered_objects_lock};
impl->registered_objects.erase(object);
}
void KernelCore::RegisterInUseObject(KAutoObject* object) {
std::scoped_lock lk{impl->registered_in_use_objects_lock};
impl->registered_in_use_objects.insert(object);
}
void KernelCore::UnregisterInUseObject(KAutoObject* object) {
std::scoped_lock lk{impl->registered_in_use_objects_lock};
impl->registered_in_use_objects.erase(object);
}
void KernelCore::RunServer(std::unique_ptr<Service::ServerManager>&& server_manager) {
auto* manager = server_manager.get();
{
std::scoped_lock lk{impl->server_lock};
if (impl->is_shutting_down) {
return;
}
impl->server_managers.emplace_back(std::move(server_manager));
}
manager->LoopProcess();
}
u32 KernelCore::CreateNewObjectID() {
return impl->next_object_id++;
}
u64 KernelCore::CreateNewThreadID() {
return impl->next_thread_id++;
}
u64 KernelCore::CreateNewKernelProcessID() {
return impl->next_kernel_process_id++;
}
u64 KernelCore::CreateNewUserProcessID() {
return impl->next_user_process_id++;
}
KHandleTable& KernelCore::GlobalHandleTable() {
return *impl->global_handle_table;
}
const KHandleTable& KernelCore::GlobalHandleTable() const {
return *impl->global_handle_table;
}
void KernelCore::RegisterCoreThread(std::size_t core_id) {
impl->RegisterCoreThread(core_id);
}
void KernelCore::RegisterHostThread(KThread* existing_thread) {
impl->RegisterHostThread(existing_thread);
if (existing_thread != nullptr) {
ASSERT(GetCurrentEmuThread() == existing_thread);
}
}
static std::jthread RunHostThreadFunc(KernelCore& kernel, KProcess* process,
std::string&& thread_name, std::function<void()>&& func) {
// Reserve a new thread from the process resource limit.
KScopedResourceReservation thread_reservation(process, LimitableResource::ThreadCountMax);
ASSERT(thread_reservation.Succeeded());
// Initialize the thread.
KThread* thread = KThread::Create(kernel);
ASSERT(R_SUCCEEDED(KThread::InitializeDummyThread(thread, process)));
// Commit the thread reservation.
thread_reservation.Commit();
return std::jthread(
[&kernel, thread, thread_name{std::move(thread_name)}, func{std::move(func)}] {
// Set the thread name.
Common::SetCurrentThreadName(thread_name.c_str());
// Register the thread.
kernel.RegisterHostThread(thread);
// Run the callback.
func();
// Close the thread.
// This will free the process if it is the last reference.
thread->Close();
});
}
std::jthread KernelCore::RunOnHostCoreProcess(std::string&& process_name,
std::function<void()> func) {
// Make a new process.
KProcess* process = KProcess::Create(*this);
ASSERT(R_SUCCEEDED(KProcess::Initialize(process, System(), "", KProcess::ProcessType::Userland,
GetSystemResourceLimit())));
// Ensure that we don't hold onto any extra references.
SCOPE_EXIT({ process->Close(); });
// Run the host thread.
return RunHostThreadFunc(*this, process, std::move(process_name), std::move(func));
}
std::jthread KernelCore::RunOnHostCoreThread(std::string&& thread_name,
std::function<void()> func) {
// Get the current process.
KProcess* process = GetCurrentProcessPointer(*this);
// Run the host thread.
return RunHostThreadFunc(*this, process, std::move(thread_name), std::move(func));
}
void KernelCore::RunOnGuestCoreProcess(std::string&& process_name, std::function<void()> func) {
constexpr s32 ServiceThreadPriority = 16;
constexpr s32 ServiceThreadCore = 3;
// Make a new process.
KProcess* process = KProcess::Create(*this);
ASSERT(R_SUCCEEDED(KProcess::Initialize(process, System(), "", KProcess::ProcessType::Userland,
GetSystemResourceLimit())));
// Ensure that we don't hold onto any extra references.
SCOPE_EXIT({ process->Close(); });
// Reserve a new thread from the process resource limit.
KScopedResourceReservation thread_reservation(process, LimitableResource::ThreadCountMax);
ASSERT(thread_reservation.Succeeded());
// Initialize the thread.
KThread* thread = KThread::Create(*this);
ASSERT(R_SUCCEEDED(KThread::InitializeServiceThread(
System(), thread, std::move(func), ServiceThreadPriority, ServiceThreadCore, process)));
// Commit the thread reservation.
thread_reservation.Commit();
// Begin running the thread.
ASSERT(R_SUCCEEDED(thread->Run()));
}
u32 KernelCore::GetCurrentHostThreadID() const {
return impl->GetCurrentHostThreadID();
}
KThread* KernelCore::GetCurrentEmuThread() const {
return impl->GetCurrentEmuThread();
}
void KernelCore::SetCurrentEmuThread(KThread* thread) {
impl->SetCurrentEmuThread(thread);
}
KObjectNameGlobalData& KernelCore::ObjectNameGlobalData() {
return *impl->object_name_global_data;
}
KMemoryManager& KernelCore::MemoryManager() {
return *impl->memory_manager;
}
const KMemoryManager& KernelCore::MemoryManager() const {
return *impl->memory_manager;
}
KSystemResource& KernelCore::GetAppSystemResource() {
return *impl->app_system_resource;
}
const KSystemResource& KernelCore::GetAppSystemResource() const {
return *impl->app_system_resource;
}
KSystemResource& KernelCore::GetSystemSystemResource() {
return *impl->sys_system_resource;
}
const KSystemResource& KernelCore::GetSystemSystemResource() const {
return *impl->sys_system_resource;
}
Kernel::KSharedMemory& KernelCore::GetHidSharedMem() {
return *impl->hid_shared_mem;
}
const Kernel::KSharedMemory& KernelCore::GetHidSharedMem() const {
return *impl->hid_shared_mem;
}
Kernel::KSharedMemory& KernelCore::GetFontSharedMem() {
return *impl->font_shared_mem;
}
const Kernel::KSharedMemory& KernelCore::GetFontSharedMem() const {
return *impl->font_shared_mem;
}
Kernel::KSharedMemory& KernelCore::GetIrsSharedMem() {
return *impl->irs_shared_mem;
}
const Kernel::KSharedMemory& KernelCore::GetIrsSharedMem() const {
return *impl->irs_shared_mem;
}
Kernel::KSharedMemory& KernelCore::GetTimeSharedMem() {
return *impl->time_shared_mem;
}
const Kernel::KSharedMemory& KernelCore::GetTimeSharedMem() const {
return *impl->time_shared_mem;
}
Kernel::KSharedMemory& KernelCore::GetHidBusSharedMem() {
return *impl->hidbus_shared_mem;
}
const Kernel::KSharedMemory& KernelCore::GetHidBusSharedMem() const {
return *impl->hidbus_shared_mem;
}
void KernelCore::SuspendApplication(bool suspended) {
const bool should_suspend{exception_exited || suspended};
const auto activity = should_suspend ? ProcessActivity::Paused : ProcessActivity::Runnable;
// Get the application process.
KScopedAutoObject<KProcess> process = ApplicationProcess();
if (process.IsNull()) {
return;
}
// Set the new activity.
process->SetActivity(activity);
// Wait for process execution to stop.
bool must_wait{should_suspend};
// KernelCore::SuspendApplication must be called from locked context,
// or we could race another call to SetActivity, interfering with waiting.
while (must_wait) {
KScopedSchedulerLock sl{*this};
// Assume that all threads have finished running.
must_wait = false;
for (auto i = 0; i < static_cast<s32>(Core::Hardware::NUM_CPU_CORES); ++i) {
if (Scheduler(i).GetSchedulerCurrentThread()->GetOwnerProcess() ==
process.GetPointerUnsafe()) {
// A thread has not finished running yet.
// Continue waiting.
must_wait = true;
}
}
}
}
void KernelCore::ShutdownCores() {
KScopedSchedulerLock lk{*this};
for (auto* thread : impl->shutdown_threads) {
void(thread->Run());
}
}
bool KernelCore::IsMulticore() const {
return impl->is_multicore;
}
bool KernelCore::IsShuttingDown() const {
return impl->IsShuttingDown();
}
void KernelCore::ExceptionalExitApplication() {
exception_exited = true;
SuspendApplication(true);
}
void KernelCore::EnterSVCProfile() {
impl->svc_ticks[CurrentPhysicalCoreIndex()] = MicroProfileEnter(MICROPROFILE_TOKEN(Kernel_SVC));
}
void KernelCore::ExitSVCProfile() {
MicroProfileLeave(MICROPROFILE_TOKEN(Kernel_SVC), impl->svc_ticks[CurrentPhysicalCoreIndex()]);
}
Init::KSlabResourceCounts& KernelCore::SlabResourceCounts() {
return impl->slab_resource_counts;
}
const Init::KSlabResourceCounts& KernelCore::SlabResourceCounts() const {
return impl->slab_resource_counts;
}
KWorkerTaskManager& KernelCore::WorkerTaskManager() {
return impl->worker_task_manager;
}
const KWorkerTaskManager& KernelCore::WorkerTaskManager() const {
return impl->worker_task_manager;
}
const KMemoryLayout& KernelCore::MemoryLayout() const {
return *impl->memory_layout;
}
bool KernelCore::IsPhantomModeForSingleCore() const {
return impl->IsPhantomModeForSingleCore();
}
void KernelCore::SetIsPhantomModeForSingleCore(bool value) {
impl->SetIsPhantomModeForSingleCore(value);
}
Core::System& KernelCore::System() {
return impl->system;
}
const Core::System& KernelCore::System() const {
return impl->system;
}
struct KernelCore::SlabHeapContainer {
KSlabHeap<KClientSession> client_session;
KSlabHeap<KEvent> event;
KSlabHeap<KPort> port;
KSlabHeap<KProcess> process;
KSlabHeap<KResourceLimit> resource_limit;
KSlabHeap<KSession> session;
KSlabHeap<KSharedMemory> shared_memory;
KSlabHeap<KSharedMemoryInfo> shared_memory_info;
KSlabHeap<KThread> thread;
KSlabHeap<KTransferMemory> transfer_memory;
KSlabHeap<KCodeMemory> code_memory;
KSlabHeap<KDeviceAddressSpace> device_address_space;
KSlabHeap<KPageBuffer> page_buffer;
KSlabHeap<KThreadLocalPage> thread_local_page;
KSlabHeap<KObjectName> object_name;
KSlabHeap<KSessionRequest> session_request;
KSlabHeap<KSecureSystemResource> secure_system_resource;
KSlabHeap<KThread::LockWithPriorityInheritanceInfo> lock_info;
KSlabHeap<KEventInfo> event_info;
KSlabHeap<KDebug> debug;
};
template <typename T>
KSlabHeap<T>& KernelCore::SlabHeap() {
if constexpr (std::is_same_v<T, KClientSession>) {
return slab_heap_container->client_session;
} else if constexpr (std::is_same_v<T, KEvent>) {
return slab_heap_container->event;
} else if constexpr (std::is_same_v<T, KPort>) {
return slab_heap_container->port;
} else if constexpr (std::is_same_v<T, KProcess>) {
return slab_heap_container->process;
} else if constexpr (std::is_same_v<T, KResourceLimit>) {
return slab_heap_container->resource_limit;
} else if constexpr (std::is_same_v<T, KSession>) {
return slab_heap_container->session;
} else if constexpr (std::is_same_v<T, KSharedMemory>) {
return slab_heap_container->shared_memory;
} else if constexpr (std::is_same_v<T, KSharedMemoryInfo>) {
return slab_heap_container->shared_memory_info;
} else if constexpr (std::is_same_v<T, KThread>) {
return slab_heap_container->thread;
} else if constexpr (std::is_same_v<T, KTransferMemory>) {
return slab_heap_container->transfer_memory;
} else if constexpr (std::is_same_v<T, KCodeMemory>) {
return slab_heap_container->code_memory;
} else if constexpr (std::is_same_v<T, KDeviceAddressSpace>) {
return slab_heap_container->device_address_space;
} else if constexpr (std::is_same_v<T, KPageBuffer>) {
return slab_heap_container->page_buffer;
} else if constexpr (std::is_same_v<T, KThreadLocalPage>) {
return slab_heap_container->thread_local_page;
} else if constexpr (std::is_same_v<T, KObjectName>) {
return slab_heap_container->object_name;
} else if constexpr (std::is_same_v<T, KSessionRequest>) {
return slab_heap_container->session_request;
} else if constexpr (std::is_same_v<T, KSecureSystemResource>) {
return slab_heap_container->secure_system_resource;
} else if constexpr (std::is_same_v<T, KThread::LockWithPriorityInheritanceInfo>) {
return slab_heap_container->lock_info;
} else if constexpr (std::is_same_v<T, KEventInfo>) {
return slab_heap_container->event_info;
} else if constexpr (std::is_same_v<T, KDebug>) {
return slab_heap_container->debug;
}
}
template KSlabHeap<KClientSession>& KernelCore::SlabHeap();
template KSlabHeap<KEvent>& KernelCore::SlabHeap();
template KSlabHeap<KPort>& KernelCore::SlabHeap();
template KSlabHeap<KProcess>& KernelCore::SlabHeap();
template KSlabHeap<KResourceLimit>& KernelCore::SlabHeap();
template KSlabHeap<KSession>& KernelCore::SlabHeap();
template KSlabHeap<KSharedMemory>& KernelCore::SlabHeap();
template KSlabHeap<KSharedMemoryInfo>& KernelCore::SlabHeap();
template KSlabHeap<KThread>& KernelCore::SlabHeap();
template KSlabHeap<KTransferMemory>& KernelCore::SlabHeap();
template KSlabHeap<KCodeMemory>& KernelCore::SlabHeap();
template KSlabHeap<KDeviceAddressSpace>& KernelCore::SlabHeap();
template KSlabHeap<KPageBuffer>& KernelCore::SlabHeap();
template KSlabHeap<KThreadLocalPage>& KernelCore::SlabHeap();
template KSlabHeap<KObjectName>& KernelCore::SlabHeap();
template KSlabHeap<KSessionRequest>& KernelCore::SlabHeap();
template KSlabHeap<KSecureSystemResource>& KernelCore::SlabHeap();
template KSlabHeap<KThread::LockWithPriorityInheritanceInfo>& KernelCore::SlabHeap();
template KSlabHeap<KEventInfo>& KernelCore::SlabHeap();
template KSlabHeap<KDebug>& KernelCore::SlabHeap();
} // namespace Kernel
View Git Blame Copy Permalink