Loader: Clean up 3dsx loader a bit, fixing a potential buffer overrun

src/core/loader/3dsx.cpp

@@ -19,7 +19,7 @@
 
 namespace Loader {
 
-/**
+/*
  * File layout:
  * - File header
  * - Code, rodata and data relocation table headers
@@ -39,13 +39,16 @@ namespace Loader {
  * The entrypoint is always the start of the code segment.
  * The BSS section must be cleared manually by the application.
  */
+
 enum THREEDSX_Error {
     ERROR_NONE = 0,
     ERROR_READ = 1,
     ERROR_FILE = 2,
     ERROR_ALLOC = 3
 };
+
 static const u32 RELOCBUFSIZE = 512;
+static const unsigned int NUM_SEGMENTS = 3;
 
 // File header
 #pragma pack(1)
@@ -116,13 +119,13 @@ static THREEDSX_Error Load3DSXFile(FileUtil::IOFile& file, u32 base_addr)
     loadinfo.seg_sizes[1] = (hdr.rodata_seg_size + 0xFFF) &~0xFFF;
     loadinfo.seg_sizes[2] = (hdr.data_seg_size + 0xFFF) &~0xFFF;
     u32 offsets[2] = { loadinfo.seg_sizes[0], loadinfo.seg_sizes[0] + loadinfo.seg_sizes[1] };
-    u32 n_reloc_tables = hdr.reloc_hdr_size / 4;
-    std::vector<u8> all_mem(loadinfo.seg_sizes[0] + loadinfo.seg_sizes[1] + loadinfo.seg_sizes[2] + 3 * n_reloc_tables);
+    u32 n_reloc_tables = hdr.reloc_hdr_size / sizeof(u32);
+    std::vector<u8> program_image(loadinfo.seg_sizes[0] + loadinfo.seg_sizes[1] + loadinfo.seg_sizes[2]);
 
     loadinfo.seg_addrs[0] = base_addr;
     loadinfo.seg_addrs[1] = loadinfo.seg_addrs[0] + loadinfo.seg_sizes[0];
     loadinfo.seg_addrs[2] = loadinfo.seg_addrs[1] + loadinfo.seg_sizes[1];
-    loadinfo.seg_ptrs[0] = &all_mem[0];
+    loadinfo.seg_ptrs[0] = program_image.data();
     loadinfo.seg_ptrs[1] = loadinfo.seg_ptrs[0] + loadinfo.seg_sizes[0];
     loadinfo.seg_ptrs[2] = loadinfo.seg_ptrs[1] + loadinfo.seg_sizes[1];
 
@@ -130,10 +133,9 @@ static THREEDSX_Error Load3DSXFile(FileUtil::IOFile& file, u32 base_addr)
     file.Seek(hdr.header_size, SEEK_SET);
 
     // Read the relocation headers
-    u32* relocs = (u32*)(loadinfo.seg_ptrs[2] + hdr.data_seg_size);
-
-    for (unsigned current_segment : {0, 1, 2}) {
-        size_t size = n_reloc_tables * 4;
+    std::vector<u32> relocs(n_reloc_tables * NUM_SEGMENTS);
+    for (unsigned int current_segment = 0; current_segment < NUM_SEGMENTS; ++current_segment) {
+        size_t size = n_reloc_tables * sizeof(u32);
         if (file.ReadBytes(&relocs[current_segment * n_reloc_tables], size) != size)
             return ERROR_READ;
     }
@@ -150,7 +152,7 @@ static THREEDSX_Error Load3DSXFile(FileUtil::IOFile& file, u32 base_addr)
     memset((char*)loadinfo.seg_ptrs[2] + hdr.data_seg_size - hdr.bss_size, 0, hdr.bss_size);
 
     // Relocate the segments
-    for (unsigned current_segment : {0, 1, 2}) {
+    for (unsigned int current_segment = 0; current_segment < NUM_SEGMENTS; ++current_segment) {
         for (unsigned current_segment_reloc_table = 0; current_segment_reloc_table < n_reloc_tables; current_segment_reloc_table++) {
             u32 n_relocs = relocs[current_segment * n_reloc_tables + current_segment_reloc_table];
             if (current_segment_reloc_table >= 2) {
@@ -158,7 +160,7 @@ static THREEDSX_Error Load3DSXFile(FileUtil::IOFile& file, u32 base_addr)
                 file.Seek(n_relocs*sizeof(THREEDSX_Reloc), SEEK_CUR);
                 continue;
             }
-            static THREEDSX_Reloc reloc_table[RELOCBUFSIZE];
+            THREEDSX_Reloc reloc_table[RELOCBUFSIZE];
 
             u32* pos = (u32*)loadinfo.seg_ptrs[current_segment];
             const u32* end_pos = pos + (loadinfo.seg_sizes[current_segment] / 4);
@@ -177,7 +179,7 @@ static THREEDSX_Error Load3DSXFile(FileUtil::IOFile& file, u32 base_addr)
                     pos += table.skip;
                     s32 num_patches = table.patch;
                     while (0 < num_patches && pos < end_pos) {
-                        u32 in_addr = (char*)pos - (char*)&all_mem[0];
+                        u32 in_addr = (u8*)pos - program_image.data();
                         u32 addr = TranslateAddr(*pos, &loadinfo, offsets);
                         LOG_TRACE(Loader, "Patching %08X <-- rel(%08X,%d) (%08X)\n",
                                   base_addr + in_addr, addr, current_segment_reloc_table, *pos);
@@ -200,7 +202,7 @@ static THREEDSX_Error Load3DSXFile(FileUtil::IOFile& file, u32 base_addr)
     }
 
     // Write the data
-    memcpy(Memory::GetPointer(base_addr), &all_mem[0], loadinfo.seg_sizes[0] + loadinfo.seg_sizes[1] + loadinfo.seg_sizes[2]);
+    memcpy(Memory::GetPointer(base_addr), program_image.data(), program_image.size());
 
     LOG_DEBUG(Loader, "code size:   0x%X", loadinfo.seg_sizes[0]);
     LOG_DEBUG(Loader, "rodata size: 0x%X", loadinfo.seg_sizes[1]);
@@ -235,7 +237,8 @@ ResultStatus AppLoader_THREEDSX::Load() {
     // Attach the default resource limit (APPLICATION) to the process
     Kernel::g_current_process->resource_limit = Kernel::ResourceLimit::GetForCategory(Kernel::ResourceLimitCategory::APPLICATION);
 
-    Load3DSXFile(*file, Memory::PROCESS_IMAGE_VADDR);
+    if (Load3DSXFile(*file, Memory::PROCESS_IMAGE_VADDR) != ERROR_NONE)
+        return ResultStatus::Error;
 
     Kernel::g_current_process->Run(Memory::PROCESS_IMAGE_VADDR, 48, Kernel::DEFAULT_STACK_SIZE);