From: Waldemar Kozaczuk <[email protected]>
Committer: Nadav Har'El <[email protected]>
Branch: master
Make OSv bootable as a Linux 64-bit ELF
This patch makes all necessary changes to OSv boot logic
to make it bootable on firecracker.
Please note that until we add virtio-mmio support
we can only boot ramfs images on firecracker like so:
./scripts/build fs=ramfs image=native-example
Signed-off-by: Waldemar Kozaczuk <[email protected]>
Message-Id: <[email protected]>
---
diff --git a/arch/x64/arch-setup.cc b/arch/x64/arch-setup.cc
--- a/arch/x64/arch-setup.cc
+++ b/arch/x64/arch-setup.cc
@@ -117,6 +117,96 @@ extern size_t elf_size;
extern void* elf_start;
extern boot_time_chart boot_time;
+// Because entry64 replaces start32 as a new entry of loader.elf we need a
way
+// to place address of start32 so that boot16 know where to jump to. We
achieve
+// it by placing address of start32 at the known offset at memory
+// as defined by section .start32_address in loader.ld
+extern "C" void start32();
+void * __attribute__((section (".start32_address"))) start32_address =
reinterpret_cast<void*>(&start32);
+
+#define OSV_MULTI_BOOT_INFO_ADDR 0x1000
+#define OSV_E820_TABLE_ADDR 0x2000
+
+//
+// Instead of defining full boot_params and setup_header structs as in
+// Linux source code, we define only handful of offsets pointing the fields
+// we need to read from there. For details please this chunk of Linux code
-
+//
https://github.com/torvalds/linux/blob/b6839ef26e549de68c10359d45163b0cfb031183/arch/x86/include/uapi/asm/bootparam.h#L151-L198
+#define LINUX_KERNEL_BOOT_FLAG_MAGIC 0xaa55
+#define LINUX_KERNEL_HDR_MAGIC 0x53726448 // "HdrS"
+
+#define SETUP_HEADER_OFFSET 0x1f1 // look at bootparam.h in linux
+#define SETUP_HEADER_FIELD_VAL(boot_params, offset, field_type) \
+ (*static_cast<field_type*>(boot_params + SETUP_HEADER_OFFSET + offset))
+
+#define BOOT_FLAG_OFFSET sizeof(u8) + 4 * sizeof(u16) + sizeof(u32)
+#define HDR_MAGIC_OFFSET sizeof(u8) + 6 * sizeof(u16) + sizeof(u32)
+
+#define E820_ENTRIES_OFFSET 0x1e8 // look at bootparam.h in linux
+#define E820_TABLE_OFFSET 0x2d0 // look at bootparam.h in linux
+
+#define CMD_LINE_PTR_OFFSET sizeof(u8) * 5 + sizeof(u16) * 11 +
sizeof(u32) * 7
+
+struct linux_e820ent {
+ u64 addr;
+ u64 size;
+ u32 type;
+} __attribute__((packed));
+
+// When OSv kernel gets booted directly as 64-bit ELF (loader.elf) as it is
+// the case on firecracker we need a way to extract all necessary
information
+// about available memory and command line. This information is provided
+// the struct boot_params (see details above) placed in memory at the
address
+// specified in the RSI register.
+// The following extract_linux_boot_params() function is called from
+// entry64 in boot.S and verifies OSV was indeed boot as Linux and
+// copies memory and cmdline information into OSv multiboot struct.
+// Please see https://www.kernel.org/doc/Documentation/x86/boot.txt for
details
+// of Linux boot protocol. Bear in mind that OSv implements very narrow
specific
+// subset of the protocol as assumed by firecracker.
+extern "C" void extract_linux_boot_params(void *boot_params)
+{ //
+ // Verify we are being booted as Linux 64-bit ELF kernel
+ assert( SETUP_HEADER_FIELD_VAL(boot_params, BOOT_FLAG_OFFSET, u16) ==
LINUX_KERNEL_BOOT_FLAG_MAGIC);
+ assert( SETUP_HEADER_FIELD_VAL(boot_params, HDR_MAGIC_OFFSET, u32) ==
LINUX_KERNEL_HDR_MAGIC);
+
+ // Set location of multiboot info struct at arbitrary place in lower
memory
+ // to copy to (happens to be the same as in boot16.S)
+ osv_multiboot_info_type* mb_info =
reinterpret_cast<osv_multiboot_info_type*>(OSV_MULTI_BOOT_INFO_ADDR);
+
+ // Copy command line pointer from boot params
+ mb_info->mb.cmdline = SETUP_HEADER_FIELD_VAL(boot_params,
CMD_LINE_PTR_OFFSET, u32);
+
+ // Copy e820 information from boot params
+ mb_info->mb.mmap_length = 0;
+ mb_info->mb.mmap_addr = OSV_E820_TABLE_ADDR;
+
+ struct linux_e820ent *source_e820_table = static_cast<struct
linux_e820ent *>(boot_params + E820_TABLE_OFFSET);
+ struct e820ent *dest_e820_table = reinterpret_cast<struct e820ent
*>(mb_info->mb.mmap_addr);
+
+ u8 en820_entries = *static_cast<u8*>(boot_params +
E820_ENTRIES_OFFSET);
+ for (int e820_index = 0; e820_index < en820_entries; e820_index++) {
+ dest_e820_table[e820_index].ent_size = 20;
+ dest_e820_table[e820_index].type =
source_e820_table[e820_index].type;
+ dest_e820_table[e820_index].addr =
source_e820_table[e820_index].addr;
+ dest_e820_table[e820_index].size =
source_e820_table[e820_index].size;
+ mb_info->mb.mmap_length += sizeof(e820ent);
+ }
+
+ auto now = processor::ticks();
+ u32 now_high = (u32)(now >> 32);
+ u32 now_low = (u32)now;
+
+ mb_info->tsc_init_hi = now_high;
+ mb_info->tsc_init = now_low;
+
+ mb_info->tsc_disk_done_hi = now_high;
+ mb_info->tsc_disk_done = now_low;
+
+ mb_info->tsc_uncompress_done_hi = now_high;
+ mb_info->tsc_uncompress_done = now_low;
+}
+
void arch_setup_free_memory()
{
static ulong edata;
@@ -272,6 +362,9 @@ void arch_init_drivers()
boot_time.event("pvpanic done");
// Enumerate PCI devices
+ //TODO: Add boot option --pci=off to skip PCI enumeration
+ // as on some platforms like firecracker enumerating "empty" bus
+ // takes up to 10ms
pci::pci_device_enumeration();
boot_time.event("pci enumerated");
diff --git a/arch/x64/boot.S b/arch/x64/boot.S
--- a/arch/x64/boot.S
+++ b/arch/x64/boot.S
@@ -5,6 +5,40 @@
#include "processor-flags.h"
+#define BOOT_CR0 ( X86_CR0_PE \
+ | X86_CR0_WP \
+ | X86_CR0_PG )
+
+#define BOOT_CR4 ( X86_CR4_DE \
+ | X86_CR4_PSE \
+ | X86_CR4_PAE \
+ | X86_CR4_PGE \
+ | X86_CR4_PCE \
+ | X86_CR4_OSFXSR \
+ | X86_CR4_OSXMMEXCPT )
+
+.macro setup_64bit_long_mode
+ // Enable PAE (Physical Address Extension) - ability to address 64GB
+ // TODO: Add more comments to processor-flags.h what each flag does
+ mov $BOOT_CR4, %eax
+ mov %eax, %cr4
+
+ // Set root of a page table in cr3
+ lea ident_pt_l4, %eax
+ mov %eax, %cr3
+
+ // Set long mode?
+ mov $0xc0000080, %ecx // EFER MSR number
+ mov $0x00000900, %eax // Set LME = 1
+ xor %edx, %edx
+ wrmsr // Write contents of EDX:EAX (0:to Model Specific Register
specified by ECX register
+
+ // Activate paging and ...?
+ // TODO: Add more comments to processor-flags.h what each flag does
+ mov $BOOT_CR0, %eax
+ mov %eax, %cr0
+.endm
+
.text
.code32
@@ -49,18 +83,6 @@ init_stack_top = .
.text
-#define BOOT_CR0 ( X86_CR0_PE \
- | X86_CR0_WP \
- | X86_CR0_PG )
-
-#define BOOT_CR4 ( X86_CR4_DE \
- | X86_CR4_PSE \
- | X86_CR4_PAE \
- | X86_CR4_PGE \
- | X86_CR4_PCE \
- | X86_CR4_OSFXSR \
- | X86_CR4_OSXMMEXCPT )
-
.globl start32
start32:
# boot16.S set %eax to ELF start address, we'll use it later
@@ -75,16 +97,7 @@ start32:
ljmp $0x18, $1f
1:
and $~7, %esp
- mov $BOOT_CR4, %eax
- mov %eax, %cr4
- lea ident_pt_l4, %eax
- mov %eax, %cr3
- mov $0xc0000080, %ecx
- mov $0x00000900, %eax
- xor %edx, %edx
- wrmsr
- mov $BOOT_CR0, %eax
- mov %eax, %cr0
+ setup_64bit_long_mode
ljmpl $8, $start64
.code64
.global start64
@@ -181,3 +194,22 @@ smpboot64:
jnz 1b
lea 4096(%rax), %rsp
call smp_main
+
+.text
+.code64
+.global entry64
+entry64:
+# The address of boot_params structed is passed in RSI
+# register so pass it to extract_linux_boot_params fuction
+# which will extract cmdline and memory information and verify
+# that loader.elf was indeed called as Linux 64-bit vmlinux ELF
+ mov %rsi, %rdi
+ call extract_linux_boot_params
+
+# Even though we are in 64-bit long mode we need to reset
+# page tables and other CPU settings the way OSv expects it
+ setup_64bit_long_mode
+
+ mov $OSV_KERNEL_BASE, %rbp
+ mov $0x1000, %rbx
+ jmp start64
diff --git a/arch/x64/boot16.S b/arch/x64/boot16.S
--- a/arch/x64/boot16.S
+++ b/arch/x64/boot16.S
@@ -12,7 +12,7 @@ target = OSV_LZKERNEL_BASE
elf_entry_point_header_offset = 0x18
lzentry = target+elf_entry_point_header_offset
loader = OSV_KERNEL_BASE
-entry = loader+elf_entry_point_header_offset
+start32 = loader+0x800
mb_info = 0x1000
// OSv information lays in the end of this struct, which is 88 bytes in
size.
@@ -168,7 +168,7 @@ done_e820:
mov %edx, mb_uncompress_hi
mov $loader, %eax
mov $mb_info, %ebx
- call *entry
+ call *start32
.org 0x1b8
.byte 0x56, 0x53, 0x4F, 0, 0, 0
diff --git a/arch/x64/loader.ld b/arch/x64/loader.ld
--- a/arch/x64/loader.ld
+++ b/arch/x64/loader.ld
@@ -14,7 +14,13 @@ SECTIONS
*
* We can't export the ELF header base as a symbol, because ld
* insists on moving stuff around if we do.
+ *
+ * Place address of start32 routine at predefined offset in memory
*/
+ . = OSV_KERNEL_BASE + 0x800;
+ .start32_address : {
+ *(.start32_address)
+ }
. = OSV_KERNEL_BASE + 0x1000;
.dynamic : { *(.dynamic) } :dynamic :text
.text : {
@@ -108,4 +114,4 @@ PHDRS {
eh_frame PT_GNU_EH_FRAME;
note PT_NOTE;
}
-ENTRY(start32);
+ENTRY(entry64);
diff --git a/loader.cc b/loader.cc
--- a/loader.cc
+++ b/loader.cc
@@ -55,6 +55,7 @@
#include "drivers/null.hh"
#include "libc/network/__dns.hh"
+#include <processor.hh>
using namespace osv;
using namespace osv::clock::literals;
@@ -426,6 +427,10 @@ void* do_main_thread(void *_main_args)
}
boot_time.event("Total time");
+ // Some hypervisors like firecracker when booting OSv
+ // look for this write to this port as a signal of end of
+ // boot time.
+ processor::outb(123, 0x3f0);
if (opt_bootchart) {
boot_time.print_chart();
--
You received this message because you are subscribed to the Google Groups "OSv
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
