loader: add support for initrd

Add support for booting the micro VM with a user provided initrd file:
- For x86_64, the initrd is loaded at the end of the low (4G) memory,
and passed to the kernel in the bootparams table.
- For aarch64, the initrd is loaded in a memory segment just before the
FDT.  The initrd start and end addresses are specified in the FDT
"/chosen" node.

Users can specify an initrd file using the "initrd_path" property
of the "boot-source" the REST API.

This also adds integration tests to verify VMs boot with initrd and
without a rootfs. Categories of tests added:
- Functional: the VM shall boot and the "/" path should be mounted as
"rootfs" filesystem type.
- Performance: VM shall boot in less than 160ms (for x86).

Co-developed-by: Marco Vedovati <[email protected]>
Co-developed-by: Tim Deegan <[email protected]>
Signed-off-by: Marco Vedovati <[email protected]>

Author: marcov

CHANGELOG.md

@@ -50,6 +50,9 @@
   all parameters specified after `--` are forwarded verbatim to Firecracker.
 - Added `KVM_PTP` support to the recommended guest kernel config.
 - Added entry in FAQ.md for Firecracker Guest timekeeping.
+- Support for booting with an initial RAM disk image. This image can be
+  specified through the new `initrd_path` field of the `/boot-source` API
+  request.
 
 ### Changed
 

src/api_server/src/request/boot_source.rs

@@ -26,10 +26,12 @@ mod tests {
 
         let body = r#"{
                 "kernel_image_path": "/foo/bar",
+                "initrd_path": "/bar/foo",
                 "boot_args": "foobar"
               }"#;
         let same_body = BootSourceConfig {
             kernel_image_path: String::from("/foo/bar"),
+            initrd_path: Some(String::from("/bar/foo")),
             boot_args: Some(String::from("foobar")),
         };
         let result = parse_put_boot_source(&Body::new(body));

src/api_server/swagger/firecracker.yaml

@@ -391,6 +391,9 @@ definitions:
       kernel_image_path:
         type: string
         description: Host level path to the kernel image used to boot the guest
+      initrd_path:
+        type: string
+        description: Host level path to the initrd image used to boot the guest
       boot_args:
         type: string
         description: Kernel boot arguments

src/arch/src/aarch64/fdt.rs

@@ -13,6 +13,7 @@ use std::ptr::null;
 use std::{io, result};
 
 use super::super::DeviceType;
+use super::super::InitrdConfig;
 use super::get_fdt_addr;
 use super::gic::GICDevice;
 use super::layout::FDT_MAX_SIZE;
@@ -89,6 +90,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug>(
     cmdline: &CStr,
     device_info: Option<&HashMap<(DeviceType, String), T>>,
     gic_device: &Box<dyn GICDevice>,
+    initrd: &Option<InitrdConfig>,
 ) -> Result<(Vec<u8>)> {
     // Alocate stuff necessary for the holding the blob.
     let mut fdt = vec![0; FDT_MAX_SIZE];
@@ -111,7 +113,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug>(
     append_property_u32(&mut fdt, "interrupt-parent", GIC_PHANDLE)?;
     create_cpu_nodes(&mut fdt, &vcpu_mpidr)?;
     create_memory_node(&mut fdt, guest_mem)?;
-    create_chosen_node(&mut fdt, cmdline)?;
+    create_chosen_node(&mut fdt, cmdline, initrd)?;
     create_gic_node(&mut fdt, gic_device)?;
     create_timer_node(&mut fdt)?;
     create_clock_node(&mut fdt)?;
@@ -341,9 +343,27 @@ fn create_memory_node(fdt: &mut Vec<u8>, guest_mem: &GuestMemory) -> Result<()>
     Ok(())
 }
 
-fn create_chosen_node(fdt: &mut Vec<u8>, cmdline: &CStr) -> Result<()> {
+fn create_chosen_node(
+    fdt: &mut Vec<u8>,
+    cmdline: &CStr,
+    initrd: &Option<InitrdConfig>,
+) -> Result<()> {
     append_begin_node(fdt, "chosen")?;
     append_property_cstring(fdt, "bootargs", cmdline)?;
+
+    if let Some(initrd_config) = initrd {
+        append_property_u64(
+            fdt,
+            "linux,initrd-start",
+            initrd_config.address.raw_value() as u64,
+        )?;
+        append_property_u64(
+            fdt,
+            "linux,initrd-end",
+            initrd_config.address.raw_value() + initrd_config.size as u64,
+        )?;
+    }
+
     append_end_node(fdt)?;
 
     Ok(())
@@ -581,6 +601,7 @@ mod tests {
             &CString::new("console=tty0").unwrap(),
             Some(&dev_info),
             &gic,
+            &None,
         )
         .is_ok())
     }
@@ -598,6 +619,7 @@ mod tests {
             &CString::new("console=tty0").unwrap(),
             None::<&std::collections::HashMap<(DeviceType, std::string::String), MMIODeviceInfo>>,
             &gic,
+            &None,
         )
         .unwrap();
 
@@ -628,4 +650,54 @@ mod tests {
         let generated_fdt = device_tree::DeviceTree::load(&dtb).unwrap();
         assert!(format!("{:?}", original_fdt) == format!("{:?}", generated_fdt));
     }
+
+    #[test]
+    fn test_create_fdt_with_initrd() {
+        let regions = arch_memory_regions(layout::FDT_MAX_SIZE + 0x1000);
+        let mem = GuestMemory::new(&regions).expect("Cannot initialize memory");
+        let kvm = Kvm::new().unwrap();
+        let vm = kvm.create_vm().unwrap();
+        let gic = create_gic(&vm, 1).unwrap();
+        let initrd = InitrdConfig {
+            address: GuestAddress(0x10000000),
+            size: 0x1000,
+        };
+
+        let mut dtb = create_fdt(
+            &mem,
+            vec![0],
+            &CString::new("console=tty0").unwrap(),
+            None::<&std::collections::HashMap<(DeviceType, std::string::String), MMIODeviceInfo>>,
+            &gic,
+            &Some(initrd),
+        )
+        .unwrap();
+
+        /* Use this code when wanting to generate a new DTB sample.
+        {
+            use std::fs;
+            use std::io::Write;
+            use std::path::PathBuf;
+            let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+            let mut output = fs::OpenOptions::new()
+                .write(true)
+                .create(true)
+                .open(path.join("src/aarch64/output_with_initrd.dtb"))
+                .unwrap();
+            output.write_all(&dtb).unwrap();
+        }
+        */
+
+        let bytes = include_bytes!("output_with_initrd.dtb");
+        let pos = 4;
+        let val = layout::FDT_MAX_SIZE;
+        let mut buf = vec![];
+        buf.extend_from_slice(bytes);
+
+        set_size(&mut buf, pos, val);
+        set_size(&mut dtb, pos, val);
+        let original_fdt = device_tree::DeviceTree::load(&buf).unwrap();
+        let generated_fdt = device_tree::DeviceTree::load(&dtb).unwrap();
+        assert!(format!("{:?}", original_fdt) == format!("{:?}", generated_fdt));
+    }
 }

src/arch/src/aarch64/mod.rs

@@ -24,6 +24,8 @@ use memory_model::{Address, GuestAddress, GuestMemory};
 pub enum Error {
     /// Failed to create a Flattened Device Tree for this aarch64 microVM.
     SetupFDT(fdt::Error),
+    /// Failed to compute the initrd address.
+    InitrdAddress,
 }
 
 /// The start of the memory area reserved for MMIO devices.
@@ -48,19 +50,23 @@ pub fn arch_memory_regions(size: usize) -> Vec<(GuestAddress, usize)> {
 /// * `cmdline_cstring` - The kernel commandline.
 /// * `vcpu_mpidr` - Array of MPIDR register values per vcpu.
 /// * `device_info` - A hashmap containing the attached devices for building FDT device nodes.
+/// * `gic_device` - The GIC device.
+/// * `initrd` - Information about an optional initrd.
 pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug>(
     guest_mem: &GuestMemory,
     cmdline_cstring: &CStr,
     vcpu_mpidr: Vec<u64>,
     device_info: Option<&HashMap<(DeviceType, String), T>>,
     gic_device: &Box<dyn GICDevice>,
+    initrd: &Option<super::InitrdConfig>,
 ) -> super::Result<()> {
     fdt::create_fdt(
         guest_mem,
         vcpu_mpidr,
         cmdline_cstring,
         device_info,
         gic_device,
+        initrd,
     )
     .map_err(Error::SetupFDT)?;
     Ok(())
@@ -71,6 +77,22 @@ pub fn get_kernel_start() -> u64 {
     layout::DRAM_MEM_START
 }
 
+/// Returns the memory address where the initrd could be loaded.
+pub fn initrd_load_addr(guest_mem: &GuestMemory, initrd_size: usize) -> super::Result<u64> {
+    let round_to_pagesize = |size| (size + (super::PAGE_SIZE - 1)) & !(super::PAGE_SIZE - 1);
+    match GuestAddress(get_fdt_addr(&guest_mem)).checked_sub(round_to_pagesize(initrd_size) as u64)
+    {
+        Some(offset) => {
+            if guest_mem.address_in_range(offset) {
+                return Ok(offset.raw_value());
+            } else {
+                return Err(Error::InitrdAddress);
+            }
+        }
+        None => return Err(Error::InitrdAddress),
+    }
+}
+
 // Auxiliary function to get the address where the device tree blob is loaded.
 fn get_fdt_addr(mem: &GuestMemory) -> u64 {
     // If the memory allocated is smaller than the size allocated for the FDT,

src/arch/src/aarch64/output_with_initrd.dtb

@@ -20,8 +20,8 @@ pub mod aarch64;
 
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::{
-    arch_memory_regions, configure_system, get_kernel_start, layout::CMDLINE_MAX_SIZE,
-    layout::IRQ_BASE, layout::IRQ_MAX, Error, MMIO_MEM_START,
+    arch_memory_regions, configure_system, get_kernel_start, initrd_load_addr,
+    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, Error, MMIO_MEM_START,
 };
 
 /// Module for x86_64 related functionality.
@@ -30,8 +30,8 @@ pub mod x86_64;
 
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::{
-    arch_memory_regions, configure_system, get_kernel_start, layout::CMDLINE_MAX_SIZE,
-    layout::IRQ_BASE, layout::IRQ_MAX, Error, MMIO_MEM_START,
+    arch_memory_regions, configure_system, get_kernel_start, initrd_load_addr,
+    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, Error, MMIO_MEM_START,
 };
 
 /// Type for returning public functions outcome.
@@ -50,6 +50,17 @@ pub enum DeviceType {
     RTC,
 }
 
+/// Type for passing information about the initrd in the guest memory.
+pub struct InitrdConfig {
+    /// Load address of initrd in guest memory
+    pub address: memory_model::GuestAddress,
+    /// Size of initrd in guest memory
+    pub size: usize,
+}
+
+/// Default (smallest) memory page size for the supported architectures.
+pub const PAGE_SIZE: usize = 4096;
+
 impl fmt::Display for DeviceType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{:?}", self)

src/arch/src/lib.rs

@@ -18,6 +18,7 @@ use std::mem;
 
 use arch_gen::x86::bootparam::{boot_params, E820_RAM};
 use memory_model::{Address, ByteValued, GuestAddress, GuestMemory};
+use InitrdConfig;
 
 // This is a workaround to the Rust enforcement specifying that any implementation of a foreign
 // trait (in this case `ByteValued`) where:
@@ -41,6 +42,8 @@ pub enum Error {
     ZeroPagePastRamEnd,
     /// Error writing the zero page of guest memory.
     ZeroPageSetup,
+    /// Failed to compute initrd address.
+    InitrdAddress,
 }
 
 // Where BIOS/VGA magic would live on a real PC.
@@ -73,18 +76,32 @@ pub fn get_kernel_start() -> u64 {
     layout::HIMEM_START
 }
 
+/// Returns the memory address where the initrd could be loaded.
+pub fn initrd_load_addr(guest_mem: &GuestMemory, initrd_size: usize) -> super::Result<u64> {
+    let lowmem_size: usize = guest_mem.region_size(0).map_err(|_| Error::InitrdAddress)?;
+
+    if lowmem_size < initrd_size {
+        return Err(Error::InitrdAddress);
+    }
+
+    let align_to_pagesize = |address| address & !(super::PAGE_SIZE - 1);
+    Ok(align_to_pagesize(lowmem_size - initrd_size) as u64)
+}
+
 /// Configures the system and should be called once per vm before starting vcpu threads.
 ///
 /// # Arguments
 ///
 /// * `guest_mem` - The memory to be used by the guest.
 /// * `cmdline_addr` - Address in `guest_mem` where the kernel command line was loaded.
 /// * `cmdline_size` - Size of the kernel command line in bytes including the null terminator.
+/// * `initrd` - Information about where the ramdisk image was loaded in the `guest_mem`.
 /// * `num_cpus` - Number of virtual CPUs the guest will have.
 pub fn configure_system(
     guest_mem: &GuestMemory,
     cmdline_addr: GuestAddress,
     cmdline_size: usize,
+    initrd: &Option<InitrdConfig>,
     num_cpus: u8,
 ) -> super::Result<()> {
     const KERNEL_BOOT_FLAG_MAGIC: u16 = 0xaa55;
@@ -107,6 +124,10 @@ pub fn configure_system(
     params.0.hdr.cmd_line_ptr = cmdline_addr.raw_value() as u32;
     params.0.hdr.cmdline_size = cmdline_size as u32;
     params.0.hdr.kernel_alignment = KERNEL_MIN_ALIGNMENT_BYTES;
+    if let Some(initrd_config) = initrd {
+        params.0.hdr.ramdisk_image = initrd_config.address.raw_value() as u32;
+        params.0.hdr.ramdisk_size = initrd_config.size as u32;
+    }
 
     add_e820_entry(&mut params.0, 0, EBDA_START, E820_RAM)?;
 
@@ -197,7 +218,7 @@ mod tests {
     fn test_system_configuration() {
         let no_vcpus = 4;
         let gm = GuestMemory::new(&[(GuestAddress(0), 0x10000)]).unwrap();
-        let config_err = configure_system(&gm, GuestAddress(0), 0, 1);
+        let config_err = configure_system(&gm, GuestAddress(0), 0, &None, 1);
         assert!(config_err.is_err());
         assert_eq!(
             config_err.unwrap_err(),
@@ -208,19 +229,19 @@ mod tests {
         let mem_size = 128 << 20;
         let arch_mem_regions = arch_memory_regions(mem_size);
         let gm = GuestMemory::new(&arch_mem_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), 0, no_vcpus).unwrap();
+        configure_system(&gm, GuestAddress(0), 0, &None, no_vcpus).unwrap();
 
         // Now assigning some memory that is equal to the start of the 32bit memory hole.
         let mem_size = 3328 << 20;
         let arch_mem_regions = arch_memory_regions(mem_size);
         let gm = GuestMemory::new(&arch_mem_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), 0, no_vcpus).unwrap();
+        configure_system(&gm, GuestAddress(0), 0, &None, no_vcpus).unwrap();
 
         // Now assigning some memory that falls after the 32bit memory hole.
         let mem_size = 3330 << 20;
         let arch_mem_regions = arch_memory_regions(mem_size);
         let gm = GuestMemory::new(&arch_mem_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), 0, no_vcpus).unwrap();
+        configure_system(&gm, GuestAddress(0), 0, &None, no_vcpus).unwrap();
     }
 
     #[test]

src/arch/src/x86_64/mod.rs

@@ -162,6 +162,15 @@ impl GuestMemory {
         self.regions.len()
     }
 
+    /// Returns the size of the region identified by passed index
+    pub fn region_size(&self, index: usize) -> Result<usize> {
+        if index >= self.regions.len() {
+            return Err(Error::NoMemoryRegions);
+        }
+
+        Ok(self.regions[index].mapping.size())
+    }
+
     /// Perform the specified action on each region's addresses.
     pub fn with_regions<F, E>(&self, cb: F) -> result::Result<(), E>
     where
@@ -747,4 +756,15 @@ mod tests {
             3
         );
     }
+
+    #[test]
+    fn test_region_size() {
+        let start_addr1 = GuestAddress(0x0);
+        let start_addr2 = GuestAddress(0x1000);
+        let mem = GuestMemory::new(&[(start_addr1, 0x100), (start_addr2, 0x400)]).unwrap();
+
+        assert_eq!(mem.region_size(0).unwrap(), 0x100);
+        assert_eq!(mem.region_size(1).unwrap(), 0x400);
+        assert!(mem.region_size(2).is_err());
+    }
 }