Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 1 | // Copyright 2018 The Chromium OS Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style license that can be |
| 3 | // found in the LICENSE file. |
| 4 | |
| 5 | extern crate arch; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 6 | extern crate data_model; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 7 | extern crate devices; |
Daniel Verkamp | 56f283b | 2018-10-05 11:40:59 -0700 | [diff] [blame] | 8 | extern crate io_jail; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 9 | extern crate kernel_cmdline; |
| 10 | extern crate kvm; |
| 11 | extern crate kvm_sys; |
| 12 | extern crate libc; |
David Tolnay | 3df3552 | 2019-03-11 12:36:30 -0700 | [diff] [blame^] | 13 | extern crate remain; |
Dylan Reid | ef7352f | 2018-05-17 18:47:11 -0700 | [diff] [blame] | 14 | extern crate resources; |
David Tolnay | 1d4d44a | 2018-12-03 23:37:46 -0800 | [diff] [blame] | 15 | extern crate sync; |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 16 | extern crate sys_util; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 17 | |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 18 | use std::error::Error as StdError; |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 19 | use std::ffi::{CStr, CString}; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 20 | use std::fmt::{self, Display}; |
| 21 | use std::fs::File; |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 22 | use std::io::{self, stdout}; |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 23 | use std::os::unix::io::FromRawFd; |
David Tolnay | 1d4d44a | 2018-12-03 23:37:46 -0800 | [diff] [blame] | 24 | use std::sync::Arc; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 25 | |
Daniel Verkamp | 56f283b | 2018-10-05 11:40:59 -0700 | [diff] [blame] | 26 | use arch::{RunnableLinuxVm, VmComponents}; |
| 27 | use devices::{Bus, BusError, PciConfigMmio, PciDevice, PciInterruptPin}; |
| 28 | use io_jail::Minijail; |
David Tolnay | 3df3552 | 2019-03-11 12:36:30 -0700 | [diff] [blame^] | 29 | use remain::sorted; |
Dylan Reid | ef7352f | 2018-05-17 18:47:11 -0700 | [diff] [blame] | 30 | use resources::{AddressRanges, SystemAllocator}; |
David Tolnay | 1d4d44a | 2018-12-03 23:37:46 -0800 | [diff] [blame] | 31 | use sync::Mutex; |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 32 | use sys_util::{EventFd, GuestAddress, GuestMemory, GuestMemoryError}; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 33 | |
| 34 | use kvm::*; |
| 35 | use kvm_sys::kvm_device_attr; |
| 36 | |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 37 | mod fdt; |
| 38 | |
| 39 | // We place the kernel at offset 8MB |
| 40 | const AARCH64_KERNEL_OFFSET: u64 = 0x80000; |
| 41 | const AARCH64_FDT_MAX_SIZE: u64 = 0x200000; |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 42 | const AARCH64_INITRD_ALIGN: u64 = 0x1000000; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 43 | |
| 44 | // These constants indicate the address space used by the ARM vGIC. |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 45 | const AARCH64_GIC_DIST_SIZE: u64 = 0x10000; |
| 46 | const AARCH64_GIC_CPUI_SIZE: u64 = 0x20000; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 47 | |
| 48 | // This indicates the start of DRAM inside the physical address space. |
| 49 | const AARCH64_PHYS_MEM_START: u64 = 0x80000000; |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 50 | const AARCH64_AXI_BASE: u64 = 0x40000000; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 51 | |
| 52 | // These constants indicate the placement of the GIC registers in the physical |
| 53 | // address space. |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 54 | const AARCH64_GIC_DIST_BASE: u64 = AARCH64_AXI_BASE - AARCH64_GIC_DIST_SIZE; |
| 55 | const AARCH64_GIC_CPUI_BASE: u64 = AARCH64_GIC_DIST_BASE - AARCH64_GIC_CPUI_SIZE; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 56 | |
| 57 | // This is the minimum number of SPI interrupts aligned to 32 + 32 for the |
| 58 | // PPI (16) and GSI (16). |
| 59 | const AARCH64_GIC_NR_IRQS: u32 = 64; |
| 60 | |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 61 | // PSR (Processor State Register) bits |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 62 | const PSR_MODE_EL1H: u64 = 0x00000005; |
| 63 | const PSR_F_BIT: u64 = 0x00000040; |
| 64 | const PSR_I_BIT: u64 = 0x00000080; |
| 65 | const PSR_A_BIT: u64 = 0x00000100; |
| 66 | const PSR_D_BIT: u64 = 0x00000200; |
| 67 | |
| 68 | macro_rules! offset__of { |
| 69 | ($str:ty, $($field:ident).+ $([$idx:expr])*) => { |
| 70 | unsafe { &(*(0 as *const $str))$(.$field)* $([$idx])* as *const _ as usize } |
| 71 | } |
| 72 | } |
| 73 | |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 74 | const KVM_REG_ARM64: u64 = 0x6000000000000000; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 75 | const KVM_REG_SIZE_U64: u64 = 0x0030000000000000; |
| 76 | const KVM_REG_ARM_COPROC_SHIFT: u64 = 16; |
| 77 | const KVM_REG_ARM_CORE: u64 = 0x0010 << KVM_REG_ARM_COPROC_SHIFT; |
| 78 | |
| 79 | macro_rules! arm64_core_reg { |
| 80 | ($reg: tt) => { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 81 | KVM_REG_ARM64 |
| 82 | | KVM_REG_SIZE_U64 |
| 83 | | KVM_REG_ARM_CORE |
| 84 | | ((offset__of!(kvm_sys::user_pt_regs, $reg) / 4) as u64) |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 85 | }; |
| 86 | } |
| 87 | |
| 88 | fn get_kernel_addr() -> GuestAddress { |
| 89 | GuestAddress(AARCH64_PHYS_MEM_START + AARCH64_KERNEL_OFFSET) |
| 90 | } |
| 91 | |
| 92 | // Place the serial device at a typical address for x86. |
| 93 | const AARCH64_SERIAL_ADDR: u64 = 0x3F8; |
| 94 | // Serial device requires 8 bytes of registers; |
| 95 | const AARCH64_SERIAL_SIZE: u64 = 0x8; |
| 96 | // This was the speed kvmtool used, not sure if it matters. |
| 97 | const AARCH64_SERIAL_SPEED: u32 = 1843200; |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 98 | // The serial device gets the first interrupt line |
| 99 | // Which gets mapped to the first SPI interrupt (physical 32). |
| 100 | const AARCH64_SERIAL_IRQ: u32 = 0; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 101 | |
Sonny Rao | e082339 | 2018-05-01 18:55:05 -0700 | [diff] [blame] | 102 | // Place the RTC device at page 2 |
| 103 | const AARCH64_RTC_ADDR: u64 = 0x2000; |
| 104 | // The RTC device gets one 4k page |
| 105 | const AARCH64_RTC_SIZE: u64 = 0x1000; |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 106 | // The RTC device gets the second interrupt line |
| 107 | const AARCH64_RTC_IRQ: u32 = 1; |
Sonny Rao | e082339 | 2018-05-01 18:55:05 -0700 | [diff] [blame] | 108 | |
Daniel Verkamp | 948b5f7 | 2018-09-24 17:51:50 -0700 | [diff] [blame] | 109 | // PCI MMIO configuration region base address. |
| 110 | const AARCH64_PCI_CFG_BASE: u64 = 0x10000; |
| 111 | // PCI MMIO configuration region size. |
| 112 | const AARCH64_PCI_CFG_SIZE: u64 = 0x1000000; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 113 | // This is the base address of MMIO devices. |
Daniel Verkamp | 948b5f7 | 2018-09-24 17:51:50 -0700 | [diff] [blame] | 114 | const AARCH64_MMIO_BASE: u64 = 0x1010000; |
| 115 | // Size of the whole MMIO region. |
| 116 | const AARCH64_MMIO_SIZE: u64 = 0x100000; |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 117 | // Virtio devices start at SPI interrupt number 2 |
| 118 | const AARCH64_IRQ_BASE: u32 = 2; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 119 | |
David Tolnay | 3df3552 | 2019-03-11 12:36:30 -0700 | [diff] [blame^] | 120 | #[sorted] |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 121 | #[derive(Debug)] |
| 122 | pub enum Error { |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 123 | CloneEventFd(sys_util::Error), |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 124 | Cmdline(kernel_cmdline::Error), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 125 | CreateDevices(Box<dyn StdError>), |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 126 | CreateEventFd(sys_util::Error), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 127 | CreateFdt(arch::fdt::Error), |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 128 | CreateGICFailure(sys_util::Error), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 129 | CreateKvm(sys_util::Error), |
| 130 | CreatePciRoot(arch::DeviceRegistrationError), |
| 131 | CreateSocket(io::Error), |
| 132 | CreateVcpu(sys_util::Error), |
| 133 | CreateVm(sys_util::Error), |
| 134 | InitrdLoadFailure(arch::LoadImageError), |
| 135 | KernelLoadFailure(arch::LoadImageError), |
| 136 | ReadPreferredTarget(sys_util::Error), |
| 137 | RegisterIrqfd(sys_util::Error), |
Daniel Verkamp | 948b5f7 | 2018-09-24 17:51:50 -0700 | [diff] [blame] | 138 | RegisterPci(BusError), |
Dylan Reid | 0f579cb | 2018-07-09 15:39:34 -0700 | [diff] [blame] | 139 | RegisterVsock(arch::DeviceRegistrationError), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 140 | SetDeviceAttr(sys_util::Error), |
| 141 | SetReg(sys_util::Error), |
| 142 | SetupGuestMemory(GuestMemoryError), |
| 143 | VcpuInit(sys_util::Error), |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 144 | } |
| 145 | |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 146 | impl Display for Error { |
David Tolnay | 3df3552 | 2019-03-11 12:36:30 -0700 | [diff] [blame^] | 147 | #[remain::check] |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 148 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
David Tolnay | c69f975 | 2019-03-01 18:07:56 -0800 | [diff] [blame] | 149 | use self::Error::*; |
| 150 | |
David Tolnay | 3df3552 | 2019-03-11 12:36:30 -0700 | [diff] [blame^] | 151 | #[sorted] |
David Tolnay | c69f975 | 2019-03-01 18:07:56 -0800 | [diff] [blame] | 152 | match self { |
| 153 | CloneEventFd(e) => write!(f, "unable to clone an EventFd: {}", e), |
| 154 | Cmdline(e) => write!(f, "the given kernel command line was invalid: {}", e), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 155 | CreateDevices(e) => write!(f, "error creating devices: {}", e), |
David Tolnay | c69f975 | 2019-03-01 18:07:56 -0800 | [diff] [blame] | 156 | CreateEventFd(e) => write!(f, "unable to make an EventFd: {}", e), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 157 | CreateFdt(e) => write!(f, "FDT could not be created: {}", e), |
| 158 | CreateGICFailure(e) => write!(f, "failed to create GIC: {}", e), |
David Tolnay | c69f975 | 2019-03-01 18:07:56 -0800 | [diff] [blame] | 159 | CreateKvm(e) => write!(f, "failed to open /dev/kvm: {}", e), |
| 160 | CreatePciRoot(e) => write!(f, "failed to create a PCI root hub: {}", e), |
| 161 | CreateSocket(e) => write!(f, "failed to create socket: {}", e), |
| 162 | CreateVcpu(e) => write!(f, "failed to create VCPU: {}", e), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 163 | CreateVm(e) => write!(f, "failed to create vm: {}", e), |
David Tolnay | c69f975 | 2019-03-01 18:07:56 -0800 | [diff] [blame] | 164 | InitrdLoadFailure(e) => write!(f, "initrd cound not be loaded: {}", e), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 165 | KernelLoadFailure(e) => write!(f, "kernel cound not be loaded: {}", e), |
| 166 | ReadPreferredTarget(e) => write!(f, "failed to read preferred target: {}", e), |
| 167 | RegisterIrqfd(e) => write!(f, "failed to register irq fd: {}", e), |
David Tolnay | c69f975 | 2019-03-01 18:07:56 -0800 | [diff] [blame] | 168 | RegisterPci(e) => write!(f, "error registering PCI bus: {}", e), |
| 169 | RegisterVsock(e) => write!(f, "error registering virtual socket device: {}", e), |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 170 | SetDeviceAttr(e) => write!(f, "failed to set device attr: {}", e), |
| 171 | SetReg(e) => write!(f, "failed to set register: {}", e), |
| 172 | SetupGuestMemory(e) => write!(f, "failed to set up guest memory: {}", e), |
| 173 | VcpuInit(e) => write!(f, "failed to initialize VCPU: {}", e), |
David Tolnay | c69f975 | 2019-03-01 18:07:56 -0800 | [diff] [blame] | 174 | } |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 175 | } |
| 176 | } |
| 177 | |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 178 | pub type Result<T> = std::result::Result<T, Error>; |
| 179 | |
| 180 | impl std::error::Error for Error {} |
| 181 | |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 182 | /// Returns a Vec of the valid memory addresses. |
| 183 | /// These should be used to configure the GuestMemory structure for the platfrom. |
| 184 | pub fn arch_memory_regions(size: u64) -> Vec<(GuestAddress, u64)> { |
| 185 | vec![(GuestAddress(AARCH64_PHYS_MEM_START), size)] |
| 186 | } |
| 187 | |
| 188 | fn fdt_offset(mem_size: u64) -> u64 { |
| 189 | // Put fdt up near the top of memory |
| 190 | // TODO(sonnyrao): will have to handle this differently if there's |
| 191 | // > 4GB memory |
| 192 | mem_size - AARCH64_FDT_MAX_SIZE - 0x10000 |
| 193 | } |
| 194 | |
| 195 | pub struct AArch64; |
| 196 | |
| 197 | impl arch::LinuxArch for AArch64 { |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 198 | type Error = Error; |
| 199 | |
| 200 | fn build_vm<F, E>( |
Miriam Zimmerman | 26ac928 | 2019-01-29 21:21:48 -0800 | [diff] [blame] | 201 | mut components: VmComponents, |
| 202 | _split_irqchip: bool, |
Jianxun Zhang | 96f2d8e | 2019-02-20 13:50:42 -0800 | [diff] [blame] | 203 | create_devices: F, |
Miriam Zimmerman | 26ac928 | 2019-01-29 21:21:48 -0800 | [diff] [blame] | 204 | ) -> Result<RunnableLinuxVm> |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 205 | where |
David Tolnay | 2bac1e7 | 2018-12-12 14:33:42 -0800 | [diff] [blame] | 206 | F: FnOnce( |
| 207 | &GuestMemory, |
| 208 | &EventFd, |
David Tolnay | fdac5ed | 2019-03-08 16:56:14 -0800 | [diff] [blame] | 209 | ) -> std::result::Result<Vec<(Box<dyn PciDevice>, Option<Minijail>)>, E>, |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 210 | E: StdError + 'static, |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 211 | { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 212 | let mut resources = |
| 213 | Self::get_resource_allocator(components.memory_mb, components.wayland_dmabuf); |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 214 | let mem = Self::setup_memory(components.memory_mb)?; |
| 215 | let kvm = Kvm::new().map_err(Error::CreateKvm)?; |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 216 | let mut vm = Vm::new(&kvm, mem.clone()).map_err(Error::CreateVm)?; |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 217 | |
| 218 | let vcpu_count = components.vcpu_count; |
| 219 | let mut vcpus = Vec::with_capacity(vcpu_count as usize); |
| 220 | for cpu_id in 0..vcpu_count { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 221 | let vcpu = Vcpu::new(cpu_id as libc::c_ulong, &kvm, &vm).map_err(Error::CreateVcpu)?; |
| 222 | Self::configure_vcpu( |
| 223 | vm.get_memory(), |
| 224 | &kvm, |
| 225 | &vm, |
| 226 | &vcpu, |
| 227 | cpu_id as u64, |
| 228 | vcpu_count as u64, |
| 229 | )?; |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 230 | vcpus.push(vcpu); |
| 231 | } |
| 232 | |
Daniel Verkamp | 107edb3 | 2019-04-05 09:58:48 -0700 | [diff] [blame] | 233 | let vcpu_affinity = components.vcpu_affinity; |
| 234 | |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 235 | let irq_chip = Self::create_irq_chip(&vm)?; |
| 236 | let mut cmdline = Self::get_base_linux_cmdline(); |
| 237 | |
| 238 | let mut mmio_bus = devices::Bus::new(); |
| 239 | |
Daniel Verkamp | c8986f1 | 2018-10-03 13:22:59 -0700 | [diff] [blame] | 240 | let exit_evt = EventFd::new().map_err(Error::CreateEventFd)?; |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 241 | |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 242 | let pci_devices = |
| 243 | create_devices(&mem, &exit_evt).map_err(|e| Error::CreateDevices(Box::new(e)))?; |
Zach Reizner | 3ba0098 | 2019-01-23 19:04:43 -0800 | [diff] [blame] | 244 | let (pci, pci_irqs, pid_debug_label_map) = |
Daniel Verkamp | 56f283b | 2018-10-05 11:40:59 -0700 | [diff] [blame] | 245 | arch::generate_pci_root(pci_devices, &mut mmio_bus, &mut resources, &mut vm) |
| 246 | .map_err(Error::CreatePciRoot)?; |
| 247 | let pci_bus = Arc::new(Mutex::new(PciConfigMmio::new(pci))); |
| 248 | |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 249 | // ARM doesn't really use the io bus like x86, so just create an empty bus. |
| 250 | let io_bus = devices::Bus::new(); |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 251 | |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 252 | let stdio_serial = Self::add_arch_devs(&mut vm, &mut mmio_bus)?; |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 253 | |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 254 | mmio_bus |
| 255 | .insert( |
| 256 | pci_bus.clone(), |
| 257 | AARCH64_PCI_CFG_BASE, |
| 258 | AARCH64_PCI_CFG_SIZE, |
| 259 | false, |
David Tolnay | 2bac1e7 | 2018-12-12 14:33:42 -0800 | [diff] [blame] | 260 | ) |
| 261 | .map_err(Error::RegisterPci)?; |
Daniel Verkamp | 948b5f7 | 2018-09-24 17:51:50 -0700 | [diff] [blame] | 262 | |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 263 | for param in components.extra_kernel_params { |
| 264 | cmdline.insert_str(¶m).map_err(Error::Cmdline)?; |
| 265 | } |
| 266 | |
Daniel Verkamp | c195616 | 2018-12-12 15:20:30 -0800 | [diff] [blame] | 267 | // separate out kernel loading from other setup to get a specific error for |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 268 | // kernel loading |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 269 | let kernel_size = arch::load_image( |
Daniel Verkamp | c195616 | 2018-12-12 15:20:30 -0800 | [diff] [blame] | 270 | &mem, |
| 271 | &mut components.kernel_image, |
| 272 | get_kernel_addr(), |
| 273 | u64::max_value(), |
| 274 | ) |
| 275 | .map_err(Error::KernelLoadFailure)?; |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 276 | let kernel_end = get_kernel_addr().offset() + kernel_size as u64; |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 277 | Self::setup_system_memory( |
| 278 | &mem, |
| 279 | components.memory_mb, |
| 280 | vcpu_count, |
| 281 | &CString::new(cmdline).unwrap(), |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 282 | components.initrd_image, |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 283 | pci_irqs, |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 284 | kernel_end, |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 285 | )?; |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 286 | |
| 287 | Ok(RunnableLinuxVm { |
| 288 | vm, |
| 289 | kvm, |
| 290 | resources, |
| 291 | stdio_serial, |
| 292 | exit_evt, |
| 293 | vcpus, |
Daniel Verkamp | 107edb3 | 2019-04-05 09:58:48 -0700 | [diff] [blame] | 294 | vcpu_affinity, |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 295 | irq_chip, |
| 296 | io_bus, |
| 297 | mmio_bus, |
Zach Reizner | 3ba0098 | 2019-01-23 19:04:43 -0800 | [diff] [blame] | 298 | pid_debug_label_map, |
Dylan Reid | 059a188 | 2018-07-23 17:58:09 -0700 | [diff] [blame] | 299 | }) |
| 300 | } |
| 301 | } |
| 302 | |
| 303 | impl AArch64 { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 304 | fn setup_system_memory( |
| 305 | mem: &GuestMemory, |
| 306 | mem_size: u64, |
| 307 | vcpu_count: u32, |
| 308 | cmdline: &CStr, |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 309 | initrd_file: Option<File>, |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 310 | pci_irqs: Vec<(u32, PciInterruptPin)>, |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 311 | kernel_end: u64, |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 312 | ) -> Result<()> { |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 313 | let initrd = match initrd_file { |
| 314 | Some(initrd_file) => { |
| 315 | let mut initrd_file = initrd_file; |
| 316 | let initrd_addr = |
| 317 | (kernel_end + (AARCH64_INITRD_ALIGN - 1)) & !(AARCH64_INITRD_ALIGN - 1); |
| 318 | let initrd_max_size = mem_size - (initrd_addr - AARCH64_PHYS_MEM_START); |
| 319 | let initrd_addr = GuestAddress(initrd_addr); |
| 320 | let initrd_size = |
| 321 | arch::load_image(mem, &mut initrd_file, initrd_addr, initrd_max_size) |
| 322 | .map_err(Error::InitrdLoadFailure)?; |
| 323 | Some((initrd_addr, initrd_size)) |
| 324 | } |
| 325 | None => None, |
| 326 | }; |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 327 | fdt::create_fdt( |
| 328 | AARCH64_FDT_MAX_SIZE as usize, |
| 329 | mem, |
| 330 | pci_irqs, |
| 331 | vcpu_count, |
| 332 | fdt_offset(mem_size), |
| 333 | cmdline, |
Daniel Verkamp | e403f5c | 2018-12-11 16:29:26 -0800 | [diff] [blame] | 334 | initrd, |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 335 | ) |
| 336 | .map_err(Error::CreateFdt)?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 337 | Ok(()) |
| 338 | } |
| 339 | |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 340 | fn setup_memory(mem_size: u64) -> Result<GuestMemory> { |
| 341 | let arch_mem_regions = arch_memory_regions(mem_size); |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 342 | let mem = GuestMemory::new(&arch_mem_regions).map_err(Error::SetupGuestMemory)?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 343 | Ok(mem) |
| 344 | } |
| 345 | |
| 346 | fn get_base_dev_pfn(mem_size: u64) -> u64 { |
Sonny Rao | 5165cb7 | 2018-05-07 18:30:10 -0700 | [diff] [blame] | 347 | (AARCH64_PHYS_MEM_START + mem_size) >> 12 |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 348 | } |
| 349 | |
| 350 | /// This returns a base part of the kernel command for this architecture |
| 351 | fn get_base_linux_cmdline() -> kernel_cmdline::Cmdline { |
| 352 | let mut cmdline = kernel_cmdline::Cmdline::new(sys_util::pagesize()); |
Daniel Verkamp | b852264 | 2019-03-04 13:42:06 -0800 | [diff] [blame] | 353 | cmdline.insert_str("console=ttyS0 panic=-1").unwrap(); |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 354 | cmdline |
| 355 | } |
| 356 | |
Dylan Reid | ef7352f | 2018-05-17 18:47:11 -0700 | [diff] [blame] | 357 | /// Returns a system resource allocator. |
Dylan Reid | 228e4a6 | 2018-06-07 15:42:41 -0700 | [diff] [blame] | 358 | fn get_resource_allocator(mem_size: u64, gpu_allocation: bool) -> SystemAllocator { |
Dylan Reid | ef7352f | 2018-05-17 18:47:11 -0700 | [diff] [blame] | 359 | let device_addr_start = Self::get_base_dev_pfn(mem_size) * sys_util::pagesize() as u64; |
| 360 | AddressRanges::new() |
| 361 | .add_device_addresses(device_addr_start, u64::max_value() - device_addr_start) |
Daniel Verkamp | 948b5f7 | 2018-09-24 17:51:50 -0700 | [diff] [blame] | 362 | .add_mmio_addresses(AARCH64_MMIO_BASE, AARCH64_MMIO_SIZE) |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 363 | .create_allocator(AARCH64_IRQ_BASE, gpu_allocation) |
| 364 | .unwrap() |
Dylan Reid | ef7352f | 2018-05-17 18:47:11 -0700 | [diff] [blame] | 365 | } |
| 366 | |
| 367 | /// This adds any early platform devices for this architecture. |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 368 | /// |
| 369 | /// # Arguments |
| 370 | /// |
Dylan Reid | ef7352f | 2018-05-17 18:47:11 -0700 | [diff] [blame] | 371 | /// * `vm` - The vm to add irqs to. |
| 372 | /// * `bus` - The bus to add devices to. |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 373 | fn add_arch_devs(vm: &mut Vm, bus: &mut Bus) -> Result<Arc<Mutex<devices::Serial>>> { |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 374 | let rtc_evt = EventFd::new().map_err(Error::CreateEventFd)?; |
| 375 | vm.register_irqfd(&rtc_evt, AARCH64_RTC_IRQ) |
| 376 | .map_err(Error::RegisterIrqfd)?; |
Sonny Rao | e082339 | 2018-05-01 18:55:05 -0700 | [diff] [blame] | 377 | |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 378 | let com_evt_1_3 = EventFd::new().map_err(Error::CreateEventFd)?; |
| 379 | vm.register_irqfd(&com_evt_1_3, AARCH64_SERIAL_IRQ) |
| 380 | .map_err(Error::RegisterIrqfd)?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 381 | let serial = Arc::new(Mutex::new(devices::Serial::new_out( |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 382 | com_evt_1_3.try_clone().map_err(Error::CloneEventFd)?, |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 383 | Box::new(stdout()), |
| 384 | ))); |
| 385 | bus.insert( |
| 386 | serial.clone(), |
| 387 | AARCH64_SERIAL_ADDR, |
| 388 | AARCH64_SERIAL_SIZE, |
| 389 | false, |
David Tolnay | 2bac1e7 | 2018-12-12 14:33:42 -0800 | [diff] [blame] | 390 | ) |
| 391 | .expect("failed to add serial device"); |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 392 | |
Sonny Rao | e082339 | 2018-05-01 18:55:05 -0700 | [diff] [blame] | 393 | let rtc = Arc::new(Mutex::new(devices::pl030::Pl030::new(rtc_evt))); |
Dylan Reid | 836466a | 2018-05-23 17:57:05 -0700 | [diff] [blame] | 394 | bus.insert(rtc, AARCH64_RTC_ADDR, AARCH64_RTC_SIZE, false) |
Dylan Reid | ef7352f | 2018-05-17 18:47:11 -0700 | [diff] [blame] | 395 | .expect("failed to add rtc device"); |
Sonny Rao | c7af4b1 | 2018-06-25 18:51:03 -0700 | [diff] [blame] | 396 | Ok(serial) |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 397 | } |
| 398 | |
| 399 | /// The creates the interrupt controller device and optionally returns the fd for it. |
| 400 | /// Some architectures may not have a separate descriptor for the interrupt |
| 401 | /// controller, so they would return None even on success. |
| 402 | /// |
| 403 | /// # Arguments |
| 404 | /// |
| 405 | /// * `vm` - the vm object |
| 406 | fn create_irq_chip(vm: &Vm) -> Result<Option<File>> { |
| 407 | let cpu_if_addr: u64 = AARCH64_GIC_CPUI_BASE; |
| 408 | let dist_if_addr: u64 = AARCH64_GIC_DIST_BASE; |
| 409 | let raw_cpu_if_addr = &cpu_if_addr as *const u64; |
| 410 | let raw_dist_if_addr = &dist_if_addr as *const u64; |
| 411 | |
| 412 | let cpu_if_attr = kvm_device_attr { |
| 413 | group: kvm_sys::KVM_DEV_ARM_VGIC_GRP_ADDR, |
| 414 | attr: kvm_sys::KVM_VGIC_V2_ADDR_TYPE_CPU as u64, |
| 415 | addr: raw_cpu_if_addr as u64, |
| 416 | flags: 0, |
| 417 | }; |
| 418 | let dist_attr = kvm_device_attr { |
| 419 | group: kvm_sys::KVM_DEV_ARM_VGIC_GRP_ADDR, |
| 420 | attr: kvm_sys::KVM_VGIC_V2_ADDR_TYPE_DIST as u64, |
| 421 | addr: raw_dist_if_addr as u64, |
| 422 | flags: 0, |
| 423 | }; |
| 424 | let mut kcd = kvm_sys::kvm_create_device { |
| 425 | type_: kvm_sys::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V2, |
| 426 | fd: 0, |
| 427 | flags: 0, |
| 428 | }; |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 429 | vm.create_device(&mut kcd) |
| 430 | .map_err(|e| Error::CreateGICFailure(e))?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 431 | |
| 432 | // Safe because the kernel is passing us an FD back inside |
| 433 | // the struct after we successfully did the create_device ioctl |
| 434 | let vgic_fd = unsafe { File::from_raw_fd(kcd.fd as i32) }; |
| 435 | |
| 436 | // Safe because we allocated the struct that's being passed in |
| 437 | let ret = unsafe { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 438 | sys_util::ioctl_with_ref(&vgic_fd, kvm_sys::KVM_SET_DEVICE_ATTR(), &cpu_if_attr) |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 439 | }; |
| 440 | if ret != 0 { |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 441 | return Err(Error::CreateGICFailure(sys_util::Error::new(ret))); |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 442 | } |
| 443 | |
| 444 | // Safe because we allocated the struct that's being passed in |
| 445 | let ret = unsafe { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 446 | sys_util::ioctl_with_ref(&vgic_fd, kvm_sys::KVM_SET_DEVICE_ATTR(), &dist_attr) |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 447 | }; |
| 448 | if ret != 0 { |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 449 | return Err(Error::CreateGICFailure(sys_util::Error::new(ret))); |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 450 | } |
| 451 | |
| 452 | // We need to tell the kernel how many irqs to support with this vgic |
| 453 | let nr_irqs: u32 = AARCH64_GIC_NR_IRQS; |
| 454 | let nr_irqs_ptr = &nr_irqs as *const u32; |
| 455 | let nr_irqs_attr = kvm_device_attr { |
| 456 | group: kvm_sys::KVM_DEV_ARM_VGIC_GRP_NR_IRQS, |
| 457 | attr: 0, |
| 458 | addr: nr_irqs_ptr as u64, |
| 459 | flags: 0, |
| 460 | }; |
| 461 | // Safe because we allocated the struct that's being passed in |
| 462 | let ret = unsafe { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 463 | sys_util::ioctl_with_ref(&vgic_fd, kvm_sys::KVM_SET_DEVICE_ATTR(), &nr_irqs_attr) |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 464 | }; |
| 465 | if ret != 0 { |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 466 | return Err(Error::CreateGICFailure(sys_util::Error::new(ret))); |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 467 | } |
| 468 | |
| 469 | // Finalize the GIC |
| 470 | let init_gic_attr = kvm_device_attr { |
| 471 | group: kvm_sys::KVM_DEV_ARM_VGIC_GRP_CTRL, |
| 472 | attr: kvm_sys::KVM_DEV_ARM_VGIC_CTRL_INIT as u64, |
| 473 | addr: 0, |
| 474 | flags: 0, |
| 475 | }; |
| 476 | |
| 477 | // Safe because we allocated the struct that's being passed in |
| 478 | let ret = unsafe { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 479 | sys_util::ioctl_with_ref(&vgic_fd, kvm_sys::KVM_SET_DEVICE_ATTR(), &init_gic_attr) |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 480 | }; |
| 481 | if ret != 0 { |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 482 | return Err(Error::SetDeviceAttr(sys_util::Error::new(ret))); |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 483 | } |
| 484 | Ok(Some(vgic_fd)) |
| 485 | } |
| 486 | |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 487 | fn configure_vcpu( |
| 488 | guest_mem: &GuestMemory, |
| 489 | _kvm: &Kvm, |
| 490 | vm: &Vm, |
| 491 | vcpu: &Vcpu, |
| 492 | cpu_id: u64, |
| 493 | _num_cpus: u64, |
| 494 | ) -> Result<()> { |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 495 | let mut kvi = kvm_sys::kvm_vcpu_init { |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 496 | target: kvm_sys::KVM_ARM_TARGET_GENERIC_V8, |
| 497 | features: [0; 7], |
| 498 | }; |
| 499 | |
Sonny Rao | a7fae25 | 2018-03-27 16:30:51 -0700 | [diff] [blame] | 500 | // This reads back the kernel's preferred target type. |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 501 | vm.arm_preferred_target(&mut kvi) |
| 502 | .map_err(Error::ReadPreferredTarget)?; |
Sonny Rao | a7fae25 | 2018-03-27 16:30:51 -0700 | [diff] [blame] | 503 | |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 504 | // TODO(sonnyrao): need to verify this feature is supported by host kernel |
| 505 | kvi.features[0] |= 1 << kvm_sys::KVM_ARM_VCPU_PSCI_0_2; |
| 506 | |
| 507 | // Non-boot cpus are powered off initially |
| 508 | if cpu_id > 0 { |
| 509 | kvi.features[0] |= 1 << kvm_sys::KVM_ARM_VCPU_POWER_OFF; |
| 510 | } |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 511 | vcpu.arm_vcpu_init(&kvi).map_err(Error::VcpuInit)?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 512 | |
| 513 | // set up registers |
| 514 | let mut data: u64; |
| 515 | let mut reg_id: u64; |
| 516 | |
| 517 | // All interrupts masked |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 518 | data = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1H; |
| 519 | reg_id = arm64_core_reg!(pstate); |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 520 | vcpu.set_one_reg(reg_id, data).map_err(Error::SetReg)?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 521 | |
| 522 | // Other cpus are powered off initially |
| 523 | if cpu_id == 0 { |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 524 | data = AARCH64_PHYS_MEM_START + AARCH64_KERNEL_OFFSET; |
| 525 | reg_id = arm64_core_reg!(pc); |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 526 | vcpu.set_one_reg(reg_id, data).map_err(Error::SetReg)?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 527 | |
| 528 | /* X0 -- fdt address */ |
| 529 | let mem_size = guest_mem.memory_size(); |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 530 | data = (AARCH64_PHYS_MEM_START + fdt_offset(mem_size)) as u64; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 531 | // hack -- can't get this to do offsetof(regs[0]) but luckily it's at offset 0 |
Zach Reizner | 55a9e50 | 2018-10-03 10:22:32 -0700 | [diff] [blame] | 532 | reg_id = arm64_core_reg!(regs); |
David Tolnay | be03426 | 2019-03-04 17:48:36 -0800 | [diff] [blame] | 533 | vcpu.set_one_reg(reg_id, data).map_err(Error::SetReg)?; |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 534 | } |
| 535 | Ok(()) |
| 536 | } |
Sonny Rao | 2ffa0cb | 2018-02-26 17:27:40 -0800 | [diff] [blame] | 537 | } |