Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / crosvm / 8be9ef0a204a84be85b5db4fb1be1d4e31229163 / . / src / linux.rs
blob: 1caed75db6e8e37bf017bf9d0dccb796afc99717 [file] [log] [blame]
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use std::cmp::{max, Reverse};
use std::collections::{BTreeMap, HashSet};
use std::convert::{TryFrom, TryInto};
#[cfg(feature = "gpu")]
use std::env;
use std::fs::{File, OpenOptions};
use std::io::prelude::*;
use std::io::stdin;
use std::iter;
use std::mem;
use std::net::Ipv4Addr;
use std::os::unix::net::UnixListener;
use std::os::unix::{io::FromRawFd, net::UnixStream, prelude::OpenOptionsExt};
use std::path::{Path, PathBuf};
use std::str;
use std::sync::{mpsc, Arc, Barrier};
use std::time::Duration;
use std::process;
use std::thread;
use std::thread::JoinHandle;
use libc::{self, c_int, c_ulong, gid_t, uid_t};
use acpi_tables::sdt::SDT;
use anyhow::{anyhow, bail, Context, Result};
use base::net::{UnixSeqpacket, UnixSeqpacketListener, UnlinkUnixSeqpacketListener};
use base::*;
use devices::serial_device::{SerialHardware, SerialParameters};
use devices::vfio::{VfioCommonSetup, VfioCommonTrait};
#[cfg(feature = "audio_cras")]
use devices::virtio::snd::cras_backend::Parameters as CrasSndParameters;
use devices::virtio::vhost::user::proxy::VirtioVhostUser;
#[cfg(feature = "audio")]
use devices::virtio::vhost::user::vmm::Snd as VhostUserSnd;
use devices::virtio::vhost::user::vmm::{
Block as VhostUserBlock, Console as VhostUserConsole, Fs as VhostUserFs,
Mac80211Hwsim as VhostUserMac80211Hwsim, Net as VhostUserNet, Vsock as VhostUserVsock,
Wl as VhostUserWl,
};
#[cfg(any(feature = "video-decoder", feature = "video-encoder"))]
use devices::virtio::VideoBackendType;
use devices::virtio::{self, Console, VirtioDevice};
#[cfg(feature = "gpu")]
use devices::virtio::{
gpu::{GpuRenderServerParameters, DEFAULT_DISPLAY_HEIGHT, DEFAULT_DISPLAY_WIDTH},
vhost::user::vmm::Gpu as VhostUserGpu,
EventDevice,
};
#[cfg(feature = "audio")]
use devices::Ac97Dev;
use devices::{
self, BusDeviceObj, HostHotPlugKey, HotPlugBus, IrqChip, IrqEventIndex, KvmKernelIrqChip,
PciAddress, PciBridge, PciDevice, PcieRootPort, StubPciDevice, VcpuRunState, VfioContainer,
VfioDevice, VfioPciDevice, VfioPlatformDevice, VirtioPciDevice,
};
use devices::{CoIommuDev, IommuDevType};
#[cfg(feature = "usb")]
use devices::{HostBackendDeviceProvider, XhciController};
use hypervisor::kvm::{Kvm, KvmVcpu, KvmVm};
use hypervisor::{HypervisorCap, ProtectionType, Vcpu, VcpuExit, VcpuRunHandle, Vm, VmCap};
use minijail::{self, Minijail};
use net_util::{MacAddress, Tap};
use resources::{Alloc, MmioType, SystemAllocator};
use rutabaga_gfx::RutabagaGralloc;
use sync::Mutex;
use vm_control::*;
use vm_memory::{GuestAddress, GuestMemory, MemoryPolicy};
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
use crate::gdb::{gdb_thread, GdbStub};
use crate::{
Config, DiskOption, Executable, SharedDir, SharedDirKind, TouchDeviceOption, VfioType,
VhostUserFsOption, VhostUserOption, VhostUserWlOption, VhostVsockDeviceParameter,
};
use arch::{
self, LinuxArch, RunnableLinuxVm, VcpuAffinity, VirtioDeviceStub, VmComponents, VmImage,
};
#[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
use {
aarch64::AArch64 as Arch,
devices::IrqChipAArch64 as IrqChipArch,
hypervisor::{VcpuAArch64 as VcpuArch, VmAArch64 as VmArch},
};
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
use {
devices::{IrqChipX86_64 as IrqChipArch, KvmSplitIrqChip},
hypervisor::{VcpuX86_64 as VcpuArch, VmX86_64 as VmArch},
x86_64::X8664arch as Arch,
};
enum TaggedControlTube {
Fs(Tube),
Vm(Tube),
VmMemory(Tube),
VmIrq(Tube),
VmMsync(Tube),
}
impl AsRef<Tube> for TaggedControlTube {
fn as_ref(&self) -> &Tube {
use self::TaggedControlTube::*;
match &self {
Fs(tube) | Vm(tube) | VmMemory(tube) | VmIrq(tube) | VmMsync(tube) => tube,
}
}
}
impl AsRawDescriptor for TaggedControlTube {
fn as_raw_descriptor(&self) -> RawDescriptor {
self.as_ref().as_raw_descriptor()
}
}
struct SandboxConfig<'a> {
limit_caps: bool,
log_failures: bool,
seccomp_policy: &'a Path,
uid_map: Option<&'a str>,
gid_map: Option<&'a str>,
remount_mode: Option<c_ulong>,
}
fn create_base_minijail(
root: &Path,
r_limit: Option<u64>,
config: Option<&SandboxConfig>,
) -> Result<Minijail> {
// All child jails run in a new user namespace without any users mapped,
// they run as nobody unless otherwise configured.
let mut j = Minijail::new().context("failed to jail device")?;
if let Some(config) = config {
j.namespace_pids();
j.namespace_user();
j.namespace_user_disable_setgroups();
if config.limit_caps {
// Don't need any capabilities.
j.use_caps(0);
}
if let Some(uid_map) = config.uid_map {
j.uidmap(uid_map).context("error setting UID map")?;
}
if let Some(gid_map) = config.gid_map {
j.gidmap(gid_map).context("error setting GID map")?;
}
// Run in a new mount namespace.
j.namespace_vfs();
// Run in an empty network namespace.
j.namespace_net();
// Don't allow the device to gain new privileges.
j.no_new_privs();
// By default we'll prioritize using the pre-compiled .bpf over the .policy
// file (the .bpf is expected to be compiled using "trap" as the failure
// behavior instead of the default "kill" behavior).
// Refer to the code comment for the "seccomp-log-failures"
// command-line parameter for an explanation about why the |log_failures|
// flag forces the use of .policy files (and the build-time alternative to
// this run-time flag).
let bpf_policy_file = config.seccomp_policy.with_extension("bpf");
if bpf_policy_file.exists() && !config.log_failures {
j.parse_seccomp_program(&bpf_policy_file)
.context("failed to parse precompiled seccomp policy")?;
} else {
// Use TSYNC only for the side effect of it using SECCOMP_RET_TRAP,
// which will correctly kill the entire device process if a worker
// thread commits a seccomp violation.
j.set_seccomp_filter_tsync();
if config.log_failures {
j.log_seccomp_filter_failures();
}
j.parse_seccomp_filters(&config.seccomp_policy.with_extension("policy"))
.context("failed to parse seccomp policy")?;
}
j.use_seccomp_filter();
// Don't do init setup.
j.run_as_init();
// Set up requested remount mode instead of default MS_PRIVATE.
if let Some(mode) = config.remount_mode {
j.set_remount_mode(mode);
}
}
// Only pivot_root if we are not re-using the current root directory.
if root != Path::new("/") {
// It's safe to call `namespace_vfs` multiple times.
j.namespace_vfs();
j.enter_pivot_root(root)
.context("failed to pivot root device")?;
}
// Most devices don't need to open many fds.
let limit = if let Some(r) = r_limit { r } else { 1024u64 };
j.set_rlimit(libc::RLIMIT_NOFILE as i32, limit, limit)
.context("error setting max open files")?;
Ok(j)
}
fn simple_jail(cfg: &Config, policy: &str) -> Result<Option<Minijail>> {
if cfg.sandbox {
let pivot_root: &str = option_env!("DEFAULT_PIVOT_ROOT").unwrap_or("/var/empty");
// A directory for a jailed device's pivot root.
let root_path = Path::new(pivot_root);
if !root_path.exists() {
bail!("{} doesn't exist, can't jail devices", pivot_root);
}
let policy_path: PathBuf = cfg.seccomp_policy_dir.join(policy);
let config = SandboxConfig {
limit_caps: true,
log_failures: cfg.seccomp_log_failures,
seccomp_policy: &policy_path,
uid_map: None,
gid_map: None,
remount_mode: None,
};
Ok(Some(create_base_minijail(root_path, None, Some(&config))?))
} else {
Ok(None)
}
}
type DeviceResult<T = VirtioDeviceStub> = Result<T>;
fn create_block_device(cfg: &Config, disk: &DiskOption, disk_device_tube: Tube) -> DeviceResult {
let raw_image: File = open_file(&disk.path, disk.read_only, disk.o_direct)
.with_context(|| format!("failed to load disk image {}", disk.path.display()))?;
// Lock the disk image to prevent other crosvm instances from using it.
let lock_op = if disk.read_only {
FlockOperation::LockShared
} else {
FlockOperation::LockExclusive
};
flock(&raw_image, lock_op, true).context("failed to lock disk image")?;
info!("Trying to attach block device: {}", disk.path.display());
let dev = if disk::async_ok(&raw_image).context("failed to check disk async_ok")? {
let async_file = disk::create_async_disk_file(raw_image)
.context("failed to create async virtual disk")?;
Box::new(
virtio::BlockAsync::new(
virtio::base_features(cfg.protected_vm),
async_file,
disk.read_only,
disk.sparse,
disk.block_size,
disk.id,
Some(disk_device_tube),
)
.context("failed to create block device")?,
) as Box<dyn VirtioDevice>
} else {
let disk_file = disk::create_disk_file(raw_image, disk::MAX_NESTING_DEPTH)
.context("failed to create virtual disk")?;
Box::new(
virtio::Block::new(
virtio::base_features(cfg.protected_vm),
disk_file,
disk.read_only,
disk.sparse,
disk.block_size,
disk.id,
Some(disk_device_tube),
)
.context("failed to create block device")?,
) as Box<dyn VirtioDevice>
};
Ok(VirtioDeviceStub {
dev,
jail: simple_jail(cfg, "block_device")?,
})
}
fn create_vhost_user_block_device(cfg: &Config, opt: &VhostUserOption) -> DeviceResult {
let dev = VhostUserBlock::new(virtio::base_features(cfg.protected_vm), &opt.socket)
.context("failed to set up vhost-user block device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
fn create_vhost_user_console_device(cfg: &Config, opt: &VhostUserOption) -> DeviceResult {
let dev = VhostUserConsole::new(virtio::base_features(cfg.protected_vm), &opt.socket)
.context("failed to set up vhost-user console device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
fn create_vhost_user_fs_device(cfg: &Config, option: &VhostUserFsOption) -> DeviceResult {
let dev = VhostUserFs::new(
virtio::base_features(cfg.protected_vm),
&option.socket,
&option.tag,
)
.context("failed to set up vhost-user fs device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
fn create_vhost_user_mac80211_hwsim_device(cfg: &Config, opt: &VhostUserOption) -> DeviceResult {
let dev = VhostUserMac80211Hwsim::new(virtio::base_features(cfg.protected_vm), &opt.socket)
.context("failed to set up vhost-user mac80211_hwsim device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
#[cfg(feature = "audio")]
fn create_vhost_user_snd_device(cfg: &Config, option: &VhostUserOption) -> DeviceResult {
let dev = VhostUserSnd::new(virtio::base_features(cfg.protected_vm), &option.socket)
.context("failed to set up vhost-user snd device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
fn create_vvu_proxy_device(cfg: &Config, opt: &VhostUserOption) -> DeviceResult {
let listener = UnixListener::bind(&opt.socket).map_err(|e| {
error!("failed to bind listener for vvu proxy device: {}", e);
e
})?;
let dev = VirtioVhostUser::new(virtio::base_features(cfg.protected_vm), listener)
.context("failed to create VVU proxy device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "vvu_proxy_device")?,
})
}
fn create_rng_device(cfg: &Config) -> DeviceResult {
let dev = virtio::Rng::new(virtio::base_features(cfg.protected_vm))
.context("failed to set up rng")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "rng_device")?,
})
}
#[cfg(feature = "audio_cras")]
fn create_cras_snd_device(cfg: &Config, cras_snd: CrasSndParameters) -> DeviceResult {
let dev = virtio::snd::cras_backend::VirtioSndCras::new(
virtio::base_features(cfg.protected_vm),
cras_snd,
)
.context("failed to create cras sound device")?;
let jail = match simple_jail(&cfg, "cras_snd_device")? {
Some(mut jail) => {
// Create a tmpfs in the device's root directory for cras_snd_device.
// The size is 20*1024, or 20 KB.
jail.mount_with_data(
Path::new("none"),
Path::new("/"),
"tmpfs",
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
"size=20480",
)?;
let run_cras_path = Path::new("/run/cras");
jail.mount_bind(run_cras_path, run_cras_path, true)?;
add_current_user_to_jail(&mut jail)?;
Some(jail)
}
None => None,
};
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail,
})
}
#[cfg(feature = "tpm")]
fn create_tpm_device(cfg: &Config) -> DeviceResult {
use std::ffi::CString;
use std::fs;
let tpm_storage: PathBuf;
let mut tpm_jail = simple_jail(cfg, "tpm_device")?;
match &mut tpm_jail {
Some(jail) => {
// Create a tmpfs in the device's root directory for tpm
// simulator storage. The size is 20*1024, or 20 KB.
jail.mount_with_data(
Path::new("none"),
Path::new("/"),
"tmpfs",
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
"size=20480",
)?;
let crosvm_ids = add_current_user_to_jail(jail)?;
let pid = process::id();
let tpm_pid_dir = format!("/run/vm/tpm.{}", pid);
tpm_storage = Path::new(&tpm_pid_dir).to_owned();
fs::create_dir_all(&tpm_storage).with_context(|| {
format!("failed to create tpm storage dir {}", tpm_storage.display())
})?;
let tpm_pid_dir_c = CString::new(tpm_pid_dir).expect("no nul bytes");
chown(&tpm_pid_dir_c, crosvm_ids.uid, crosvm_ids.gid)
.context("failed to chown tpm storage")?;
jail.mount_bind(&tpm_storage, &tpm_storage, true)?;
}
None => {
// Path used inside cros_sdk which does not have /run/vm.
tpm_storage = Path::new("/tmp/tpm-simulator").to_owned();
}
}
let dev = virtio::Tpm::new(tpm_storage);
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: tpm_jail,
})
}
fn create_single_touch_device(
cfg: &Config,
single_touch_spec: &TouchDeviceOption,
idx: u32,
) -> DeviceResult {
let socket = single_touch_spec
.get_path()
.into_unix_stream()
.map_err(|e| {
error!("failed configuring virtio single touch: {:?}", e);
e
})?;
let (width, height) = single_touch_spec.get_size();
let dev = virtio::new_single_touch(
idx,
socket,
width,
height,
virtio::base_features(cfg.protected_vm),
)
.context("failed to set up input device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
})
}
fn create_multi_touch_device(
cfg: &Config,
multi_touch_spec: &TouchDeviceOption,
idx: u32,
) -> DeviceResult {
let socket = multi_touch_spec
.get_path()
.into_unix_stream()
.map_err(|e| {
error!("failed configuring virtio multi touch: {:?}", e);
e
})?;
let (width, height) = multi_touch_spec.get_size();
let dev = virtio::new_multi_touch(
idx,
socket,
width,
height,
virtio::base_features(cfg.protected_vm),
)
.context("failed to set up input device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
})
}
fn create_trackpad_device(
cfg: &Config,
trackpad_spec: &TouchDeviceOption,
idx: u32,
) -> DeviceResult {
let socket = trackpad_spec.get_path().into_unix_stream().map_err(|e| {
error!("failed configuring virtio trackpad: {:#}", e);
e
})?;
let (width, height) = trackpad_spec.get_size();
let dev = virtio::new_trackpad(
idx,
socket,
width,
height,
virtio::base_features(cfg.protected_vm),
)
.context("failed to set up input device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
})
}
fn create_mouse_device<T: IntoUnixStream>(cfg: &Config, mouse_socket: T, idx: u32) -> DeviceResult {
let socket = mouse_socket.into_unix_stream().map_err(|e| {
error!("failed configuring virtio mouse: {:#}", e);
e
})?;
let dev = virtio::new_mouse(idx, socket, virtio::base_features(cfg.protected_vm))
.context("failed to set up input device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
})
}
fn create_keyboard_device<T: IntoUnixStream>(
cfg: &Config,
keyboard_socket: T,
idx: u32,
) -> DeviceResult {
let socket = keyboard_socket.into_unix_stream().map_err(|e| {
error!("failed configuring virtio keyboard: {:#}", e);
e
})?;
let dev = virtio::new_keyboard(idx, socket, virtio::base_features(cfg.protected_vm))
.context("failed to set up input device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
})
}
fn create_switches_device<T: IntoUnixStream>(
cfg: &Config,
switches_socket: T,
idx: u32,
) -> DeviceResult {
let socket = switches_socket.into_unix_stream().map_err(|e| {
error!("failed configuring virtio switches: {:#}", e);
e
})?;
let dev = virtio::new_switches(idx, socket, virtio::base_features(cfg.protected_vm))
.context("failed to set up input device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
})
}
fn create_vinput_device(cfg: &Config, dev_path: &Path) -> DeviceResult {
let dev_file = OpenOptions::new()
.read(true)
.write(true)
.open(dev_path)
.with_context(|| format!("failed to open vinput device {}", dev_path.display()))?;
let dev = virtio::new_evdev(dev_file, virtio::base_features(cfg.protected_vm))
.context("failed to set up input device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
})
}
fn create_balloon_device(
cfg: &Config,
tube: Tube,
inflate_tube: Option<Tube>,
init_balloon_size: u64,
) -> DeviceResult {
let dev = virtio::Balloon::new(
virtio::base_features(cfg.protected_vm),
tube,
inflate_tube,
init_balloon_size,
)
.context("failed to create balloon")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "balloon_device")?,
})
}
/// Generic method for creating a network device. `create_device` is a closure that takes the virtio
/// features and number of queue pairs as parameters, and is responsible for creating the device
/// itself.
fn create_net_device<F, T>(cfg: &Config, policy: &str, create_device: F) -> DeviceResult
where
F: Fn(u64, u16) -> Result<T>,
T: VirtioDevice + 'static,
{
let mut vq_pairs = cfg.net_vq_pairs.unwrap_or(1);
let vcpu_count = cfg.vcpu_count.unwrap_or(1);
if vcpu_count < vq_pairs as usize {
warn!("the number of net vq pairs must not exceed the vcpu count, falling back to single queue mode");
vq_pairs = 1;
}
let features = virtio::base_features(cfg.protected_vm);
let dev = create_device(features, vq_pairs)?;
Ok(VirtioDeviceStub {
dev: Box::new(dev) as Box<dyn VirtioDevice>,
jail: simple_jail(cfg, policy)?,
})
}
/// Returns a network device created from a new TAP interface configured with `host_ip`, `netmask`,
/// and `mac_address`.
fn create_net_device_from_config(
cfg: &Config,
host_ip: Ipv4Addr,
netmask: Ipv4Addr,
mac_address: MacAddress,
) -> DeviceResult {
let policy = if cfg.vhost_net {
"vhost_net_device"
} else {
"net_device"
};
if cfg.vhost_net {
create_net_device(cfg, policy, |features, _vq_pairs| {
virtio::vhost::Net::<Tap, vhost::Net<Tap>>::new(
&cfg.vhost_net_device_path,
features,
host_ip,
netmask,
mac_address,
)
.context("failed to set up vhost networking")
})
} else {
create_net_device(cfg, policy, |features, vq_pairs| {
virtio::Net::<Tap>::new(features, host_ip, netmask, mac_address, vq_pairs)
.context("failed to create virtio network device")
})
}
}
/// Returns a network device from a file descriptor to a configured TAP interface.
fn create_tap_net_device_from_fd(cfg: &Config, tap_fd: RawDescriptor) -> DeviceResult {
create_net_device(cfg, "net_device", |features, vq_pairs| {
// Safe because we ensure that we get a unique handle to the fd.
let tap = unsafe {
Tap::from_raw_descriptor(
validate_raw_descriptor(tap_fd).context("failed to validate tap descriptor")?,
)
.context("failed to create tap device")?
};
virtio::Net::from(features, tap, vq_pairs).context("failed to create tap net device")
})
}
/// Returns a network device created by opening the persistent, configured TAP interface `tap_name`.
fn create_tap_net_device_from_name(cfg: &Config, tap_name: &[u8]) -> DeviceResult {
create_net_device(cfg, "net_device", |features, vq_pairs| {
virtio::Net::<Tap>::new_from_name(features, tap_name, vq_pairs)
.context("failed to create configured virtio network device")
})
}
fn create_vhost_user_net_device(cfg: &Config, opt: &VhostUserOption) -> DeviceResult {
let dev = VhostUserNet::new(virtio::base_features(cfg.protected_vm), &opt.socket)
.context("failed to set up vhost-user net device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
fn create_vhost_user_vsock_device(cfg: &Config, opt: &VhostUserOption) -> DeviceResult {
let dev = VhostUserVsock::new(virtio::base_features(cfg.protected_vm), &opt.socket)
.context("failed to set up vhost-user vsock device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
fn create_vhost_user_wl_device(cfg: &Config, opt: &VhostUserWlOption) -> DeviceResult {
// The crosvm wl device expects us to connect the tube before it will accept a vhost-user
// connection.
let dev = VhostUserWl::new(virtio::base_features(cfg.protected_vm), &opt.socket)
.context("failed to set up vhost-user wl device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
#[cfg(feature = "gpu")]
fn create_vhost_user_gpu_device(
cfg: &Config,
opt: &VhostUserOption,
host_tube: Tube,
device_tube: Tube,
) -> DeviceResult {
// The crosvm gpu device expects us to connect the tube before it will accept a vhost-user
// connection.
let dev = VhostUserGpu::new(
virtio::base_features(cfg.protected_vm),
&opt.socket,
host_tube,
device_tube,
)
.context("failed to set up vhost-user gpu device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
/// Mirror-mount all the directories in `dirs` into `jail` on a best-effort basis.
///
/// This function will not return an error if any of the directories in `dirs` is missing.
#[cfg(any(feature = "gpu", feature = "video-decoder", feature = "video-encoder"))]
fn jail_mount_bind_if_exists<P: AsRef<std::ffi::OsStr>>(
jail: &mut Minijail,
dirs: &[P],
) -> Result<()> {
for dir in dirs {
let dir_path = Path::new(dir);
if dir_path.exists() {
jail.mount_bind(dir_path, dir_path, false)?;
}
}
Ok(())
}
#[cfg(feature = "gpu")]
fn gpu_jail(cfg: &Config, policy: &str) -> Result<Option<Minijail>> {
match simple_jail(cfg, policy)? {
Some(mut jail) => {
// Create a tmpfs in the device's root directory so that we can bind mount the
// dri directory into it. The size=67108864 is size=64*1024*1024 or size=64MB.
jail.mount_with_data(
Path::new("none"),
Path::new("/"),
"tmpfs",
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
"size=67108864",
)?;
// Device nodes required for DRM.
let sys_dev_char_path = Path::new("/sys/dev/char");
jail.mount_bind(sys_dev_char_path, sys_dev_char_path, false)?;
let sys_devices_path = Path::new("/sys/devices");
jail.mount_bind(sys_devices_path, sys_devices_path, false)?;
let drm_dri_path = Path::new("/dev/dri");
if drm_dri_path.exists() {
jail.mount_bind(drm_dri_path, drm_dri_path, false)?;
}
// If the ARM specific devices exist on the host, bind mount them in.
let mali0_path = Path::new("/dev/mali0");
if mali0_path.exists() {
jail.mount_bind(mali0_path, mali0_path, true)?;
}
let pvr_sync_path = Path::new("/dev/pvr_sync");
if pvr_sync_path.exists() {
jail.mount_bind(pvr_sync_path, pvr_sync_path, true)?;
}
// If the udmabuf driver exists on the host, bind mount it in.
let udmabuf_path = Path::new("/dev/udmabuf");
if udmabuf_path.exists() {
jail.mount_bind(udmabuf_path, udmabuf_path, true)?;
}
// Libraries that are required when mesa drivers are dynamically loaded.
jail_mount_bind_if_exists(
&mut jail,
&[
"/usr/lib",
"/usr/lib64",
"/lib",
"/lib64",
"/usr/share/drirc.d",
"/usr/share/glvnd",
"/usr/share/vulkan",
],
)?;
// pvr driver requires read access to /proc/self/task/*/comm.
let proc_path = Path::new("/proc");
jail.mount(
proc_path,
proc_path,
"proc",
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC | libc::MS_RDONLY) as usize,
)?;
// To enable perfetto tracing, we need to give access to the perfetto service IPC
// endpoints.
let perfetto_path = Path::new("/run/perfetto");
if perfetto_path.exists() {
jail.mount_bind(perfetto_path, perfetto_path, true)?;
}
Ok(Some(jail))
}
None => Ok(None),
}
}
#[cfg(feature = "gpu")]
struct GpuCacheInfo<'a> {
directory: Option<&'a str>,
environment: Vec<(&'a str, &'a str)>,
}
#[cfg(feature = "gpu")]
fn get_gpu_cache_info<'a>(
cache_dir: Option<&'a String>,
cache_size: Option<&'a String>,
sandbox: bool,
) -> GpuCacheInfo<'a> {
let mut dir = None;
let mut env = Vec::new();
if let Some(cache_dir) = cache_dir {
if !Path::new(cache_dir).exists() {
warn!("shader caching dir {} does not exist", cache_dir);
env.push(("MESA_GLSL_CACHE_DISABLE", "true"));
} else if cfg!(any(target_arch = "arm", target_arch = "aarch64")) && sandbox {
warn!("shader caching not yet supported on ARM with sandbox enabled");
env.push(("MESA_GLSL_CACHE_DISABLE", "true"));
} else {
dir = Some(cache_dir.as_str());
env.push(("MESA_GLSL_CACHE_DISABLE", "false"));
env.push(("MESA_GLSL_CACHE_DIR", cache_dir.as_str()));
if let Some(cache_size) = cache_size {
env.push(("MESA_GLSL_CACHE_MAX_SIZE", cache_size.as_str()));
}
}
}
GpuCacheInfo {
directory: dir,
environment: env,
}
}
#[cfg(feature = "gpu")]
fn create_gpu_device(
cfg: &Config,
exit_evt: &Event,
gpu_device_tube: Tube,
resource_bridges: Vec<Tube>,
wayland_socket_path: Option<&PathBuf>,
x_display: Option<String>,
render_server_fd: Option<SafeDescriptor>,
event_devices: Vec<EventDevice>,
map_request: Arc<Mutex<Option<ExternalMapping>>>,
) -> DeviceResult {
let mut display_backends = vec![
virtio::DisplayBackend::X(x_display),
virtio::DisplayBackend::Stub,
];
let wayland_socket_dirs = cfg
.wayland_socket_paths
.iter()
.map(|(_name, path)| path.parent())
.collect::<Option<Vec<_>>>()
.ok_or_else(|| anyhow!("wayland socket path has no parent or file name"))?;
if let Some(socket_path) = wayland_socket_path {
display_backends.insert(
0,
virtio::DisplayBackend::Wayland(Some(socket_path.to_owned())),
);
}
let dev = virtio::Gpu::new(
exit_evt.try_clone().context("failed to clone event")?,
Some(gpu_device_tube),
resource_bridges,
display_backends,
cfg.gpu_parameters.as_ref().unwrap(),
render_server_fd,
event_devices,
map_request,
cfg.sandbox,
virtio::base_features(cfg.protected_vm),
cfg.wayland_socket_paths.clone(),
);
let jail = match gpu_jail(cfg, "gpu_device")? {
Some(mut jail) => {
// Prepare GPU shader disk cache directory.
let (cache_dir, cache_size) = cfg
.gpu_parameters
.as_ref()
.map(|params| (params.cache_path.as_ref(), params.cache_size.as_ref()))
.unwrap();
let cache_info = get_gpu_cache_info(cache_dir, cache_size, cfg.sandbox);
if let Some(dir) = cache_info.directory {
jail.mount_bind(dir, dir, true)?;
}
for (key, val) in cache_info.environment {
env::set_var(key, val);
}
// Bind mount the wayland socket's directory into jail's root. This is necessary since
// each new wayland context must open() the socket. If the wayland socket is ever
// destroyed and remade in the same host directory, new connections will be possible
// without restarting the wayland device.
for dir in &wayland_socket_dirs {
jail.mount_bind(dir, dir, true)?;
}
add_current_user_to_jail(&mut jail)?;
Some(jail)
}
None => None,
};
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail,
})
}
#[cfg(feature = "gpu")]
fn get_gpu_render_server_environment(cache_info: &GpuCacheInfo) -> Result<Vec<String>> {
let mut env = Vec::new();
let mut cache_env_keys = HashSet::with_capacity(cache_info.environment.len());
for (key, val) in cache_info.environment.iter() {
env.push(format!("{}={}", key, val));
cache_env_keys.insert(*key);
}
for (key_os, val_os) in env::vars_os() {
// minijail should accept OsStr rather than str...
let into_string_err = |_| anyhow!("invalid environment key/val");
let key = key_os.into_string().map_err(into_string_err)?;
let val = val_os.into_string().map_err(into_string_err)?;
if !cache_env_keys.contains(key.as_str()) {
env.push(format!("{}={}", key, val));
}
}
Ok(env)
}
#[cfg(feature = "gpu")]
struct ScopedMinijail(Minijail);
#[cfg(feature = "gpu")]
impl Drop for ScopedMinijail {
fn drop(&mut self) {
let _ = self.0.kill();
}
}
#[cfg(feature = "gpu")]
fn start_gpu_render_server(
cfg: &Config,
render_server_parameters: &GpuRenderServerParameters,
) -> Result<(Minijail, SafeDescriptor)> {
let (server_socket, client_socket) =
UnixSeqpacket::pair().context("failed to create render server socket")?;
let mut env = None;
let jail = match gpu_jail(cfg, "gpu_render_server")? {
Some(mut jail) => {
let cache_info = get_gpu_cache_info(
render_server_parameters.cache_path.as_ref(),
render_server_parameters.cache_size.as_ref(),
cfg.sandbox,
);
if let Some(dir) = cache_info.directory {
jail.mount_bind(dir, dir, true)?;
}
if !cache_info.environment.is_empty() {
env = Some(get_gpu_render_server_environment(&cache_info)?);
}
// bind mount /dev/log for syslog
let log_path = Path::new("/dev/log");
if log_path.exists() {
jail.mount_bind(log_path, log_path, true)?;
}
// Run as root in the jail to keep capabilities after execve, which is needed for
// mounting to work. All capabilities will be dropped afterwards.
add_current_user_as_root_to_jail(&mut jail)?;
jail
}
None => Minijail::new().context("failed to create jail")?,
};
let inheritable_fds = [
server_socket.as_raw_descriptor(),
libc::STDOUT_FILENO,
libc::STDERR_FILENO,
];
let cmd = &render_server_parameters.path;
let cmd_str = cmd
.to_str()
.ok_or_else(|| anyhow!("invalid render server path"))?;
let fd_str = server_socket.as_raw_descriptor().to_string();
let args = [cmd_str, "--socket-fd", &fd_str];
let mut envp: Option<Vec<&str>> = None;
if let Some(ref env) = env {
envp = Some(env.iter().map(AsRef::as_ref).collect());
}
jail.run_command(minijail::Command::new_for_path(
cmd,
&inheritable_fds,
&args,
envp.as_deref(),
)?)
.context("failed to start gpu render server")?;
Ok((jail, SafeDescriptor::from(client_socket)))
}
fn create_wayland_device(
cfg: &Config,
control_tube: Tube,
resource_bridge: Option<Tube>,
) -> DeviceResult {
let wayland_socket_dirs = cfg
.wayland_socket_paths
.iter()
.map(|(_name, path)| path.parent())
.collect::<Option<Vec<_>>>()
.ok_or_else(|| anyhow!("wayland socket path has no parent or file name"))?;
let features = virtio::base_features(cfg.protected_vm);
let dev = virtio::Wl::new(
features,
cfg.wayland_socket_paths.clone(),
control_tube,
resource_bridge,
)
.context("failed to create wayland device")?;
let jail = match simple_jail(cfg, "wl_device")? {
Some(mut jail) => {
// Create a tmpfs in the device's root directory so that we can bind mount the wayland
// socket directory into it. The size=67108864 is size=64*1024*1024 or size=64MB.
jail.mount_with_data(
Path::new("none"),
Path::new("/"),
"tmpfs",
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
"size=67108864",
)?;
// Bind mount the wayland socket's directory into jail's root. This is necessary since
// each new wayland context must open() the socket. If the wayland socket is ever
// destroyed and remade in the same host directory, new connections will be possible
// without restarting the wayland device.
for dir in &wayland_socket_dirs {
jail.mount_bind(dir, dir, true)?;
}
add_current_user_to_jail(&mut jail)?;
Some(jail)
}
None => None,
};
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail,
})
}
#[cfg(any(feature = "video-decoder", feature = "video-encoder"))]
fn create_video_device(
backend: VideoBackendType,
cfg: &Config,
typ: devices::virtio::VideoDeviceType,
resource_bridge: Tube,
) -> DeviceResult {
let jail = match simple_jail(cfg, "video_device")? {
Some(mut jail) => {
match typ {
#[cfg(feature = "video-decoder")]
devices::virtio::VideoDeviceType::Decoder => add_current_user_to_jail(&mut jail)?,
#[cfg(feature = "video-encoder")]
devices::virtio::VideoDeviceType::Encoder => add_current_user_to_jail(&mut jail)?,
};
// Create a tmpfs in the device's root directory so that we can bind mount files.
jail.mount_with_data(
Path::new("none"),
Path::new("/"),
"tmpfs",
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
"size=67108864",
)?;
#[cfg(feature = "libvda")]
// Render node for libvda.
if backend == VideoBackendType::Libvda || backend == VideoBackendType::LibvdaVd {
// follow the implementation at:
// https://source.corp.google.com/chromeos_public/src/platform/minigbm/cros_gralloc/cros_gralloc_driver.cc;l=90;bpv=0;cl=c06cc9cccb3cf3c7f9d2aec706c27c34cd6162a0
const DRM_NUM_NODES: u32 = 63;
const DRM_RENDER_NODE_START: u32 = 128;
for offset in 0..DRM_NUM_NODES {
let path_str = format!("/dev/dri/renderD{}", DRM_RENDER_NODE_START + offset);
let dev_dri_path = Path::new(&path_str);
if !dev_dri_path.exists() {
break;
}
jail.mount_bind(dev_dri_path, dev_dri_path, false)?;
}
}
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{
// Device nodes used by libdrm through minigbm in libvda on AMD devices.
let sys_dev_char_path = Path::new("/sys/dev/char");
jail.mount_bind(sys_dev_char_path, sys_dev_char_path, false)?;
let sys_devices_path = Path::new("/sys/devices");
jail.mount_bind(sys_devices_path, sys_devices_path, false)?;
// Required for loading dri libraries loaded by minigbm on AMD devices.
jail_mount_bind_if_exists(&mut jail, &["/usr/lib64"])?;
}
// Device nodes required by libchrome which establishes Mojo connection in libvda.
let dev_urandom_path = Path::new("/dev/urandom");
jail.mount_bind(dev_urandom_path, dev_urandom_path, false)?;
let system_bus_socket_path = Path::new("/run/dbus/system_bus_socket");
jail.mount_bind(system_bus_socket_path, system_bus_socket_path, true)?;
Some(jail)
}
None => None,
};
Ok(VirtioDeviceStub {
dev: Box::new(devices::virtio::VideoDevice::new(
virtio::base_features(cfg.protected_vm),
typ,
backend,
Some(resource_bridge),
)),
jail,
})
}
#[cfg(any(feature = "video-decoder", feature = "video-encoder"))]
fn register_video_device(
backend: VideoBackendType,
devs: &mut Vec<VirtioDeviceStub>,
video_tube: Tube,
cfg: &Config,
typ: devices::virtio::VideoDeviceType,
) -> Result<()> {
devs.push(create_video_device(backend, cfg, typ, video_tube)?);
Ok(())
}
fn create_vhost_vsock_device(cfg: &Config, cid: u64) -> DeviceResult {
let features = virtio::base_features(cfg.protected_vm);
let device_file = match cfg
.vhost_vsock_device
.as_ref()
.unwrap_or(&VhostVsockDeviceParameter::default())
{
VhostVsockDeviceParameter::Fd(fd) => {
let fd = validate_raw_descriptor(*fd)
.context("failed to validate fd for virtual socker device")?;
// Safe because the `fd` is actually owned by this process and
// we have a unique handle to it.
unsafe { File::from_raw_fd(fd) }
}
VhostVsockDeviceParameter::Path(path) => OpenOptions::new()
.read(true)
.write(true)
.custom_flags(libc::O_CLOEXEC | libc::O_NONBLOCK)
.open(path)
.context("failed to open virtual socket device")?,
};
let dev = virtio::vhost::Vsock::new(device_file, features, cid)
.context("failed to set up virtual socket device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "vhost_vsock_device")?,
})
}
fn create_fs_device(
cfg: &Config,
uid_map: &str,
gid_map: &str,
src: &Path,
tag: &str,
fs_cfg: virtio::fs::passthrough::Config,
device_tube: Tube,
) -> DeviceResult {
let max_open_files =
base::get_max_open_files().context("failed to get max number of open files")?;
let j = if cfg.sandbox {
let seccomp_policy = cfg.seccomp_policy_dir.join("fs_device");
let config = SandboxConfig {
limit_caps: false,
uid_map: Some(uid_map),
gid_map: Some(gid_map),
log_failures: cfg.seccomp_log_failures,
seccomp_policy: &seccomp_policy,
// We want bind mounts from the parent namespaces to propagate into the fs device's
// namespace.
remount_mode: Some(libc::MS_SLAVE),
};
create_base_minijail(src, Some(max_open_files), Some(&config))?
} else {
create_base_minijail(src, Some(max_open_files), None)?
};
let features = virtio::base_features(cfg.protected_vm);
// TODO(chirantan): Use more than one worker once the kernel driver has been fixed to not panic
// when num_queues > 1.
let dev = virtio::fs::Fs::new(features, tag, 1, fs_cfg, device_tube)
.context("failed to create fs device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: Some(j),
})
}
fn create_9p_device(
cfg: &Config,
uid_map: &str,
gid_map: &str,
src: &Path,
tag: &str,
mut p9_cfg: p9::Config,
) -> DeviceResult {
let max_open_files =
base::get_max_open_files().context("failed to get max number of open files")?;
let (jail, root) = if cfg.sandbox {
let seccomp_policy = cfg.seccomp_policy_dir.join("9p_device");
let config = SandboxConfig {
limit_caps: false,
uid_map: Some(uid_map),
gid_map: Some(gid_map),
log_failures: cfg.seccomp_log_failures,
seccomp_policy: &seccomp_policy,
// We want bind mounts from the parent namespaces to propagate into the 9p server's
// namespace.
remount_mode: Some(libc::MS_SLAVE),
};
let jail = create_base_minijail(src, Some(max_open_files), Some(&config))?;
// The shared directory becomes the root of the device's file system.
let root = Path::new("/");
(Some(jail), root)
} else {
// There's no mount namespace so we tell the server to treat the source directory as the
// root.
(None, src)
};
let features = virtio::base_features(cfg.protected_vm);
p9_cfg.root = root.into();
let dev = virtio::P9::new(features, tag, p9_cfg).context("failed to create 9p device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail,
})
}
fn create_pmem_device(
cfg: &Config,
vm: &mut impl Vm,
resources: &mut SystemAllocator,
disk: &DiskOption,
index: usize,
pmem_device_tube: Tube,
) -> DeviceResult {
let fd = open_file(&disk.path, disk.read_only, false /*O_DIRECT*/)
.with_context(|| format!("failed to load disk image {}", disk.path.display()))?;
let (disk_size, arena_size) = {
let metadata = std::fs::metadata(&disk.path).with_context(|| {
format!("failed to get disk image {} metadata", disk.path.display())
})?;
let disk_len = metadata.len();
// Linux requires pmem region sizes to be 2 MiB aligned. Linux will fill any partial page
// at the end of an mmap'd file and won't write back beyond the actual file length, but if
// we just align the size of the file to 2 MiB then access beyond the last page of the
// mapped file will generate SIGBUS. So use a memory mapping arena that will provide
// padding up to 2 MiB.
let alignment = 2 * 1024 * 1024;
let align_adjust = if disk_len % alignment != 0 {
alignment - (disk_len % alignment)
} else {
0
};
(
disk_len,
disk_len
.checked_add(align_adjust)
.ok_or_else(|| anyhow!("pmem device image too big"))?,
)
};
let protection = {
if disk.read_only {
Protection::read()
} else {
Protection::read_write()
}
};
let arena = {
// Conversion from u64 to usize may fail on 32bit system.
let arena_size = usize::try_from(arena_size).context("pmem device image too big")?;
let disk_size = usize::try_from(disk_size).context("pmem device image too big")?;
let mut arena =
MemoryMappingArena::new(arena_size).context("failed to reserve pmem memory")?;
arena
.add_fd_offset_protection(0, disk_size, &fd, 0, protection)
.context("failed to reserve pmem memory")?;
// If the disk is not a multiple of the page size, the OS will fill the remaining part
// of the page with zeroes. However, the anonymous mapping added below must start on a
// page boundary, so round up the size before calculating the offset of the anon region.
let disk_size = round_up_to_page_size(disk_size);
if arena_size > disk_size {
// Add an anonymous region with the same protection as the disk mapping if the arena
// size was aligned.
arena
.add_anon_protection(disk_size, arena_size - disk_size, protection)
.context("failed to reserve pmem padding")?;
}
arena
};
let mapping_address = resources
.mmio_allocator(MmioType::High)
.reverse_allocate_with_align(
arena_size,
Alloc::PmemDevice(index),
format!("pmem_disk_image_{}", index),
// Linux kernel requires pmem namespaces to be 128 MiB aligned.
128 * 1024 * 1024, /* 128 MiB */
)
.context("failed to allocate memory for pmem device")?;
let slot = vm
.add_memory_region(
GuestAddress(mapping_address),
Box::new(arena),
/* read_only = */ disk.read_only,
/* log_dirty_pages = */ false,
)
.context("failed to add pmem device memory")?;
let dev = virtio::Pmem::new(
virtio::base_features(cfg.protected_vm),
fd,
GuestAddress(mapping_address),
slot,
arena_size,
Some(pmem_device_tube),
)
.context("failed to create pmem device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev) as Box<dyn VirtioDevice>,
jail: simple_jail(cfg, "pmem_device")?,
})
}
fn create_iommu_device(
cfg: &Config,
phys_max_addr: u64,
endpoints: BTreeMap<u32, Arc<Mutex<VfioContainer>>>,
) -> DeviceResult {
let dev = virtio::Iommu::new(
virtio::base_features(cfg.protected_vm),
endpoints,
phys_max_addr,
)
.context("failed to create IOMMU device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "iommu_device")?,
})
}
fn create_console_device(cfg: &Config, param: &SerialParameters) -> DeviceResult {
let mut keep_rds = Vec::new();
let evt = Event::new().context("failed to create event")?;
let dev = param
.create_serial_device::<Console>(cfg.protected_vm, &evt, &mut keep_rds)
.context("failed to create console device")?;
let jail = match simple_jail(cfg, "serial")? {
Some(mut jail) => {
// Create a tmpfs in the device's root directory so that we can bind mount the
// log socket directory into it.
// The size=67108864 is size=64*1024*1024 or size=64MB.
jail.mount_with_data(
Path::new("none"),
Path::new("/"),
"tmpfs",
(libc::MS_NODEV | libc::MS_NOEXEC | libc::MS_NOSUID) as usize,
"size=67108864",
)?;
add_current_user_to_jail(&mut jail)?;
let res = param.add_bind_mounts(&mut jail);
if res.is_err() {
error!("failed to add bind mounts for console device");
}
Some(jail)
}
None => None,
};
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail, // TODO(dverkamp): use a separate policy for console?
})
}
#[cfg(feature = "audio")]
fn create_sound_device(path: &Path, cfg: &Config) -> DeviceResult {
let dev = virtio::new_sound(path, virtio::base_features(cfg.protected_vm))
.context("failed to create sound device")?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "vios_audio_device")?,
})
}
// gpu_device_tube is not used when GPU support is disabled.
#[cfg_attr(not(feature = "gpu"), allow(unused_variables))]
fn create_virtio_devices(
cfg: &Config,
vm: &mut impl Vm,
resources: &mut SystemAllocator,
_exit_evt: &Event,
wayland_device_tube: Tube,
gpu_device_tube: Tube,
vhost_user_gpu_tubes: Vec<(Tube, Tube)>,
balloon_device_tube: Option<Tube>,
balloon_inflate_tube: Option<Tube>,
init_balloon_size: u64,
disk_device_tubes: &mut Vec<Tube>,
pmem_device_tubes: &mut Vec<Tube>,
map_request: Arc<Mutex<Option<ExternalMapping>>>,
fs_device_tubes: &mut Vec<Tube>,
#[cfg(feature = "gpu")] render_server_fd: Option<SafeDescriptor>,
) -> DeviceResult<Vec<VirtioDeviceStub>> {
let mut devs = Vec::new();
for (_, param) in cfg
.serial_parameters
.iter()
.filter(|(_k, v)| v.hardware == SerialHardware::VirtioConsole)
{
let dev = create_console_device(cfg, param)?;
devs.push(dev);
}
for disk in &cfg.disks {
let disk_device_tube = disk_device_tubes.remove(0);
devs.push(create_block_device(cfg, disk, disk_device_tube)?);
}
for blk in &cfg.vhost_user_blk {
devs.push(create_vhost_user_block_device(cfg, blk)?);
}
for console in &cfg.vhost_user_console {
devs.push(create_vhost_user_console_device(cfg, console)?);
}
for (index, pmem_disk) in cfg.pmem_devices.iter().enumerate() {
let pmem_device_tube = pmem_device_tubes.remove(0);
devs.push(create_pmem_device(
cfg,
vm,
resources,
pmem_disk,
index,
pmem_device_tube,
)?);
}
devs.push(create_rng_device(cfg)?);
#[cfg(feature = "tpm")]
{
if cfg.software_tpm {
devs.push(create_tpm_device(cfg)?);
}
}
for (idx, single_touch_spec) in cfg.virtio_single_touch.iter().enumerate() {
devs.push(create_single_touch_device(
cfg,
single_touch_spec,
idx as u32,
)?);
}
for (idx, multi_touch_spec) in cfg.virtio_multi_touch.iter().enumerate() {
devs.push(create_multi_touch_device(
cfg,
multi_touch_spec,
idx as u32,
)?);
}
for (idx, trackpad_spec) in cfg.virtio_trackpad.iter().enumerate() {
devs.push(create_trackpad_device(cfg, trackpad_spec, idx as u32)?);
}
for (idx, mouse_socket) in cfg.virtio_mice.iter().enumerate() {
devs.push(create_mouse_device(cfg, mouse_socket, idx as u32)?);
}
for (idx, keyboard_socket) in cfg.virtio_keyboard.iter().enumerate() {
devs.push(create_keyboard_device(cfg, keyboard_socket, idx as u32)?);
}
for (idx, switches_socket) in cfg.virtio_switches.iter().enumerate() {
devs.push(create_switches_device(cfg, switches_socket, idx as u32)?);
}
for dev_path in &cfg.virtio_input_evdevs {
devs.push(create_vinput_device(cfg, dev_path)?);
}
if let Some(balloon_device_tube) = balloon_device_tube {
devs.push(create_balloon_device(
cfg,
balloon_device_tube,
balloon_inflate_tube,
init_balloon_size,
)?);
}
// We checked above that if the IP is defined, then the netmask is, too.
for tap_fd in &cfg.tap_fd {
devs.push(create_tap_net_device_from_fd(cfg, *tap_fd)?);
}
if let (Some(host_ip), Some(netmask), Some(mac_address)) =
(cfg.host_ip, cfg.netmask, cfg.mac_address)
{
if !cfg.vhost_user_net.is_empty() {
bail!("vhost-user-net cannot be used with any of --host_ip, --netmask or --mac");
}
devs.push(create_net_device_from_config(
cfg,
host_ip,
netmask,
mac_address,
)?);
}
for tap_name in &cfg.tap_name {
devs.push(create_tap_net_device_from_name(cfg, tap_name.as_bytes())?);
}
for net in &cfg.vhost_user_net {
devs.push(create_vhost_user_net_device(cfg, net)?);
}
for vsock in &cfg.vhost_user_vsock {
devs.push(create_vhost_user_vsock_device(cfg, vsock)?);
}
for opt in &cfg.vhost_user_wl {
devs.push(create_vhost_user_wl_device(cfg, opt)?);
}
#[cfg(feature = "gpu")]
for (opt, (host_tube, device_tube)) in cfg.vhost_user_gpu.iter().zip(vhost_user_gpu_tubes) {
devs.push(create_vhost_user_gpu_device(
cfg,
opt,
host_tube,
device_tube,
)?);
}
for opt in &cfg.vvu_proxy {
devs.push(create_vvu_proxy_device(cfg, opt)?);
}
#[cfg_attr(not(feature = "gpu"), allow(unused_mut))]
let mut resource_bridges = Vec::<Tube>::new();
if !cfg.wayland_socket_paths.is_empty() {
#[cfg_attr(not(feature = "gpu"), allow(unused_mut))]
let mut wl_resource_bridge = None::<Tube>;
#[cfg(feature = "gpu")]
{
if cfg.gpu_parameters.is_some() {
let (wl_socket, gpu_socket) = Tube::pair().context("failed to create tube")?;
resource_bridges.push(gpu_socket);
wl_resource_bridge = Some(wl_socket);
}
}
devs.push(create_wayland_device(
cfg,
wayland_device_tube,
wl_resource_bridge,
)?);
}
#[cfg(feature = "video-decoder")]
let video_dec_cfg = if let Some(backend) = cfg.video_dec {
let (video_tube, gpu_tube) = Tube::pair().context("failed to create tube")?;
resource_bridges.push(gpu_tube);
Some((video_tube, backend))
} else {
None
};
#[cfg(feature = "video-encoder")]
let video_enc_cfg = if let Some(backend) = cfg.video_enc {
let (video_tube, gpu_tube) = Tube::pair().context("failed to create tube")?;
resource_bridges.push(gpu_tube);
Some((video_tube, backend))
} else {
None
};
#[cfg(feature = "gpu")]
{
if let Some(gpu_parameters) = &cfg.gpu_parameters {
let mut gpu_display_w = DEFAULT_DISPLAY_WIDTH;
let mut gpu_display_h = DEFAULT_DISPLAY_HEIGHT;
if !gpu_parameters.displays.is_empty() {
gpu_display_w = gpu_parameters.displays[0].width;
gpu_display_h = gpu_parameters.displays[0].height;
}
let mut event_devices = Vec::new();
if cfg.display_window_mouse {
let (event_device_socket, virtio_dev_socket) =
UnixStream::pair().context("failed to create socket")?;
let (multi_touch_width, multi_touch_height) = cfg
.virtio_multi_touch
.first()
.as_ref()
.map(|multi_touch_spec| multi_touch_spec.get_size())
.unwrap_or((gpu_display_w, gpu_display_h));
let dev = virtio::new_multi_touch(
// u32::MAX is the least likely to collide with the indices generated above for
// the multi_touch options, which begin at 0.
u32::MAX,
virtio_dev_socket,
multi_touch_width,
multi_touch_height,
virtio::base_features(cfg.protected_vm),
)
.context("failed to set up mouse device")?;
devs.push(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
});
event_devices.push(EventDevice::touchscreen(event_device_socket));
}
if cfg.display_window_keyboard {
let (event_device_socket, virtio_dev_socket) =
UnixStream::pair().context("failed to create socket")?;
let dev = virtio::new_keyboard(
// u32::MAX is the least likely to collide with the indices generated above for
// the multi_touch options, which begin at 0.
u32::MAX,
virtio_dev_socket,
virtio::base_features(cfg.protected_vm),
)
.context("failed to set up keyboard device")?;
devs.push(VirtioDeviceStub {
dev: Box::new(dev),
jail: simple_jail(cfg, "input_device")?,
});
event_devices.push(EventDevice::keyboard(event_device_socket));
}
devs.push(create_gpu_device(
cfg,
_exit_evt,
gpu_device_tube,
resource_bridges,
// Use the unnamed socket for GPU display screens.
cfg.wayland_socket_paths.get(""),
cfg.x_display.clone(),
render_server_fd,
event_devices,
map_request,
)?);
}
}
#[cfg(feature = "audio_cras")]
{
for cras_snd in &cfg.cras_snds {
devs.push(create_cras_snd_device(cfg, cras_snd.clone())?);
}
}
#[cfg(feature = "video-decoder")]
{
if let Some((video_dec_tube, video_dec_backend)) = video_dec_cfg {
register_video_device(
video_dec_backend,
&mut devs,
video_dec_tube,
cfg,
devices::virtio::VideoDeviceType::Decoder,
)?;
}
}
#[cfg(feature = "video-encoder")]
{
if let Some((video_enc_tube, video_enc_backend)) = video_enc_cfg {
register_video_device(
video_enc_backend,
&mut devs,
video_enc_tube,
cfg,
devices::virtio::VideoDeviceType::Encoder,
)?;
}
}
if let Some(cid) = cfg.cid {
devs.push(create_vhost_vsock_device(cfg, cid)?);
}
for vhost_user_fs in &cfg.vhost_user_fs {
devs.push(create_vhost_user_fs_device(cfg, vhost_user_fs)?);
}
#[cfg(feature = "audio")]
for vhost_user_snd in &cfg.vhost_user_snd {
devs.push(create_vhost_user_snd_device(cfg, vhost_user_snd)?);
}
for shared_dir in &cfg.shared_dirs {
let SharedDir {
src,
tag,
kind,
uid_map,
gid_map,
fs_cfg,
p9_cfg,
} = shared_dir;
let dev = match kind {
SharedDirKind::FS => {
let device_tube = fs_device_tubes.remove(0);
create_fs_device(cfg, uid_map, gid_map, src, tag, fs_cfg.clone(), device_tube)?
}
SharedDirKind::P9 => create_9p_device(cfg, uid_map, gid_map, src, tag, p9_cfg.clone())?,
};
devs.push(dev);
}
if let Some(vhost_user_mac80211_hwsim) = &cfg.vhost_user_mac80211_hwsim {
devs.push(create_vhost_user_mac80211_hwsim_device(
cfg,
vhost_user_mac80211_hwsim,
)?);
}
#[cfg(feature = "audio")]
if let Some(path) = &cfg.sound {
devs.push(create_sound_device(path, cfg)?);
}
Ok(devs)
}
fn create_vfio_device(
cfg: &Config,
vm: &impl Vm,
resources: &mut SystemAllocator,
control_tubes: &mut Vec<TaggedControlTube>,
vfio_path: &Path,
bus_num: Option<u8>,
iommu_endpoints: &mut BTreeMap<u32, Arc<Mutex<VfioContainer>>>,
coiommu_endpoints: Option<&mut Vec<u16>>,
iommu_dev: IommuDevType,
) -> DeviceResult<(Box<VfioPciDevice>, Option<Minijail>)> {
let vfio_container = VfioCommonSetup::vfio_get_container(iommu_dev, Some(vfio_path))
.context("failed to get vfio container")?;
// create MSI, MSI-X, and Mem request sockets for each vfio device
let (vfio_host_tube_msi, vfio_device_tube_msi) =
Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmIrq(vfio_host_tube_msi));
let (vfio_host_tube_msix, vfio_device_tube_msix) =
Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmIrq(vfio_host_tube_msix));
let (vfio_host_tube_mem, vfio_device_tube_mem) =
Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmMemory(vfio_host_tube_mem));
let hotplug = bus_num.is_some();
let vfio_device_tube_vm = if hotplug {
let (vfio_host_tube_vm, device_tube_vm) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::Vm(vfio_host_tube_vm));
Some(device_tube_vm)
} else {
None
};
let vfio_device = VfioDevice::new_passthrough(
&vfio_path,
vm,
vfio_container.clone(),
iommu_dev != IommuDevType::NoIommu,
)
.context("failed to create vfio device")?;
let mut vfio_pci_device = Box::new(VfioPciDevice::new(
vfio_device,
bus_num,
vfio_device_tube_msi,
vfio_device_tube_msix,
vfio_device_tube_mem,
vfio_device_tube_vm,
));
// early reservation for pass-through PCI devices.
let endpoint_addr = vfio_pci_device
.allocate_address(resources)
.context("failed to allocate resources early for vfio pci dev")?;
match iommu_dev {
IommuDevType::NoIommu => {}
IommuDevType::VirtioIommu => {
iommu_endpoints.insert(endpoint_addr.to_u32(), vfio_container);
}
IommuDevType::CoIommu => {
if let Some(endpoints) = coiommu_endpoints {
endpoints.push(endpoint_addr.to_u32() as u16);
} else {
bail!("Missed coiommu_endpoints vector to store the endpoint addr");
}
}
}
if hotplug {
Ok((vfio_pci_device, None))
} else {
Ok((vfio_pci_device, simple_jail(cfg, "vfio_device")?))
}
}
fn create_vfio_platform_device(
cfg: &Config,
vm: &impl Vm,
_resources: &mut SystemAllocator,
control_tubes: &mut Vec<TaggedControlTube>,
vfio_path: &Path,
_endpoints: &mut BTreeMap<u32, Arc<Mutex<VfioContainer>>>,
iommu_dev: IommuDevType,
) -> DeviceResult<(VfioPlatformDevice, Option<Minijail>)> {
let vfio_container = VfioCommonSetup::vfio_get_container(iommu_dev, Some(vfio_path))
.context("Failed to create vfio device")?;
let (vfio_host_tube_mem, vfio_device_tube_mem) =
Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmMemory(vfio_host_tube_mem));
let vfio_device = VfioDevice::new_passthrough(
&vfio_path,
vm,
vfio_container,
iommu_dev != IommuDevType::NoIommu,
)
.context("Failed to create vfio device")?;
let vfio_plat_dev = VfioPlatformDevice::new(vfio_device, vfio_device_tube_mem);
Ok((vfio_plat_dev, simple_jail(cfg, "vfio_platform_device")?))
}
fn create_devices(
cfg: &Config,
vm: &mut impl Vm,
resources: &mut SystemAllocator,
exit_evt: &Event,
phys_max_addr: u64,
control_tubes: &mut Vec<TaggedControlTube>,
wayland_device_tube: Tube,
gpu_device_tube: Tube,
vhost_user_gpu_tubes: Vec<(Tube, Tube)>,
balloon_device_tube: Option<Tube>,
init_balloon_size: u64,
disk_device_tubes: &mut Vec<Tube>,
pmem_device_tubes: &mut Vec<Tube>,
fs_device_tubes: &mut Vec<Tube>,
#[cfg(feature = "usb")] usb_provider: HostBackendDeviceProvider,
map_request: Arc<Mutex<Option<ExternalMapping>>>,
#[cfg(feature = "gpu")] render_server_fd: Option<SafeDescriptor>,
) -> DeviceResult<Vec<(Box<dyn BusDeviceObj>, Option<Minijail>)>> {
let mut devices: Vec<(Box<dyn BusDeviceObj>, Option<Minijail>)> = Vec::new();
let mut balloon_inflate_tube: Option<Tube> = None;
if !cfg.vfio.is_empty() {
let mut iommu_attached_endpoints: BTreeMap<u32, Arc<Mutex<VfioContainer>>> =
BTreeMap::new();
let mut coiommu_attached_endpoints = Vec::new();
for vfio_dev in cfg
.vfio
.iter()
.filter(|dev| dev.get_type() == VfioType::Pci)
{
let vfio_path = &vfio_dev.vfio_path;
let (vfio_pci_device, jail) = create_vfio_device(
cfg,
vm,
resources,
control_tubes,
vfio_path.as_path(),
None,
&mut iommu_attached_endpoints,
Some(&mut coiommu_attached_endpoints),
vfio_dev.iommu_dev_type(),
)?;
devices.push((vfio_pci_device, jail));
}
for vfio_dev in cfg
.vfio
.iter()
.filter(|dev| dev.get_type() == VfioType::Platform)
{
let vfio_path = &vfio_dev.vfio_path;
let (vfio_plat_dev, jail) = create_vfio_platform_device(
cfg,
vm,
resources,
control_tubes,
vfio_path.as_path(),
&mut iommu_attached_endpoints,
IommuDevType::NoIommu, // Virtio IOMMU is not supported yet
)?;
devices.push((Box::new(vfio_plat_dev), jail));
}
if !coiommu_attached_endpoints.is_empty() || !iommu_attached_endpoints.is_empty() {
let mut buf = mem::MaybeUninit::<libc::rlimit>::zeroed();
let res = unsafe { libc::getrlimit(libc::RLIMIT_MEMLOCK, buf.as_mut_ptr()) };
if res == 0 {
let limit = unsafe { buf.assume_init() };
let rlim_new = limit
.rlim_cur
.saturating_add(vm.get_memory().memory_size() as libc::rlim_t);
let rlim_max = max(limit.rlim_max, rlim_new);
if limit.rlim_cur < rlim_new {
let limit_arg = libc::rlimit {
rlim_cur: rlim_new as libc::rlim_t,
rlim_max: rlim_max as libc::rlim_t,
};
let res = unsafe { libc::setrlimit(libc::RLIMIT_MEMLOCK, &limit_arg) };
if res != 0 {
bail!("Set rlimit failed");
}
}
} else {
bail!("Get rlimit failed");
}
}
if !iommu_attached_endpoints.is_empty() {
let iommu_dev = create_iommu_device(cfg, phys_max_addr, iommu_attached_endpoints)?;
let (msi_host_tube, msi_device_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmIrq(msi_host_tube));
let mut dev =
VirtioPciDevice::new(vm.get_memory().clone(), iommu_dev.dev, msi_device_tube)
.context("failed to create virtio pci dev")?;
// early reservation for viommu.
dev.allocate_address(resources)
.context("failed to allocate resources early for virtio pci dev")?;
let dev = Box::new(dev);
devices.push((dev, iommu_dev.jail));
}
if !coiommu_attached_endpoints.is_empty() {
let vfio_container =
VfioCommonSetup::vfio_get_container(IommuDevType::CoIommu, None as Option<&Path>)
.context("failed to get vfio container")?;
let (coiommu_host_tube, coiommu_device_tube) =
Tube::pair().context("failed to create coiommu tube")?;
control_tubes.push(TaggedControlTube::VmMemory(coiommu_host_tube));
let vcpu_count = cfg.vcpu_count.unwrap_or(1) as u64;
let (coiommu_tube, balloon_tube) =
Tube::pair().context("failed to create coiommu tube")?;
balloon_inflate_tube = Some(balloon_tube);
let dev = CoIommuDev::new(
vm.get_memory().clone(),
vfio_container,
coiommu_device_tube,
coiommu_tube,
coiommu_attached_endpoints,
vcpu_count,
cfg.coiommu_param.unwrap_or_default(),
)
.context("failed to create coiommu device")?;
devices.push((Box::new(dev), simple_jail(cfg, "coiommu")?));
}
}
let stubs = create_virtio_devices(
cfg,
vm,
resources,
exit_evt,
wayland_device_tube,
gpu_device_tube,
vhost_user_gpu_tubes,
balloon_device_tube,
balloon_inflate_tube,
init_balloon_size,
disk_device_tubes,
pmem_device_tubes,
map_request,
fs_device_tubes,
#[cfg(feature = "gpu")]
render_server_fd,
)?;
for stub in stubs {
let (msi_host_tube, msi_device_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmIrq(msi_host_tube));
let dev = VirtioPciDevice::new(vm.get_memory().clone(), stub.dev, msi_device_tube)
.context("failed to create virtio pci dev")?;
let dev = Box::new(dev) as Box<dyn BusDeviceObj>;
devices.push((dev, stub.jail));
}
#[cfg(feature = "audio")]
for ac97_param in &cfg.ac97_parameters {
let dev = Ac97Dev::try_new(vm.get_memory().clone(), ac97_param.clone())
.context("failed to create ac97 device")?;
let jail = simple_jail(cfg, dev.minijail_policy())?;
devices.push((Box::new(dev), jail));
}
#[cfg(feature = "usb")]
{
// Create xhci controller.
let usb_controller = Box::new(XhciController::new(vm.get_memory().clone(), usb_provider));
devices.push((usb_controller, simple_jail(cfg, "xhci")?));
}
for params in &cfg.stub_pci_devices {
// Stub devices don't need jailing since they don't do anything.
devices.push((Box::new(StubPciDevice::new(params)), None));
}
Ok(devices)
}
fn create_file_backed_mappings(
cfg: &Config,
vm: &mut impl Vm,
resources: &mut SystemAllocator,
) -> Result<()> {
for mapping in &cfg.file_backed_mappings {
let file = OpenOptions::new()
.read(true)
.write(mapping.writable)
.custom_flags(if mapping.sync { libc::O_SYNC } else { 0 })
.open(&mapping.path)
.context("failed to open file for file-backed mapping")?;
let prot = if mapping.writable {
Protection::read_write()
} else {
Protection::read()
};
let size = mapping
.size
.try_into()
.context("Invalid size for file-backed mapping")?;
let memory_mapping = MemoryMappingBuilder::new(size)
.from_file(&file)
.offset(mapping.offset)
.protection(prot)
.build()
.context("failed to map backing file for file-backed mapping")?;
resources
.mmio_allocator_any()
.allocate_at(
mapping.address,
mapping.size,
Alloc::FileBacked(mapping.address),
"file-backed mapping".to_owned(),
)
.context("failed to allocate guest address for file-backed mapping")?;
vm.add_memory_region(
GuestAddress(mapping.address),
Box::new(memory_mapping),
!mapping.writable,
/* log_dirty_pages = */ false,
)
.context("failed to configure file-backed mapping")?;
}
Ok(())
}
#[derive(Copy, Clone)]
#[cfg_attr(not(feature = "tpm"), allow(dead_code))]
struct Ids {
uid: uid_t,
gid: gid_t,
}
// Set the uid/gid for the jailed process and give a basic id map. This is
// required for bind mounts to work.
fn add_current_user_to_jail(jail: &mut Minijail) -> Result<Ids> {
let crosvm_uid = geteuid();
let crosvm_gid = getegid();
jail.uidmap(&format!("{0} {0} 1", crosvm_uid))
.context("error setting UID map")?;
jail.gidmap(&format!("{0} {0} 1", crosvm_gid))
.context("error setting GID map")?;
if crosvm_uid != 0 {
jail.change_uid(crosvm_uid);
}
if crosvm_gid != 0 {
jail.change_gid(crosvm_gid);
}
Ok(Ids {
uid: crosvm_uid,
gid: crosvm_gid,
})
}
fn add_current_user_as_root_to_jail(jail: &mut Minijail) -> Result<Ids> {
let crosvm_uid = geteuid();
let crosvm_gid = getegid();
jail.uidmap(&format!("0 {0} 1", crosvm_uid))
.context("error setting UID map")?;
jail.gidmap(&format!("0 {0} 1", crosvm_gid))
.context("error setting GID map")?;
Ok(Ids {
uid: crosvm_uid,
gid: crosvm_gid,
})
}
trait IntoUnixStream {
fn into_unix_stream(self) -> Result<UnixStream>;
}
impl<'a> IntoUnixStream for &'a Path {
fn into_unix_stream(self) -> Result<UnixStream> {
if let Some(fd) = safe_descriptor_from_path(self).context("failed to open event device")? {
Ok(fd.into())
} else {
UnixStream::connect(self).context("failed to open event device")
}
}
}
impl<'a> IntoUnixStream for &'a PathBuf {
fn into_unix_stream(self) -> Result<UnixStream> {
self.as_path().into_unix_stream()
}
}
impl IntoUnixStream for UnixStream {
fn into_unix_stream(self) -> Result<UnixStream> {
Ok(self)
}
}
fn setup_vcpu_signal_handler<T: Vcpu>(use_hypervisor_signals: bool) -> Result<()> {
if use_hypervisor_signals {
unsafe {
extern "C" fn handle_signal(_: c_int) {}
// Our signal handler does nothing and is trivially async signal safe.
register_rt_signal_handler(SIGRTMIN() + 0, handle_signal)
.context("error registering signal handler")?;
}
block_signal(SIGRTMIN() + 0).context("failed to block signal")?;
} else {
unsafe {
extern "C" fn handle_signal<T: Vcpu>(_: c_int) {
T::set_local_immediate_exit(true);
}
register_rt_signal_handler(SIGRTMIN() + 0, handle_signal::<T>)
.context("error registering signal handler")?;
}
}
Ok(())
}
// Sets up a vcpu and converts it into a runnable vcpu.
fn runnable_vcpu<V>(
cpu_id: usize,
kvm_vcpu_id: usize,
vcpu: Option<V>,
vm: impl VmArch,
irq_chip: &mut dyn IrqChipArch,
vcpu_count: usize,
run_rt: bool,
vcpu_affinity: Vec<usize>,
no_smt: bool,
has_bios: bool,
use_hypervisor_signals: bool,
enable_per_vm_core_scheduling: bool,
host_cpu_topology: bool,
vcpu_cgroup_tasks_file: Option<File>,
) -> Result<(V, VcpuRunHandle)>
where
V: VcpuArch,
{
let mut vcpu = match vcpu {
Some(v) => v,
None => {
// If vcpu is None, it means this arch/hypervisor requires create_vcpu to be called from
// the vcpu thread.
match vm
.create_vcpu(kvm_vcpu_id)
.context("failed to create vcpu")?
.downcast::<V>()
{
Ok(v) => *v,
Err(_) => panic!("VM created wrong type of VCPU"),
}
}
};
irq_chip
.add_vcpu(cpu_id, &vcpu)
.context("failed to add vcpu to irq chip")?;
if !vcpu_affinity.is_empty() {
if let Err(e) = set_cpu_affinity(vcpu_affinity) {
error!("Failed to set CPU affinity: {}", e);
}
}
Arch::configure_vcpu(
&vm,
vm.get_hypervisor(),
irq_chip,
&mut vcpu,
cpu_id,
vcpu_count,
has_bios,
no_smt,
host_cpu_topology,
)
.context("failed to configure vcpu")?;
if !enable_per_vm_core_scheduling {
// Do per-vCPU core scheduling by setting a unique cookie to each vCPU.
if let Err(e) = enable_core_scheduling() {
error!("Failed to enable core scheduling: {}", e);
}
}
// Move vcpu thread to cgroup
if let Some(mut f) = vcpu_cgroup_tasks_file {
f.write_all(base::gettid().to_string().as_bytes())
.context("failed to write vcpu tid to cgroup tasks")?;
}
if run_rt {
const DEFAULT_VCPU_RT_LEVEL: u16 = 6;
if let Err(e) = set_rt_prio_limit(u64::from(DEFAULT_VCPU_RT_LEVEL))
.and_then(|_| set_rt_round_robin(i32::from(DEFAULT_VCPU_RT_LEVEL)))
{
warn!("Failed to set vcpu to real time: {}", e);
}
}
if use_hypervisor_signals {
let mut v = get_blocked_signals().context("failed to retrieve signal mask for vcpu")?;
v.retain(|&x| x != SIGRTMIN() + 0);
vcpu.set_signal_mask(&v)
.context("failed to set the signal mask for vcpu")?;
}
let vcpu_run_handle = vcpu
.take_run_handle(Some(SIGRTMIN() + 0))
.context("failed to set thread id for vcpu")?;
Ok((vcpu, vcpu_run_handle))
}
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
fn handle_debug_msg<V>(
cpu_id: usize,
vcpu: &V,
guest_mem: &GuestMemory,
d: VcpuDebug,
reply_tube: &mpsc::Sender<VcpuDebugStatusMessage>,
) -> Result<()>
where
V: VcpuArch + 'static,
{
match d {
VcpuDebug::ReadRegs => {
let msg = VcpuDebugStatusMessage {
cpu: cpu_id as usize,
msg: VcpuDebugStatus::RegValues(
Arch::debug_read_registers(vcpu as &V)
.context("failed to handle a gdb ReadRegs command")?,
),
};
reply_tube
.send(msg)
.context("failed to send a debug status to GDB thread")
}
VcpuDebug::WriteRegs(regs) => {
Arch::debug_write_registers(vcpu as &V, &regs)
.context("failed to handle a gdb WriteRegs command")?;
reply_tube
.send(VcpuDebugStatusMessage {
cpu: cpu_id as usize,
msg: VcpuDebugStatus::CommandComplete,
})
.context("failed to send a debug status to GDB thread")
}
VcpuDebug::ReadMem(vaddr, len) => {
let msg = VcpuDebugStatusMessage {
cpu: cpu_id as usize,
msg: VcpuDebugStatus::MemoryRegion(
Arch::debug_read_memory(vcpu as &V, guest_mem, vaddr, len)
.unwrap_or(Vec::new()),
),
};
reply_tube
.send(msg)
.context("failed to send a debug status to GDB thread")
}
VcpuDebug::WriteMem(vaddr, buf) => {
Arch::debug_write_memory(vcpu as &V, guest_mem, vaddr, &buf)
.context("failed to handle a gdb WriteMem command")?;
reply_tube
.send(VcpuDebugStatusMessage {
cpu: cpu_id as usize,
msg: VcpuDebugStatus::CommandComplete,
})
.context("failed to send a debug status to GDB thread")
}
VcpuDebug::EnableSinglestep => {
Arch::debug_enable_singlestep(vcpu as &V)
.context("failed to handle a gdb EnableSingleStep command")?;
reply_tube
.send(VcpuDebugStatusMessage {
cpu: cpu_id as usize,
msg: VcpuDebugStatus::CommandComplete,
})
.context("failed to send a debug status to GDB thread")
}
VcpuDebug::SetHwBreakPoint(addrs) => {
Arch::debug_set_hw_breakpoints(vcpu as &V, &addrs)
.context("failed to handle a gdb SetHwBreakPoint command")?;
reply_tube
.send(VcpuDebugStatusMessage {
cpu: cpu_id as usize,
msg: VcpuDebugStatus::CommandComplete,
})
.context("failed to send a debug status to GDB thread")
}
}
}
fn run_vcpu<V>(
cpu_id: usize,
kvm_vcpu_id: usize,
vcpu: Option<V>,
vm: impl VmArch + 'static,
mut irq_chip: Box<dyn IrqChipArch + 'static>,
vcpu_count: usize,
run_rt: bool,
vcpu_affinity: Vec<usize>,
delay_rt: bool,
no_smt: bool,
start_barrier: Arc<Barrier>,
has_bios: bool,
mut io_bus: devices::Bus,
mut mmio_bus: devices::Bus,
exit_evt: Event,
requires_pvclock_ctrl: bool,
from_main_tube: mpsc::Receiver<VcpuControl>,
use_hypervisor_signals: bool,
#[cfg(all(target_arch = "x86_64", feature = "gdb"))] to_gdb_tube: Option<
mpsc::Sender<VcpuDebugStatusMessage>,
>,
enable_per_vm_core_scheduling: bool,
host_cpu_topology: bool,
vcpu_cgroup_tasks_file: Option<File>,
) -> Result<JoinHandle<()>>
where
V: VcpuArch + 'static,
{
thread::Builder::new()
.name(format!("crosvm_vcpu{}", cpu_id))
.spawn(move || {
// The VCPU thread must trigger the `exit_evt` in all paths, and a `ScopedEvent`'s Drop
// implementation accomplishes that.
let _scoped_exit_evt = ScopedEvent::from(exit_evt);
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
let guest_mem = vm.get_memory().clone();
let runnable_vcpu = runnable_vcpu(
cpu_id,
kvm_vcpu_id,
vcpu,
vm,
irq_chip.as_mut(),
vcpu_count,
run_rt && !delay_rt,
vcpu_affinity,
no_smt,
has_bios,
use_hypervisor_signals,
enable_per_vm_core_scheduling,
host_cpu_topology,
vcpu_cgroup_tasks_file,
);
start_barrier.wait();
let (vcpu, vcpu_run_handle) = match runnable_vcpu {
Ok(v) => v,
Err(e) => {
error!("failed to start vcpu {}: {:#}", cpu_id, e);
return;
}
};
let mut run_mode = VmRunMode::Running;
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
if to_gdb_tube.is_some() {
// Wait until a GDB client attaches
run_mode = VmRunMode::Breakpoint;
}
let mut interrupted_by_signal = false;
mmio_bus.set_access_id(cpu_id);
io_bus.set_access_id(cpu_id);
'vcpu_loop: loop {
// Start by checking for messages to process and the run state of the CPU.
// An extra check here for Running so there isn't a need to call recv unless a
// message is likely to be ready because a signal was sent.
if interrupted_by_signal || run_mode != VmRunMode::Running {
'state_loop: loop {
// Tries to get a pending message without blocking first.
let msg = match from_main_tube.try_recv() {
Ok(m) => m,
Err(mpsc::TryRecvError::Empty) if run_mode == VmRunMode::Running => {
// If the VM is running and no message is pending, the state won't
// change.
break 'state_loop;
}
Err(mpsc::TryRecvError::Empty) => {
// If the VM is not running, wait until a message is ready.
match from_main_tube.recv() {
Ok(m) => m,
Err(mpsc::RecvError) => {
error!("Failed to read from main tube in vcpu");
break 'vcpu_loop;
}
}
}
Err(mpsc::TryRecvError::Disconnected) => {
error!("Failed to read from main tube in vcpu");
break 'vcpu_loop;
}
};
// Collect all pending messages.
let mut messages = vec![msg];
messages.append(&mut from_main_tube.try_iter().collect());
for msg in messages {
match msg {
VcpuControl::RunState(new_mode) => {
run_mode = new_mode;
match run_mode {
VmRunMode::Running => break 'state_loop,
VmRunMode::Suspending => {
// On KVM implementations that use a paravirtualized
// clock (e.g. x86), a flag must be set to indicate to
// the guest kernel that a vCPU was suspended. The guest
// kernel will use this flag to prevent the soft lockup
// detection from triggering when this vCPU resumes,
// which could happen days later in realtime.
if requires_pvclock_ctrl {
if let Err(e) = vcpu.pvclock_ctrl() {
error!(
"failed to tell hypervisor vcpu {} is suspending: {}",
cpu_id, e
);
}
}
}
VmRunMode::Breakpoint => {}
VmRunMode::Exiting => break 'vcpu_loop,
}
}
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
VcpuControl::Debug(d) => {
match &to_gdb_tube {
Some(ref ch) => {
if let Err(e) = handle_debug_msg(
cpu_id, &vcpu, &guest_mem, d, ch,
) {
error!("Failed to handle gdb message: {}", e);
}
},
None => {
error!("VcpuControl::Debug received while GDB feature is disabled: {:?}", d);
}
}
}
VcpuControl::MakeRT => {
if run_rt && delay_rt {
info!("Making vcpu {} RT\n", cpu_id);
const DEFAULT_VCPU_RT_LEVEL: u16 = 6;
if let Err(e) = set_rt_prio_limit(
u64::from(DEFAULT_VCPU_RT_LEVEL))
.and_then(|_|
set_rt_round_robin(
i32::from(DEFAULT_VCPU_RT_LEVEL)
))
{
warn!("Failed to set vcpu to real time: {}", e);
}
}
}
}
}
}
}
interrupted_by_signal = false;
// Vcpus may have run a HLT instruction, which puts them into a state other than
// VcpuRunState::Runnable. In that case, this call to wait_until_runnable blocks
// until either the irqchip receives an interrupt for this vcpu, or until the main
// thread kicks this vcpu as a result of some VmControl operation. In most IrqChip
// implementations HLT instructions do not make it to crosvm, and thus this is a
// no-op that always returns VcpuRunState::Runnable.
match irq_chip.wait_until_runnable(&vcpu) {
Ok(VcpuRunState::Runnable) => {}
Ok(VcpuRunState::Interrupted) => interrupted_by_signal = true,
Err(e) => error!(
"error waiting for vcpu {} to become runnable: {}",
cpu_id, e
),
}
if !interrupted_by_signal {
match vcpu.run(&vcpu_run_handle) {
Ok(VcpuExit::IoIn { port, mut size }) => {
let mut data = [0; 8];
if size > data.len() {
error!("unsupported IoIn size of {} bytes at port {:#x}", size, port);
size = data.len();
}
io_bus.read(port as u64, &mut data[..size]);
if let Err(e) = vcpu.set_data(&data[..size]) {
error!("failed to set return data for IoIn at port {:#x}: {}", port, e);
}
}
Ok(VcpuExit::IoOut {
port,
mut size,
data,
}) => {
if size > data.len() {
error!("unsupported IoOut size of {} bytes at port {:#x}", size, port);
size = data.len();
}
io_bus.write(port as u64, &data[..size]);
}
Ok(VcpuExit::MmioRead { address, size }) => {
let mut data = [0; 8];
mmio_bus.read(address, &mut data[..size]);
// Setting data for mmio can not fail.
let _ = vcpu.set_data(&data[..size]);
}
Ok(VcpuExit::MmioWrite {
address,
size,
data,
}) => {
mmio_bus.write(address, &data[..size]);
}
Ok(VcpuExit::IoapicEoi { vector }) => {
if let Err(e) = irq_chip.broadcast_eoi(vector) {
error!(
"failed to broadcast eoi {} on vcpu {}: {}",
vector, cpu_id, e
);
}
}
Ok(VcpuExit::IrqWindowOpen) => {}
Ok(VcpuExit::Hlt) => irq_chip.halted(cpu_id),
Ok(VcpuExit::Shutdown) => break,
Ok(VcpuExit::FailEntry {
hardware_entry_failure_reason,
}) => {
error!("vcpu hw run failure: {:#x}", hardware_entry_failure_reason);
break;
}
Ok(VcpuExit::SystemEvent(_, _)) => break,
Ok(VcpuExit::Debug { .. }) => {
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
{
let msg = VcpuDebugStatusMessage {
cpu: cpu_id as usize,
msg: VcpuDebugStatus::HitBreakPoint,
};
if let Some(ref ch) = to_gdb_tube {
if let Err(e) = ch.send(msg) {
error!("failed to notify breakpoint to GDB thread: {}", e);
break;
}
}
run_mode = VmRunMode::Breakpoint;
}
}
Ok(r) => warn!("unexpected vcpu exit: {:?}", r),
Err(e) => match e.errno() {
libc::EINTR => interrupted_by_signal = true,
libc::EAGAIN => {}
_ => {
error!("vcpu hit unknown error: {}", e);
break;
}
},
}
}
if interrupted_by_signal {
if use_hypervisor_signals {
// Try to clear the signal that we use to kick VCPU if it is pending before
// attempting to handle pause requests.
if let Err(e) = clear_signal(SIGRTMIN() + 0) {
error!("failed to clear pending signal: {}", e);
break;
}
} else {
vcpu.set_immediate_exit(false);
}
}
if let Err(e) = irq_chip.inject_interrupts(&vcpu) {
error!("failed to inject interrupts for vcpu {}: {}", cpu_id, e);
}
}
})
.context("failed to spawn VCPU thread")
}
fn setup_vm_components(cfg: &Config) -> Result<VmComponents> {
let initrd_image = if let Some(initrd_path) = &cfg.initrd_path {
Some(
open_file(
initrd_path,
true, /*read_only*/
false, /*O_DIRECT*/
)
.with_context(|| format!("failed to open initrd {}", initrd_path.display()))?,
)
} else {
None
};
let vm_image = match cfg.executable_path {
Some(Executable::Kernel(ref kernel_path)) => VmImage::Kernel(
open_file(
kernel_path,
true, /*read_only*/
false, /*O_DIRECT*/
)
.with_context(|| format!("failed to open kernel image {}", kernel_path.display()))?,
),
Some(Executable::Bios(ref bios_path)) => VmImage::Bios(
open_file(bios_path, true /*read_only*/, false /*O_DIRECT*/)
.with_context(|| format!("failed to open bios {}", bios_path.display()))?,
),
_ => panic!("Did not receive a bios or kernel, should be impossible."),
};
let swiotlb = if let Some(size) = cfg.swiotlb {
Some(
size.checked_mul(1024 * 1024)
.ok_or_else(|| anyhow!("requested swiotlb size too large"))?,
)
} else {
match cfg.protected_vm {
ProtectionType::Protected | ProtectionType::ProtectedWithoutFirmware => {
Some(64 * 1024 * 1024)
}
ProtectionType::Unprotected => None,
}
};
Ok(VmComponents {
memory_size: cfg
.memory
.unwrap_or(256)
.checked_mul(1024 * 1024)
.ok_or_else(|| anyhow!("requested memory size too large"))?,
swiotlb,
vcpu_count: cfg.vcpu_count.unwrap_or(1),
vcpu_affinity: cfg.vcpu_affinity.clone(),
cpu_clusters: cfg.cpu_clusters.clone(),
cpu_capacity: cfg.cpu_capacity.clone(),
no_smt: cfg.no_smt,
hugepages: cfg.hugepages,
vm_image,
android_fstab: cfg
.android_fstab
.as_ref()
.map(|x| {
File::open(x)
.with_context(|| format!("failed to open android fstab file {}", x.display()))
})
.map_or(Ok(None), |v| v.map(Some))?,
pstore: cfg.pstore.clone(),
initrd_image,
extra_kernel_params: cfg.params.clone(),
acpi_sdts: cfg
.acpi_tables
.iter()
.map(|path| {
SDT::from_file(path)
.with_context(|| format!("failed to open ACPI file {}", path.display()))
})
.collect::<Result<Vec<SDT>>>()?,
rt_cpus: cfg.rt_cpus.clone(),
delay_rt: cfg.delay_rt,
protected_vm: cfg.protected_vm,
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
gdb: None,
dmi_path: cfg.dmi_path.clone(),
no_legacy: cfg.no_legacy,
host_cpu_topology: cfg.host_cpu_topology,
})
}
pub enum ExitState {
Reset,
Stop,
}
pub fn run_config(cfg: Config) -> Result<ExitState> {
let components = setup_vm_components(&cfg)?;
let guest_mem_layout =
Arch::guest_memory_layout(&components).context("failed to create guest memory layout")?;
let guest_mem = GuestMemory::new(&guest_mem_layout).context("failed to create guest memory")?;
let mut mem_policy = MemoryPolicy::empty();
if components.hugepages {
mem_policy |= MemoryPolicy::USE_HUGEPAGES;
}
guest_mem.set_memory_policy(mem_policy);
let kvm = Kvm::new_with_path(&cfg.kvm_device_path).context("failed to create kvm")?;
let vm = KvmVm::new(&kvm, guest_mem, components.protected_vm).context("failed to create vm")?;
let vm_clone = vm.try_clone().context("failed to clone vm")?;
enum KvmIrqChip {
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Split(KvmSplitIrqChip),
Kernel(KvmKernelIrqChip),
}
impl KvmIrqChip {
fn as_mut(&mut self) -> &mut dyn IrqChipArch {
match self {
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
KvmIrqChip::Split(i) => i,
KvmIrqChip::Kernel(i) => i,
}
}
}
let ioapic_host_tube;
let mut irq_chip = if cfg.split_irqchip {
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
unimplemented!("KVM split irqchip mode only supported on x86 processors");
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{
let (host_tube, ioapic_device_tube) = Tube::pair().context("failed to create tube")?;
ioapic_host_tube = Some(host_tube);
KvmIrqChip::Split(
KvmSplitIrqChip::new(
vm_clone,
components.vcpu_count,
ioapic_device_tube,
Some(120),
)
.context("failed to create IRQ chip")?,
)
}
} else {
ioapic_host_tube = None;
KvmIrqChip::Kernel(
KvmKernelIrqChip::new(vm_clone, components.vcpu_count)
.context("failed to create IRQ chip")?,
)
};
run_vm::<KvmVcpu, KvmVm>(cfg, components, vm, irq_chip.as_mut(), ioapic_host_tube)
}
fn run_vm<Vcpu, V>(
cfg: Config,
#[allow(unused_mut)] mut components: VmComponents,
mut vm: V,
irq_chip: &mut dyn IrqChipArch,
ioapic_host_tube: Option<Tube>,
) -> Result<ExitState>
where
Vcpu: VcpuArch + 'static,
V: VmArch + 'static,
{
if cfg.sandbox {
// Printing something to the syslog before entering minijail so that libc's syslogger has a
// chance to open files necessary for its operation, like `/etc/localtime`. After jailing,
// access to those files will not be possible.
info!("crosvm entering multiprocess mode");
}
#[cfg(feature = "usb")]
let (usb_control_tube, usb_provider) =
HostBackendDeviceProvider::new().context("failed to create usb provider")?;
// Masking signals is inherently dangerous, since this can persist across clones/execs. Do this
// before any jailed devices have been spawned, so that we can catch any of them that fail very
// quickly.
let sigchld_fd = SignalFd::new(libc::SIGCHLD).context("failed to create signalfd")?;
let control_server_socket = match &cfg.socket_path {
Some(path) => Some(UnlinkUnixSeqpacketListener(
UnixSeqpacketListener::bind(path).context("failed to create control server")?,
)),
None => None,
};
let mut control_tubes = Vec::new();
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
if let Some(port) = cfg.gdb {
// GDB needs a control socket to interrupt vcpus.
let (gdb_host_tube, gdb_control_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::Vm(gdb_host_tube));
components.gdb = Some((port, gdb_control_tube));
}
for wl_cfg in &cfg.vhost_user_wl {
let wayland_host_tube = UnixSeqpacket::connect(&wl_cfg.vm_tube)
.map(Tube::new)
.context("failed to connect to wayland tube")?;
control_tubes.push(TaggedControlTube::VmMemory(wayland_host_tube));
}
let mut vhost_user_gpu_tubes = Vec::with_capacity(cfg.vhost_user_gpu.len());
for _ in 0..cfg.vhost_user_gpu.len() {
let (host_tube, device_tube) = Tube::pair().context("failed to create tube")?;
vhost_user_gpu_tubes.push((
host_tube.try_clone().context("failed to clone tube")?,
device_tube,
));
control_tubes.push(TaggedControlTube::VmMemory(host_tube));
}
let (wayland_host_tube, wayland_device_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmMemory(wayland_host_tube));
let (balloon_host_tube, balloon_device_tube) = if cfg.balloon {
if let Some(ref path) = cfg.balloon_control {
(
None,
Some(Tube::new(
UnixSeqpacket::connect(path).context("failed to create balloon control")?,
)),
)
} else {
// Balloon gets a special socket so balloon requests can be forwarded
// from the main process.
let (host, device) = Tube::pair().context("failed to create tube")?;
// Set recv timeout to avoid deadlock on sending BalloonControlCommand
// before the guest is ready.
host.set_recv_timeout(Some(Duration::from_millis(100)))
.context("failed to set timeout")?;
(Some(host), Some(device))
}
} else {
(None, None)
};
// Create one control socket per disk.
let mut disk_device_tubes = Vec::new();
let mut disk_host_tubes = Vec::new();
let disk_count = cfg.disks.len();
for _ in 0..disk_count {
let (disk_host_tub, disk_device_tube) = Tube::pair().context("failed to create tube")?;
disk_host_tubes.push(disk_host_tub);
disk_device_tubes.push(disk_device_tube);
}
let mut pmem_device_tubes = Vec::new();
let pmem_count = cfg.pmem_devices.len();
for _ in 0..pmem_count {
let (pmem_host_tube, pmem_device_tube) = Tube::pair().context("failed to create tube")?;
pmem_device_tubes.push(pmem_device_tube);
control_tubes.push(TaggedControlTube::VmMsync(pmem_host_tube));
}
let (gpu_host_tube, gpu_device_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmMemory(gpu_host_tube));
if let Some(ioapic_host_tube) = ioapic_host_tube {
control_tubes.push(TaggedControlTube::VmIrq(ioapic_host_tube));
}
let battery = if cfg.battery_type.is_some() {
#[cfg_attr(not(feature = "power-monitor-powerd"), allow(clippy::manual_map))]
let jail = match simple_jail(&cfg, "battery")? {
#[cfg_attr(not(feature = "power-monitor-powerd"), allow(unused_mut))]
Some(mut jail) => {
// Setup a bind mount to the system D-Bus socket if the powerd monitor is used.
#[cfg(feature = "power-monitor-powerd")]
{
add_current_user_to_jail(&mut jail)?;
// Create a tmpfs in the device's root directory so that we can bind mount files.
jail.mount_with_data(
Path::new("none"),
Path::new("/"),
"tmpfs",
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
"size=67108864",
)?;
let system_bus_socket_path = Path::new("/run/dbus/system_bus_socket");
jail.mount_bind(system_bus_socket_path, system_bus_socket_path, true)?;
}
Some(jail)
}
None => None,
};
(&cfg.battery_type, jail)
} else {
(&cfg.battery_type, None)
};
let map_request: Arc<Mutex<Option<ExternalMapping>>> = Arc::new(Mutex::new(None));
let fs_count = cfg
.shared_dirs
.iter()
.filter(|sd| sd.kind == SharedDirKind::FS)
.count();
let mut fs_device_tubes = Vec::with_capacity(fs_count);
for _ in 0..fs_count {
let (fs_host_tube, fs_device_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::Fs(fs_host_tube));
fs_device_tubes.push(fs_device_tube);
}
let exit_evt = Event::new().context("failed to create event")?;
let reset_evt = Event::new().context("failed to create event")?;
let mut sys_allocator = Arch::create_system_allocator(&vm);
// Allocate the ramoops region first. AArch64::build_vm() assumes this.
let ramoops_region = match &components.pstore {
Some(pstore) => Some(
arch::pstore::create_memory_region(&mut vm, &mut sys_allocator, pstore)
.context("failed to allocate pstore region")?,
),
None => None,
};
create_file_backed_mappings(&cfg, &mut vm, &mut sys_allocator)?;
let phys_max_addr = (1u64 << vm.get_guest_phys_addr_bits()) - 1;
#[cfg(feature = "gpu")]
// Hold on to the render server jail so it keeps running until we exit run_vm()
let mut _render_server_jail = None;
#[cfg(feature = "gpu")]
let mut render_server_fd = None;
#[cfg(feature = "gpu")]
if let Some(gpu_parameters) = &cfg.gpu_parameters {
if let Some(ref render_server_parameters) = gpu_parameters.render_server {
let (jail, fd) = start_gpu_render_server(&cfg, render_server_parameters)?;
_render_server_jail = Some(ScopedMinijail(jail));
render_server_fd = Some(fd);
}
}
let init_balloon_size = components
.memory_size
.checked_sub(cfg.init_memory.map_or(components.memory_size, |m| {
m.checked_mul(1024 * 1024).unwrap_or(u64::MAX)
}))
.context("failed to calculate init balloon size")?;
let mut devices = create_devices(
&cfg,
&mut vm,
&mut sys_allocator,
&exit_evt,
phys_max_addr,
&mut control_tubes,
wayland_device_tube,
gpu_device_tube,
vhost_user_gpu_tubes,
balloon_device_tube,
init_balloon_size,
&mut disk_device_tubes,
&mut pmem_device_tubes,
&mut fs_device_tubes,
#[cfg(feature = "usb")]
usb_provider,
Arc::clone(&map_request),
#[cfg(feature = "gpu")]
render_server_fd,
)?;
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
for device in devices
.iter_mut()
.filter_map(|(dev, _)| dev.as_pci_device_mut())
{
let sdts = device
.generate_acpi(components.acpi_sdts)
.or_else(|| {
error!("ACPI table generation error");
None
})
.ok_or_else(|| anyhow!("failed to generate ACPI table"))?;
components.acpi_sdts = sdts;
}
// KVM_CREATE_VCPU uses apic id for x86 and uses cpu id for others.
let mut kvm_vcpu_ids = Vec::new();
#[cfg_attr(not(feature = "direct"), allow(unused_mut))]
let mut linux = Arch::build_vm::<V, Vcpu>(
components,
&exit_evt,
&reset_evt,
&mut sys_allocator,
&cfg.serial_parameters,
simple_jail(&cfg, "serial")?,
battery,
vm,
ramoops_region,
devices,
irq_chip,
&mut kvm_vcpu_ids,
)
.context("the architecture failed to build the vm")?;
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{
// Create Pcie Root Port
let pcie_root_port = Arc::new(Mutex::new(PcieRootPort::new()));
let (msi_host_tube, msi_device_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmIrq(msi_host_tube));
let sec_bus = (1..255)
.find(|&bus_num| sys_allocator.pci_bus_empty(bus_num))
.context("failed to find empty bus for Pci hotplug")?;
let pci_bridge = Box::new(PciBridge::new(
pcie_root_port.clone(),
msi_device_tube,
0,
sec_bus,
));
Arch::register_pci_device(&mut linux, pci_bridge, None, &mut sys_allocator)
.context("Failed to configure pci bridge device")?;
linux.hotplug_bus.push(pcie_root_port);
}
#[cfg(feature = "direct")]
if let Some(pmio) = &cfg.direct_pmio {
let direct_io = Arc::new(
devices::DirectIo::new(&pmio.path, false).context("failed to open direct io device")?,
);
for range in pmio.ranges.iter() {
linux
.io_bus
.insert_sync(direct_io.clone(), range.base, range.len)
.unwrap();
}
};
#[cfg(feature = "direct")]
if let Some(mmio) = &cfg.direct_mmio {
let direct_mmio = Arc::new(
devices::DirectMmio::new(&mmio.path, false, &mmio.ranges)
.context("failed to open direct mmio device")?,
);
for range in mmio.ranges.iter() {
linux
.mmio_bus
.insert_sync(direct_mmio.clone(), range.base, range.len)
.unwrap();
}
};
#[cfg(feature = "direct")]
let mut irqs = Vec::new();
#[cfg(feature = "direct")]
for irq in &cfg.direct_level_irq {
if !sys_allocator.reserve_irq(*irq) {
warn!("irq {} already reserved.", irq);
}
let trigger = Event::new().context("failed to create event")?;
let resample = Event::new().context("failed to create event")?;
linux
.irq_chip
.register_irq_event(*irq, &trigger, Some(&resample))
.unwrap();
let direct_irq = devices::DirectIrq::new(trigger, Some(resample))
.context("failed to enable interrupt forwarding")?;
direct_irq
.irq_enable(*irq)
.context("failed to enable interrupt forwarding")?;
irqs.push(direct_irq);
}
#[cfg(feature = "direct")]
for irq in &cfg.direct_edge_irq {
if !sys_allocator.reserve_irq(*irq) {
warn!("irq {} already reserved.", irq);
}
let trigger = Event::new().context("failed to create event")?;
linux
.irq_chip
.register_irq_event(*irq, &trigger, None)
.unwrap();
let direct_irq = devices::DirectIrq::new(trigger, None)
.context("failed to enable interrupt forwarding")?;
direct_irq
.irq_enable(*irq)
.context("failed to enable interrupt forwarding")?;
irqs.push(direct_irq);
}
let gralloc = RutabagaGralloc::new().context("failed to create gralloc")?;
run_control(
linux,
sys_allocator,
cfg,
control_server_socket,
control_tubes,
balloon_host_tube,
&disk_host_tubes,
#[cfg(feature = "usb")]
usb_control_tube,
exit_evt,
reset_evt,
sigchld_fd,
Arc::clone(&map_request),
gralloc,
kvm_vcpu_ids,
)
}
fn get_hp_bus<V: VmArch, Vcpu: VcpuArch>(
linux: &RunnableLinuxVm<V, Vcpu>,
host_addr: PciAddress,
) -> Result<(Arc<Mutex<dyn HotPlugBus>>, u8)> {
for hp_bus in linux.hotplug_bus.iter() {
if let Some(number) = hp_bus.lock().is_match(host_addr) {
return Ok((hp_bus.clone(), number));
}
}
Err(anyhow!("Failed to find a suitable hotplug bus"))
}
fn add_vfio_device<V: VmArch, Vcpu: VcpuArch>(
linux: &mut RunnableLinuxVm<V, Vcpu>,
sys_allocator: &mut SystemAllocator,
cfg: &Config,
control_tubes: &mut Vec<TaggedControlTube>,
vfio_path: &Path,
) -> Result<()> {
let host_os_str = vfio_path
.file_name()
.ok_or_else(|| anyhow!("failed to parse or find vfio path"))?;
let host_str = host_os_str
.to_str()
.ok_or_else(|| anyhow!("failed to parse or find vfio path"))?;
let host_addr = PciAddress::from_string(host_str);
let (hp_bus, bus_num) = get_hp_bus(linux, host_addr)?;
let mut endpoints: BTreeMap<u32, Arc<Mutex<VfioContainer>>> = BTreeMap::new();
let (vfio_pci_device, jail) = create_vfio_device(
cfg,
&linux.vm,
sys_allocator,
control_tubes,
vfio_path,
Some(bus_num),
&mut endpoints,
None,
IommuDevType::NoIommu,
)?;
let pci_address = Arch::register_pci_device(linux, vfio_pci_device, jail, sys_allocator)
.context("Failed to configure pci hotplug device")?;
let host_os_str = vfio_path
.file_name()
.ok_or_else(|| anyhow!("failed to parse or find vfio path"))?;
let host_str = host_os_str
.to_str()
.ok_or_else(|| anyhow!("failed to parse or find vfio path"))?;
let host_addr = PciAddress::from_string(host_str);
let host_key = HostHotPlugKey::Vfio { host_addr };
let mut hp_bus = hp_bus.lock();
hp_bus.add_hotplug_device(host_key, pci_address);
hp_bus.hot_plug(pci_address);
Ok(())
}
fn remove_vfio_device<V: VmArch, Vcpu: VcpuArch>(
linux: &RunnableLinuxVm<V, Vcpu>,
sys_allocator: &mut SystemAllocator,
vfio_path: &Path,
) -> Result<()> {
let host_os_str = vfio_path
.file_name()
.ok_or_else(|| anyhow!("failed to parse or find vfio path"))?;
let host_str = host_os_str
.to_str()
.ok_or_else(|| anyhow!("failed to parse or find vfio path"))?;
let host_addr = PciAddress::from_string(host_str);
let host_key = HostHotPlugKey::Vfio { host_addr };
for hp_bus in linux.hotplug_bus.iter() {
let mut hp_bus_lock = hp_bus.lock();
if let Some(pci_addr) = hp_bus_lock.get_hotplug_device(host_key) {
hp_bus_lock.hot_unplug(pci_addr);
sys_allocator.release_pci(pci_addr.bus, pci_addr.dev, pci_addr.func);
return Ok(());
}
}
Err(anyhow!("HotPlugBus hasn't been implemented"))
}
fn handle_vfio_command<V: VmArch, Vcpu: VcpuArch>(
linux: &mut RunnableLinuxVm<V, Vcpu>,
sys_allocator: &mut SystemAllocator,
cfg: &Config,
add_tubes: &mut Vec<TaggedControlTube>,
vfio_path: &Path,
add: bool,
) -> VmResponse {
let ret = if add {
add_vfio_device(linux, sys_allocator, cfg, add_tubes, vfio_path)
} else {
remove_vfio_device(linux, sys_allocator, vfio_path)
};
match ret {
Ok(()) => VmResponse::Ok,
Err(e) => {
error!("hanlde_vfio_command failure: {}", e);
add_tubes.clear();
VmResponse::Err(base::Error::new(libc::EINVAL))
}
}
}
/// Signals all running VCPUs to vmexit, sends VcpuControl message to each VCPU tube, and tells
/// `irq_chip` to stop blocking halted VCPUs. The channel message is set first because both the
/// signal and the irq_chip kick could cause the VCPU thread to continue through the VCPU run
/// loop.
fn kick_all_vcpus(
vcpu_handles: &[(JoinHandle<()>, mpsc::Sender<vm_control::VcpuControl>)],
irq_chip: &dyn IrqChip,
message: VcpuControl,
) {
for (handle, tube) in vcpu_handles {
if let Err(e) = tube.send(message.clone()) {
error!("failed to send VcpuControl: {}", e);
}
let _ = handle.kill(SIGRTMIN() + 0);
}
irq_chip.kick_halted_vcpus();
}
fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
mut linux: RunnableLinuxVm<V, Vcpu>,
mut sys_allocator: SystemAllocator,
cfg: Config,
control_server_socket: Option<UnlinkUnixSeqpacketListener>,
mut control_tubes: Vec<TaggedControlTube>,
balloon_host_tube: Option<Tube>,
disk_host_tubes: &[Tube],
#[cfg(feature = "usb")] usb_control_tube: Tube,
exit_evt: Event,
reset_evt: Event,
sigchld_fd: SignalFd,
map_request: Arc<Mutex<Option<ExternalMapping>>>,
mut gralloc: RutabagaGralloc,
kvm_vcpu_ids: Vec<usize>,
) -> Result<ExitState> {
#[derive(PollToken)]
enum Token {
Exit,
Reset,
Suspend,
ChildSignal,
IrqFd { index: IrqEventIndex },
VmControlServer,
VmControl { index: usize },
}
stdin()
.set_raw_mode()
.expect("failed to set terminal raw mode");
let wait_ctx = WaitContext::build_with(&[
(&exit_evt, Token::Exit),
(&reset_evt, Token::Reset),
(&linux.suspend_evt, Token::Suspend),
(&sigchld_fd, Token::ChildSignal),
])
.context("failed to add descriptor to wait context")?;
if let Some(socket_server) = &control_server_socket {
wait_ctx
.add(socket_server, Token::VmControlServer)
.context("failed to add descriptor to wait context")?;
}
for (index, socket) in control_tubes.iter().enumerate() {
wait_ctx
.add(socket.as_ref(), Token::VmControl { index })
.context("failed to add descriptor to wait context")?;
}
let events = linux
.irq_chip
.irq_event_tokens()
.context("failed to add descriptor to wait context")?;
for (index, _gsi, evt) in events {
wait_ctx
.add(&evt, Token::IrqFd { index })
.context("failed to add descriptor to wait context")?;
}
if cfg.sandbox {
// Before starting VCPUs, in case we started with some capabilities, drop them all.
drop_capabilities().context("failed to drop process capabilities")?;
}
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
// Create a channel for GDB thread.
let (to_gdb_channel, from_vcpu_channel) = if linux.gdb.is_some() {
let (s, r) = mpsc::channel();
(Some(s), Some(r))
} else {
(None, None)
};
let mut vcpu_handles = Vec::with_capacity(linux.vcpu_count);
let vcpu_thread_barrier = Arc::new(Barrier::new(linux.vcpu_count + 1));
let use_hypervisor_signals = !linux
.vm
.get_hypervisor()
.check_capability(&HypervisorCap::ImmediateExit);
setup_vcpu_signal_handler::<Vcpu>(use_hypervisor_signals)?;
let vcpus: Vec<Option<_>> = match linux.vcpus.take() {
Some(vec) => vec.into_iter().map(Some).collect(),
None => iter::repeat_with(|| None).take(linux.vcpu_count).collect(),
};
// Enable core scheduling before creating vCPUs so that the cookie will be
// shared by all vCPU threads.
// TODO(b/199312402): Avoid enabling core scheduling for the crosvm process
// itself for even better performance. Only vCPUs need the feature.
if cfg.per_vm_core_scheduling {
if let Err(e) = enable_core_scheduling() {
error!("Failed to enable core scheduling: {}", e);
}
}
let vcpu_cgroup_tasks_file = match &cfg.vcpu_cgroup_path {
None => None,
Some(cgroup_path) => {
// Move main process to cgroup_path
let mut f = File::create(&cgroup_path.join("tasks"))?;
f.write_all(process::id().to_string().as_bytes())?;
Some(f)
}
};
for (cpu_id, vcpu) in vcpus.into_iter().enumerate() {
let (to_vcpu_channel, from_main_channel) = mpsc::channel();
let vcpu_affinity = match linux.vcpu_affinity.clone() {
Some(VcpuAffinity::Global(v)) => v,
Some(VcpuAffinity::PerVcpu(mut m)) => m.remove(&cpu_id).unwrap_or_default(),
None => Default::default(),
};
let handle = run_vcpu(
cpu_id,
kvm_vcpu_ids[cpu_id],
vcpu,
linux.vm.try_clone().context("failed to clone vm")?,
linux
.irq_chip
.try_box_clone()
.context("failed to clone irqchip")?,
linux.vcpu_count,
linux.rt_cpus.contains(&cpu_id),
vcpu_affinity,
linux.delay_rt,
linux.no_smt,
vcpu_thread_barrier.clone(),
linux.has_bios,
(*linux.io_bus).clone(),
(*linux.mmio_bus).clone(),
exit_evt.try_clone().context("failed to clone event")?,
linux.vm.check_capability(VmCap::PvClockSuspend),
from_main_channel,
use_hypervisor_signals,
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
to_gdb_channel.clone(),
cfg.per_vm_core_scheduling,
cfg.host_cpu_topology,
match vcpu_cgroup_tasks_file {
None => None,
Some(ref f) => Some(
f.try_clone()
.context("failed to clone vcpu cgroup tasks file")?,
),
},
)?;
vcpu_handles.push((handle, to_vcpu_channel));
}
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
// Spawn GDB thread.
if let Some((gdb_port_num, gdb_control_tube)) = linux.gdb.take() {
let to_vcpu_channels = vcpu_handles
.iter()
.map(|(_handle, channel)| channel.clone())
.collect();
let target = GdbStub::new(
gdb_control_tube,
to_vcpu_channels,
from_vcpu_channel.unwrap(), // Must succeed to unwrap()
);
thread::Builder::new()
.name("gdb".to_owned())
.spawn(move || gdb_thread(target, gdb_port_num))
.context("failed to spawn GDB thread")?;
};
vcpu_thread_barrier.wait();
let mut exit_state = ExitState::Stop;
let mut balloon_stats_id: u64 = 0;
'wait: loop {
let events = {
match wait_ctx.wait() {
Ok(v) => v,
Err(e) => {
error!("failed to poll: {}", e);
break;
}
}
};
if let Err(e) = linux.irq_chip.process_delayed_irq_events() {
warn!("can't deliver delayed irqs: {}", e);
}
let mut vm_control_indices_to_remove = Vec::new();
for event in events.iter().filter(|e| e.is_readable) {
match event.token {
Token::Exit => {
info!("vcpu requested shutdown");
break 'wait;
}
Token::Reset => {
info!("vcpu requested reset");
exit_state = ExitState::Reset;
break 'wait;
}
Token::Suspend => {
info!("VM requested suspend");
linux.suspend_evt.read().unwrap();
kick_all_vcpus(
&vcpu_handles,
linux.irq_chip.as_irq_chip(),
VcpuControl::RunState(VmRunMode::Suspending),
);
}
Token::ChildSignal => {
// Print all available siginfo structs, then exit the loop.
while let Some(siginfo) =
sigchld_fd.read().context("failed to create signalfd")?
{
let pid = siginfo.ssi_pid;
let pid_label = match linux.pid_debug_label_map.get(&pid) {
Some(label) => format!("{} (pid {})", label, pid),
None => format!("pid {}", pid),
};
error!(
"child {} died: signo {}, status {}, code {}",
pid_label, siginfo.ssi_signo, siginfo.ssi_status, siginfo.ssi_code
);
}
break 'wait;
}
Token::IrqFd { index } => {
if let Err(e) = linux.irq_chip.service_irq_event(index) {
error!("failed to signal irq {}: {}", index, e);
}
}
Token::VmControlServer => {
if let Some(socket_server) = &control_server_socket {
match socket_server.accept() {
Ok(socket) => {
wait_ctx
.add(
&socket,
Token::VmControl {
index: control_tubes.len(),
},
)
.context("failed to add descriptor to wait context")?;
control_tubes.push(TaggedControlTube::Vm(Tube::new(socket)));
}
Err(e) => error!("failed to accept socket: {}", e),
}
}
}
Token::VmControl { index } => {
let mut add_tubes = Vec::new();
if let Some(socket) = control_tubes.get(index) {
match socket {
TaggedControlTube::Vm(tube) => match tube.recv::<VmRequest>() {
Ok(request) => {
let mut run_mode_opt = None;
let response = match request {
VmRequest::VfioCommand { vfio_path, add } => {
handle_vfio_command(
&mut linux,
&mut sys_allocator,
&cfg,
&mut add_tubes,
&vfio_path,
add,
)
}
_ => request.execute(
&mut run_mode_opt,
balloon_host_tube.as_ref(),
&mut balloon_stats_id,
disk_host_tubes,
#[cfg(feature = "usb")]
Some(&usb_control_tube),
#[cfg(not(feature = "usb"))]
None,
&mut linux.bat_control,
&vcpu_handles,
),
};
if let Err(e) = tube.send(&response) {
error!("failed to send VmResponse: {}", e);
}
if let Some(run_mode) = run_mode_opt {
info!("control socket changed run mode to {}", run_mode);
match run_mode {
VmRunMode::Exiting => {
break 'wait;
}
other => {
if other == VmRunMode::Running {
for dev in &linux.resume_notify_devices {
dev.lock().resume_imminent();
}
}
kick_all_vcpus(
&vcpu_handles,
linux.irq_chip.as_irq_chip(),
VcpuControl::RunState(other),
);
}
}
}
}
Err(e) => {
if let TubeError::Disconnected = e {
vm_control_indices_to_remove.push(index);
} else {
error!("failed to recv VmRequest: {}", e);
}
}
},
TaggedControlTube::VmMemory(tube) => {
match tube.recv::<VmMemoryRequest>() {
Ok(request) => {
let response = request.execute(
&mut linux.vm,
&mut sys_allocator,
Arc::clone(&map_request),
&mut gralloc,
);
if let Err(e) = tube.send(&response) {
error!("failed to send VmMemoryControlResponse: {}", e);
}
}
Err(e) => {
if let TubeError::Disconnected = e {
vm_control_indices_to_remove.push(index);
} else {
error!("failed to recv VmMemoryControlRequest: {}", e);
}
}
}
}
TaggedControlTube::VmIrq(tube) => match tube.recv::<VmIrqRequest>() {
Ok(request) => {
let response = {
let irq_chip = &mut linux.irq_chip;
request.execute(
|setup| match setup {
IrqSetup::Event(irq, ev) => {
if let Some(event_index) = irq_chip
.register_irq_event(irq, ev, None)?
{
match wait_ctx.add(
ev,
Token::IrqFd {
index: event_index
},
) {
Err(e) => {
warn!("failed to add IrqFd to poll context: {}", e);
Err(e)
},
Ok(_) => {
Ok(())
}
}
} else {
Ok(())
}
}
IrqSetup::Route(route) => irq_chip.route_irq(route),
IrqSetup::UnRegister(irq, ev) => irq_chip.unregister_irq_event(irq, ev),
},
&mut sys_allocator,
)
};
if let Err(e) = tube.send(&response) {
error!("failed to send VmIrqResponse: {}", e);
}
}
Err(e) => {
if let TubeError::Disconnected = e {
vm_control_indices_to_remove.push(index);
} else {
error!("failed to recv VmIrqRequest: {}", e);
}
}
},
TaggedControlTube::VmMsync(tube) => {
match tube.recv::<VmMsyncRequest>() {
Ok(request) => {
let response = request.execute(&mut linux.vm);
if let Err(e) = tube.send(&response) {
error!("failed to send VmMsyncResponse: {}", e);
}
}
Err(e) => {
if let TubeError::Disconnected = e {
vm_control_indices_to_remove.push(index);
} else {
error!("failed to recv VmMsyncRequest: {}", e);
}
}
}
}
TaggedControlTube::Fs(tube) => match tube.recv::<FsMappingRequest>() {
Ok(request) => {
let response =
request.execute(&mut linux.vm, &mut sys_allocator);
if let Err(e) = tube.send(&response) {
error!("failed to send VmResponse: {}", e);
}
}
Err(e) => {
if let TubeError::Disconnected = e {
vm_control_indices_to_remove.push(index);
} else {
error!("failed to recv VmResponse: {}", e);
}
}
},
}
}
if !add_tubes.is_empty() {
for (idx, socket) in add_tubes.iter().enumerate() {
wait_ctx
.add(
socket.as_ref(),
Token::VmControl {
index: idx + control_tubes.len(),
},
)
.context(
"failed to add hotplug vfio-pci descriptor ot wait context",
)?;
}
control_tubes.append(&mut add_tubes);
}
}
}
}
// It's possible more data is readable and buffered while the socket is hungup,
// so don't delete the tube from the poll context until we're sure all the
// data is read.
// Below case covers a condition where we have received a hungup event and the tube is not
// readable.
// In case of readable tube, once all data is read, any attempt to read more data on hungup
// tube should fail. On such failure, we get Disconnected error and index gets added to
// vm_control_indices_to_remove by the time we reach here.
for event in events.iter().filter(|e| e.is_hungup && !e.is_readable) {
if let Token::VmControl { index } = event.token {
vm_control_indices_to_remove.push(index);
}
}
// Sort in reverse so the highest indexes are removed first. This removal algorithm
// preserves correct indexes as each element is removed.
vm_control_indices_to_remove.sort_unstable_by_key(|&k| Reverse(k));
vm_control_indices_to_remove.dedup();
for index in vm_control_indices_to_remove {
// Delete the socket from the `wait_ctx` synchronously. Otherwise, the kernel will do
// this automatically when the FD inserted into the `wait_ctx` is closed after this
// if-block, but this removal can be deferred unpredictably. In some instances where the
// system is under heavy load, we can even get events returned by `wait_ctx` for an FD
// that has already been closed. Because the token associated with that spurious event
// now belongs to a different socket, the control loop will start to interact with
// sockets that might not be ready to use. This can cause incorrect hangup detection or
// blocking on a socket that will never be ready. See also: crbug.com/1019986
if let Some(socket) = control_tubes.get(index) {
wait_ctx
.delete(socket)
.context("failed to remove descriptor from wait context")?;
}
// This line implicitly drops the socket at `index` when it gets returned by
// `swap_remove`. After this line, the socket at `index` is not the one from
// `vm_control_indices_to_remove`. Because of this socket's change in index, we need to
// use `wait_ctx.modify` to change the associated index in its `Token::VmControl`.
control_tubes.swap_remove(index);
if let Some(tube) = control_tubes.get(index) {
wait_ctx
.modify(tube, EventType::Read, Token::VmControl { index })
.context("failed to add descriptor to wait context")?;
}
}
}
kick_all_vcpus(
&vcpu_handles,
linux.irq_chip.as_irq_chip(),
VcpuControl::RunState(VmRunMode::Exiting),
);
for (handle, _) in vcpu_handles {
if let Err(e) = handle.join() {
error!("failed to join vcpu thread: {:?}", e);
}
}
// Explicitly drop the VM structure here to allow the devices to clean up before the
// control sockets are closed when this function exits.
mem::drop(linux);
stdin()
.set_canon_mode()
.expect("failed to restore canonical mode for terminal");
Ok(exit_state)
}