From 8ad6754328f04f547a6ab779630c7bc630219d49 Mon Sep 17 00:00:00 2001 From: Joakim Hulthe Date: Sat, 26 Sep 2026 13:23:21 +0200 Subject: [PATCH] Add initial virtiofs support --- example/test.sh | 13 +-- init/src/main.rs | 20 ++--- kernel/config | 14 +-- src/main.rs | 1 + src/qemu.rs | 78 +++++++++++------ src/virtiofs.rs | 222 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 301 insertions(+), 47 deletions(-) mode change 100755 => 100644 example/test.sh create mode 100644 src/virtiofs.rs diff --git a/example/test.sh b/example/test.sh old mode 100755 new mode 100644 index 3da3b38..2940ebf --- a/example/test.sh +++ b/example/test.sh @@ -137,7 +137,7 @@ EOF } test_mount() { - echo "=== Testing --mount (9p shares) ===" + echo "=== Testing --mount (virtio-fs shares) ===" work="$(mktemp -d)" @@ -179,12 +179,10 @@ EOF output=$(timeout "$TIMEOUT" "$BOCO_BIN" boot "$img" \ --mount "${share_dir}:${guest_dir}" \ --mount "${share_dir2}:${guest_dir2}" \ - --cmd "for f in /sys/bus/virtio/drivers/9pnet_virtio/virtio*/mount_tag; do [ -f \"\$f\" ] && echo \"\$f: \$(tr -d '\\0' < \"\$f\")\"; done; mount -v; cat ${guest_dir}/testfile.txt 2>/dev/null || echo NO_FILE; cat ${guest_dir2}/testfile2.txt 2>/dev/null || echo NO_FILE2; echo MOUNT_VERIFY_DONE; poweroff -f" \ + --cmd "mount -v; cat ${guest_dir}/testfile.txt 2>/dev/null || echo NO_FILE; cat ${guest_dir2}/testfile2.txt 2>/dev/null || echo NO_FILE2; echo MOUNT_VERIFY_DONE; poweroff -f" \ 2>&1 || true) - check_output "$output" "mount_tag: boco0" "--mount 9p share detected" - check_output "$output" "mount_tag: boco1" "--mount second 9p share detected" - check_output "$output" "type 9p" "--mount 9p filesystem in mount list" + check_output "$output" "type virtiofs" "--mount virtiofs filesystem in mount list" check_output "$output" "hello from host" "--mount file accessible at guest path" check_output "$output" "second share" "--mount second file accessible at guest path" check_output "$output" "MOUNT_VERIFY_DONE" "--mount VM ran to completion" @@ -225,11 +223,14 @@ EOF --cmd "mount -v; cat /app/data/relfile.txt 2>/dev/null || echo NO_REL_FILE; echo REL_VERIFY_DONE; poweroff -f" \ 2>&1 || true) - check_output "$output" "type 9p" "--mount relative 9p filesystem in mount list" + check_output "$output" "type virtiofs" "--mount relative virtiofs filesystem in mount list" check_output "$output" "relative share" "--mount relative file accessible at WORKINGDIR/data" check_output "$output" "REL_VERIFY_DONE" "--mount relative VM ran to completion" cleanup "$img" "$img" + # Safety net: virtiofsd should exit when QEMU disconnects, but + # ensure no daemons linger if a boot failed mid-way. + pkill -f virtiofsd 2>/dev/null || true rm -rf "$work" } diff --git a/init/src/main.rs b/init/src/main.rs index c510a74..7ed746e 100644 --- a/init/src/main.rs +++ b/init/src/main.rs @@ -1,9 +1,9 @@ //! boco universal init. //! //! Injected by `boco build` at /boco/init and invoked via init=/boco/init. -//! Sets up devices, filesystems, networking, and 9p shares, then execs the -//! container's CMD/ENTRYPOINT (or a runtime override) with execvp so the -//! command becomes PID 1 directly. +//! Sets up devices, filesystems, networking, and virtio-fs shares, then +//! execs the container's CMD/ENTRYPOINT (or a runtime override) with +//! execvp so the command becomes PID 1 directly. use base64::Engine; use nix::mount::{MsFlags, mount}; @@ -208,7 +208,7 @@ fn setup_networking() { }); } -fn setup_9p_shares(workdir: &str) { +fn setup_virtiofs_shares(workdir: &str) { let cmdline = match fs::read_to_string("/proc/cmdline") { Ok(c) => c, Err(e) => { @@ -227,12 +227,12 @@ fn setup_9p_shares(workdir: &str) { Ok(bytes) => match String::from_utf8(bytes) { Ok(s) => s, Err(e) => { - warn(&format!("9p: invalid UTF-8 in guest path: {e}")); + warn(&format!("virtiofs: invalid UTF-8 in guest path: {e}")); continue; } }, Err(e) => { - warn(&format!("9p: base64 decode failed: {e}")); + warn(&format!("virtiofs: base64 decode failed: {e}")); continue; } }; @@ -248,9 +248,9 @@ fn setup_9p_shares(workdir: &str) { try_mount( Some(tag), &dest, - Some("9p"), + Some("virtiofs"), MsFlags::empty(), - Some("trans=virtio,version=9p2000.L"), + None, ); } } @@ -412,9 +412,9 @@ fn main() { // === Networking === setup_networking(); - // === 9p shares === + // === virtio-fs shares === let workdir = read_file_opt("/boco/workdir").unwrap_or_default(); - setup_9p_shares(&workdir); + setup_virtiofs_shares(&workdir); // === Environment & working directory === apply_env(); diff --git a/kernel/config b/kernel/config index 2070488..b35fca9 100644 --- a/kernel/config +++ b/kernel/config @@ -11,10 +11,9 @@ CONFIG_AS_VERSION=24700 CONFIG_LD_IS_BFD=y CONFIG_LD_VERSION=24700 CONFIG_LLD_VERSION=0 -CONFIG_RUSTC_VERSION=109701 -CONFIG_RUST_IS_AVAILABLE=y -CONFIG_RUSTC_LLVM_VERSION=220106 -CONFIG_RUSTC_LLVM_MAJOR_VERSION=22 +CONFIG_RUSTC_VERSION=109401 +CONFIG_RUSTC_LLVM_VERSION=210108 +CONFIG_RUSTC_LLVM_MAJOR_VERSION=21 CONFIG_CC_CAN_LINK=y CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT=y @@ -292,7 +291,6 @@ CONFIG_PERF_EVENTS=y # end of Kernel Performance Events And Counters CONFIG_PROFILING=y -# CONFIG_RUST is not set CONFIG_TRACEPOINTS=y # @@ -3204,7 +3202,11 @@ CONFIG_QUOTA_TREE=y CONFIG_QFMT_V2=y CONFIG_QUOTACTL=y CONFIG_AUTOFS_FS=y -# CONFIG_FUSE_FS is not set +CONFIG_FUSE_FS=y +CONFIG_CUSE=y +CONFIG_VIRTIO_FS=y +CONFIG_FUSE_PASSTHROUGH=y +CONFIG_FUSE_IO_URING=y CONFIG_OVERLAY_FS=y # CONFIG_OVERLAY_FS_REDIRECT_DIR is not set CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y diff --git a/src/main.rs b/src/main.rs index 7acf046..03ac237 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ mod image; mod inject; mod qemu; mod registry; +mod virtiofs; use anyhow::Result; use clap::{Parser, Subcommand}; diff --git a/src/qemu.rs b/src/qemu.rs index 7d2ab5e..a9dab63 100644 --- a/src/qemu.rs +++ b/src/qemu.rs @@ -3,7 +3,11 @@ //! at $XDG_DATA_HOME/boco/registry/vmlinuz. A universal /boco/init script (injected //! at build time) handles VM bootstrap; /boco/exec runs the container's //! CMD/ENTRYPOINT. Runtime overrides are delivered as `boco.cmd=` -//! on the kernel cmdline. +//! on the kernel cmdline. Host directories are shared into the guest via +//! virtio-fs: one virtiofsd daemon (under `podman unshare` for rootless +//! UID mapping) per `--mount`, plus a `-chardev socket` + `-device +//! vhost-user-fs-pci` pair on the QEMU side. The guest mounts each share +//! with `mount -t virtiofs ` (see `boco.mount=` cmdline). use anyhow::{Context, Result, bail}; use base64::Engine; @@ -13,6 +17,7 @@ use std::{fmt::Write as _, process::Command}; use crate::{ forward::PortForward, registry::{registry_base_dir, registry_dir}, + virtiofs::{self, VirtiofsShare}, }; #[derive(Args, Debug)] @@ -29,11 +34,11 @@ pub struct BootCmd { forward: Vec, /// Override the command to exec in the VM (base64-encoded on the kernel - /// cmdline as boco.cmd=). Overrides the CMD inferred at build time. + /// cmdline as `boco.cmd=`). Overrides the CMD inferred at build time. #[clap(long)] cmd: Option, - /// Share a host directory into the VM via 9p. Format: + /// Share a host directory into the VM via virtio-fs. Format: /// `` or `:`. When the guest path is /// omitted, the host path is used. Non-absolute guest paths are /// relative to the image's WORKINGDIR. Can be repeated. @@ -91,35 +96,55 @@ pub(crate) fn boot( cmdline.push(format!("boco.cmd={encoded}")); } - // Build 9p shares for each --mount. Tags are short (boco0, boco1, …) - // because 9p mount_tag has a ~31-byte limit. The guest destination path - // is passed on the kernel cmdline as boco.mount=:. - // If no guest path is specified, the host path is used as the guest path. - let mut virtfs_args: Vec = Vec::new(); + // Build virtio-fs shares for each `--mount`. Each share gets its own + // virtiofsd daemon under `podman unshare` (rootless UID mapping) and a + // matching `-chardev socket` + `-device vhost-user-fs-pci` pair on the + // QEMU side. The guest destination path is passed on the kernel cmdline + // as `boco.mount=:` and resolved by /boco/init. + let mut shares: Vec = Vec::with_capacity(mount.len()); + let mut fsdev_args: Vec = Vec::new(); for (i, spec) in mount.iter().enumerate() { let (host_path, guest_path) = match spec.split_once(':') { - Some((h, g)) => (h, g), - None => (spec.as_str(), spec.as_str()), + Some((h, g)) => (h, g.to_string()), + None => (spec.as_str(), spec.to_string()), }; - let canonical = std::path::Path::new(host_path) - .canonicalize() - .with_context(|| format!("Cannot resolve mount path '{host_path}'"))?; - let host_str = canonical + let share = virtiofs::spawn_share(i, host_path, guest_path)?; + let tag = share.tag.clone(); + let chardev = format!("char{i}"); + let socket = share + .socket_path() .to_str() - .context("Mount path is not valid UTF-8")?; - let tag = format!("boco{i}"); - let dest_b64 = base64::engine::general_purpose::STANDARD.encode(guest_path.as_bytes()); + .context("socket path is not UTF-8")? + .to_string(); + let dest_b64 = + base64::engine::general_purpose::STANDARD.encode(share.guest_path.as_bytes()); - // QEMU's QemuOpts splits on commas — escape literal commas in the - // path as ",," per QEMU convention. - let host_escaped = host_str.replace(',', ",,"); - virtfs_args.push("-virtfs".into()); - virtfs_args.push(format!( - "local,path={host_escaped},mount_tag={tag},security_model=mapped-xattr" + fsdev_args.push("-chardev".into()); + fsdev_args.push(format!("socket,id={chardev},path={socket}")); + fsdev_args.push("-device".into()); + fsdev_args.push(format!( + "vhost-user-fs-pci,queue-size=1024,chardev={chardev},tag={tag}" )); cmdline.push(format!("boco.mount={tag}:{dest_b64}")); + + shares.push(share); } + // vhost-user devices require shared guest RAM so virtiofsd (a separate + // process) can DMA into it. + let memory_args: Vec = if shares.is_empty() { + vec!["-m".into(), memory.clone()] + } else { + vec![ + "-m".into(), + memory.clone(), + "-object".into(), + format!("memory-backend-memfd,id=mem,size={memory},share=on"), + "-numa".into(), + "node,memdev=mem".into(), + ] + }; + println!("Booting {name} from registry: {}", reg_dir.display()); let cmdline = cmdline.join(" "); @@ -136,11 +161,11 @@ pub(crate) fn boot( command .args(["-accel", "kvm"]) .args(["-cpu", "host", "-smp", "cpus=8"]) - .args(["-m", &memory]) + .args(&memory_args) .arg("-kernel") .arg(vmlinuz_path.as_os_str()) .args(&fs_args) - .args(&virtfs_args) + .args(&fsdev_args) .args(["-snapshot"]) .args(["-no-reboot"]) .args(["-append", &cmdline]) @@ -153,6 +178,9 @@ pub(crate) fn boot( .status() .context("Failed to launch qemu-system-x86_64")?; + // Ensure virtiofsd daemons are torn down even on error returns below. + drop(shares); + match status.code() { Some(0) => println!("QEMU exited cleanly."), Some(126) | Some(127) => { diff --git a/src/virtiofs.rs b/src/virtiofs.rs new file mode 100644 index 0000000..a4ab416 --- /dev/null +++ b/src/virtiofs.rs @@ -0,0 +1,222 @@ +//! Virtiofs: spawn and manage one virtiofsd daemon per `--mount` share. +//! +//! virtiofsd is a vhost-user device backend that runs in a separate +//! process from QEMU and serves files from a host directory over the +//! FUSE protocol via a UNIX socket. boco spawns one daemon per share, +//! hands the socket path to QEMU via `-chardev socket,path=...`, and +//! cleans up when the VM exits. +//! +//! To serve files owned by arbitrary guest UIDs (e.g. when an image +//! uses the `USER` directive) without running as root, virtiofsd is +//! launched under `podman unshare` in a user namespace with +//! `--sandbox chroot`. This reuses rootless podman's existing +//! subuid/subgid mapping: guest UID 0 maps to the invoking user, and +//! other guest UIDs map into the subuid range, so each guest UID +//! becomes a distinct host UID. +//! +//! Lifecycle: boco spawns all virtiofsd children before launching QEMU, +//! waits for each listening socket to appear, then drops the +//! [`VirtiofsShare`] guard after QEMU returns — `Drop` sends SIGTERM, +//! waits briefly, escalates to SIGKILL, and `TempDir` removes the +//! socket files. On signals that kill boco directly (SIGINT/SIGTERM +//! without a handler), the guard does not run and the daemons linger; +//! callers like `example/test.sh` should `pkill virtiofsd` in cleanup. + +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail}; +use tempfile::TempDir; + +use crate::command::cmd; + +/// Locations searched for the virtiofsd binary, in order. +const VIRTIOFSD_CANDIDATES: &[&str] = &[ + "/usr/libexec/virtiofsd", + "/usr/lib/qemu/virtiofsd", + "/usr/lib/virtiofsd", +]; + +/// How long to wait for virtiofsd to create its listening socket. +const SOCKET_READY_TIMEOUT: Duration = Duration::from_secs(5); +const SOCKET_POLL_INTERVAL: Duration = Duration::from_millis(10); +/// Grace period after SIGTERM before escalating to SIGKILL. +const STOP_GRACE: Duration = Duration::from_secs(2); + +/// A running virtiofsd daemon plus the resources it owns. +/// +/// `Drop` stops the daemon and removes its socket. The `TempDir` +/// holding the socket is owned by the share so its path stays valid +/// for the lifetime of `socket`. +pub struct VirtiofsShare { + /// Tag advertised to the guest (e.g. `boco0`). + pub tag: String, + /// Guest-side mount destination, as supplied by the user (may be + /// relative to the image's WORKINGDIR; resolved by the guest init). + pub guest_path: String, + socket: PathBuf, + child: Option, + _dir: TempDir, +} + +/// Locate the virtiofsd binary. +/// +/// Searches well-known distro paths first, then falls back to `$PATH` +/// lookup via `command -v`. +pub fn find_virtiofsd() -> Result { + for path in VIRTIOFSD_CANDIDATES { + if Path::new(path).exists() { + return Ok(PathBuf::from(path)); + } + } + let which = cmd(&["sh", "-c", "command -v virtiofsd"])?; + let trimmed = which.trim(); + if trimmed.is_empty() { + bail!( + "virtiofsd not found. Install it (often shipped with qemu, or \ + build https://gitlab.com/virtio-fs/virtiofsd) and ensure it is \ + in PATH or at one of: {}", + VIRTIOFSD_CANDIDATES.join(", ") + ); + } + Ok(PathBuf::from(trimmed)) +} + +fn resolve_host_dir(host: &str) -> Result { + let path = Path::new(host) + .canonicalize() + .with_context(|| format!("Cannot resolve mount path '{host}'"))?; + if !path.is_dir() { + bail!("Mount source '{}' is not a directory", path.display()); + } + Ok(path) +} + +/// Wait for `socket` to appear as a filesystem entry. +fn wait_for_socket(socket: &Path) -> Result<()> { + let start = Instant::now(); + while start.elapsed() < SOCKET_READY_TIMEOUT { + if socket.exists() { + return Ok(()); + } + thread::sleep(SOCKET_POLL_INTERVAL); + } + bail!( + "virtiofsd did not create socket at {} within {:.1}s", + socket.display(), + SOCKET_READY_TIMEOUT.as_secs_f64() + ); +} + +/// Spawn one virtiofsd daemon for a share. +/// +/// `index` becomes the suffix of the tag (`boco{index}`) and is used to +/// derive a unique socket filename. +pub fn spawn_share(index: usize, host_path: &str, guest_path: String) -> Result { + let virtiofsd = find_virtiofsd()?; + let canonical = resolve_host_dir(host_path)?; + let host_str = canonical + .to_str() + .context("Mount source path is not valid UTF-8")?; + let virtiofsd_str = virtiofsd + .to_str() + .context("virtiofsd path is not valid UTF-8")?; + + let dir = TempDir::new()?; + let tag = format!("boco{index}"); // short tag to fit on cmdline + let socket = dir.path().join(format!("{tag}.sock")); + + // TODO: Make rootless mode optional + + // Run virtiofsd under `podman unshare` so it can serve files owned + // by arbitrary guest UIDs without root, mapping them into the host + // user's subuid range. `--sandbox chroot` is required because the + // outer `podman unshare` already entered a user namespace (nested + // user namespaces are not allowed); chroot still prevents symlink + // escapes from the shared dir. + let mut command = Command::new("podman"); + command + .args(["unshare", "--", virtiofsd_str, "--socket-path"]) + .arg(&socket) + .args([ + "--shared-dir", + host_str, + "--tag", + &tag, + "--sandbox", + "chroot", + "--cache", + "auto", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); + + println!("{command:?}"); + let child = command + .spawn() + .with_context(|| format!("Failed to spawn virtiofsd: {command:?}"))?; + + wait_for_socket(&socket) + .with_context(|| anyhow!("virtiofsd for share '{tag}' failed to become ready"))?; + + Ok(VirtiofsShare { + tag, + guest_path, + socket, + child: Some(child), + _dir: dir, + }) +} + +impl VirtiofsShare { + /// Socket path to pass to QEMU's `-chardev socket,path=...`. + pub fn socket_path(&self) -> &Path { + &self.socket + } + + /// Send SIGTERM, wait briefly, then SIGKILL if still alive. + fn stop(&mut self) { + let Some(child) = self.child.as_mut() else { + return; + }; + + // SIGTERM first for a graceful exit. + let pid = child.id(); + let _ = Command::new("kill") + .arg("-TERM") + .arg(pid.to_string()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + + let start = Instant::now(); + while start.elapsed() < STOP_GRACE { + match child.try_wait() { + Ok(Some(_)) => { + self.child = None; + return; + } + Ok(None) => thread::sleep(Duration::from_millis(50)), + Err(_) => break, + } + } + + // Escalate to SIGKILL. + if let Some(child) = self.child.as_mut() { + let _ = child.kill(); + let _ = child.wait(); + } + self.child = None; + } +} + +impl Drop for VirtiofsShare { + fn drop(&mut self) { + self.stop(); + // TempDir drop removes the socket file and its parent dir. + } +}