From 3b009b3225008a2585e384498947ca6ffe715a48 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 17 Aug 2026 22:23:23 +0200 Subject: [PATCH 1/2] feat(virtio-blk): add a block device driver with FAT support Add a virtio-blk driver (PCI and MMIO) and mount a FAT volume from the block device at /root via hadris-fat. - await requests and lock each virtqueue separately - read and write the device in runs of sectors - serialize file system operations on the volume - add fsync and synchronize the file system on shutdown I used AI to identify bootlenecks and obvious errors Assisted-by: Claude:claude-opus-5.5 coccinelle sparse --- Cargo.lock | 186 ++++- Cargo.toml | 4 + src/arch/aarch64/kernel/mmio.rs | 29 +- src/arch/riscv64/kernel/devicetree.rs | 7 + src/arch/riscv64/kernel/mmio.rs | 24 +- src/arch/x86_64/kernel/mmio.rs | 29 +- src/drivers/blk.rs | 475 +++++++++++ src/drivers/mmio.rs | 2 + src/drivers/mod.rs | 2 + src/drivers/pci.rs | 27 +- src/drivers/virtio/mod.rs | 7 + src/drivers/virtio/transport/mmio.rs | 15 + src/drivers/virtio/transport/pci.rs | 23 + src/fd/delegate.rs | 14 + src/fs/mod.rs | 13 + src/fs/vfat.rs | 1037 +++++++++++++++++++++++++ src/init_cell.rs | 3 +- src/syscalls/mod.rs | 5 + 18 files changed, 1858 insertions(+), 44 deletions(-) create mode 100644 src/drivers/blk.rs create mode 100644 src/fs/vfat.rs diff --git a/Cargo.lock b/Cargo.lock index 76fc46be78..ea0e62c42e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,9 +173,9 @@ dependencies = [ [[package]] name = "arm-sysregs" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eea2d77c3eac6be0a300ba179e7bf051ffaec797c39dee0b631c21668b12e1f3" +checksum = "4ade7ce6337d4740899df84854c60a5c104175f0e35978f131724697d9ba3a6d" dependencies = [ "arm-sysregs-aarch32", "arm-sysregs-common", @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "arm-sysregs-aarch32" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeac85b36432f9427827c31c2da2b42fe4928e35b3dbedbe9c9ded5e4d013789" +checksum = "f574312ff652ad9bccdf3c0885a4fadfbfb2b4ca26aefafa91736a9fc1e1e97c" dependencies = [ "arm-sysregs-common", "bitflags 2.13.2", @@ -195,19 +195,20 @@ dependencies = [ [[package]] name = "arm-sysregs-common" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11447113c834b1cb206262297477a467c090a3768765bef861858b6098dcac39" +checksum = "362d54f215a3d7413160b2e36bf7972b7990831bc9ac45b76776866d4d99d9bd" dependencies = [ + "bitflags 2.13.2", "num_enum", "pastey", ] [[package]] name = "arm-sysregs-el0" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a680f4741d246b3f72f977da72f7df76cb9c3227e565acad801ca3c504947d92" +checksum = "1e73ff608bf096287e4c59c13686b47d76177a76315391edd144cebd3a7efd38" dependencies = [ "arm-sysregs-aarch32", "arm-sysregs-common", @@ -216,9 +217,9 @@ dependencies = [ [[package]] name = "arm-sysregs-el1" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dcc832aff5e63bad65c4cdcb17a7a5c07a711a4ce365d7a69785efb3a64fc47" +checksum = "140996601708ec9c1841c526a4de934394bb489c3345913cdee90f4ab0212bf6" dependencies = [ "arm-sysregs-common", "arm-sysregs-el0", @@ -247,9 +248,9 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.4.2" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" dependencies = [ "event-listener", "event-listener-strategy", @@ -358,6 +359,26 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -411,9 +432,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" dependencies = [ "find-msvc-tools", "jobserver", @@ -549,9 +570,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" dependencies = [ "cfg-if", ] @@ -663,9 +684,9 @@ checksum = "55dd888a213fc57e957abf2aa305ee3e8a28dbe05687a251f33b637cd46b0070" [[package]] name = "elf-symbols" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c04d6bdc16d03d4e204d6cd018db65f0947013bb4129d1f9a935ff9562af9e" +checksum = "555cb7dce915d70854d42a9cd4c1adbbb90334c7d948bca43e9aa34fd3687ad0" [[package]] name = "embedded-hal" @@ -679,6 +700,15 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eb1aa714776b75c7e67e1da744b81a129b3ff919c8712b5e1b32252c1f07cc7" +[[package]] +name = "embedded-io-async" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2564b9f813c544241430e147d8bc454815ef9ac998878d30cc3055449f7fd4c0" +dependencies = [ + "embedded-io", +] + [[package]] name = "endian-num" version = "0.2.0" @@ -909,6 +939,69 @@ dependencies = [ "scroll", ] +[[package]] +name = "hadris-common" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0848bdca19cf3cdced59392757a2dfcdc4165cb2ca8751f9ea65953e84e7d9ec" +dependencies = [ + "hadris-fixed", + "hadris-io", + "hadris-path", + "heapless", + "zerocopy", +] + +[[package]] +name = "hadris-fat" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2663ed402fcc8d12126b3ef5ee83e81b8610a86771f3ce0e6fa4f58243e08247" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "chrono", + "hadris-common", + "hadris-fixed", + "hadris-io", + "hadris-macros", + "hadris-path", + "parse-size", + "spin", +] + +[[package]] +name = "hadris-fixed" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9113caea96eb0e12db9045d69e8baefe6677384b2d21b7adff491819bdd0f8f" + +[[package]] +name = "hadris-io" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16435cbe44eb91d440d8c6c85086c265ef51d8e3d5f95db5ef5f9aa82bb044e4" +dependencies = [ + "bytemuck", + "embedded-io", + "embedded-io-async", +] + +[[package]] +name = "hadris-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d6d0b20f4982244c26a3d796d8479d7f89f2b562d70d4df3c73d67eb1070d0" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "hadris-path" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0179c8acf85bf76fb3134c67e41dc53420457f02331f520172bbeb6ac1ab19b5" + [[package]] name = "hash32" version = "0.3.1" @@ -996,6 +1089,7 @@ dependencies = [ "free-list", "fuse-abi", "goblin", + "hadris-fat", "hashbrown", "heapless", "hermit-entry", @@ -1157,9 +1251,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.106" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" dependencies = [ "cfg-if", "futures-util", @@ -1225,9 +1319,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" [[package]] name = "lzma-rs" @@ -1469,6 +1563,12 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parse-size" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" + [[package]] name = "pastey" version = "0.2.3" @@ -1938,11 +2038,20 @@ dependencies = [ "managed", ] +[[package]] +name = "spin" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10" +dependencies = [ + "lock_api", +] + [[package]] name = "spinning_top" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +checksum = "9cf02766a1eb909bc7653557c66b384e44edc3ad979e6257c6173c65ea51bc7d" dependencies = [ "lock_api", ] @@ -2177,9 +2286,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.5.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" +checksum = "e7be76686bdf7198002062423ca0d8cf9b75a27c8e1de804513b97e97b555d9d" dependencies = [ "base64", "log", @@ -2193,9 +2302,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" +checksum = "622dce33c252115466d83f63e9744a18f12b4406aae51a01c15ad8fcf1185eb3" dependencies = [ "base64", "http", @@ -2277,9 +2386,9 @@ dependencies = [ [[package]] name = "vsock" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba782755fc073877e567c2253c0be48e4aa9a254c232d36d3985dfae0bd5205" +checksum = "9b7d4dba01e366e2ffbdf0700b1876f747c548db72dced534de41c54945c0148" dependencies = [ "libc", "nix", @@ -2312,12 +2421,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-bindgen" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" dependencies = [ "cfg-if", - "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", @@ -2325,9 +2433,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2335,9 +2443,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" dependencies = [ "bumpalo", "proc-macro2", @@ -2348,9 +2456,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" dependencies = [ "unicode-ident", ] diff --git a/Cargo.toml b/Cargo.toml index 2e05bb3bb1..60b0fd536e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -218,6 +218,9 @@ gem-net = ["net", "dep:tock-registers"] #! ### Virtio Drivers +## Enables the virtio-blk driver. +virtio-blk = ["virtio", "hadris-fat"] + ## Enables the virtio-console driver. virtio-console = ["virtio"] @@ -379,6 +382,7 @@ env_filter = { version = "2.0.0", default-features = false } free-list = "0.3" fuse-abi = { version = "0.2", optional = true, features = ["linux"] } goblin = { version = "0.10", optional = true, default-features = false, features = ["elf64"] } +hadris-fat = { version = "2.5.0", default-features = false, features = ["read", "write", "alloc", "lfn", "async"], optional = true } hashbrown = { version = "0.17", default-features = false } heapless = "0.9" hermit-entry = { version = "0.10", features = ["kernel"], optional = true } diff --git a/src/arch/aarch64/kernel/mmio.rs b/src/arch/aarch64/kernel/mmio.rs index 983960743e..2b2c8faf07 100644 --- a/src/arch/aarch64/kernel/mmio.rs +++ b/src/arch/aarch64/kernel/mmio.rs @@ -6,7 +6,7 @@ use arm_gic::{IntId, Trigger}; #[cfg(any( feature = "virtio-fs", feature = "virtio-rng", - feature = "virtio-vsock", + feature = "virtio-vsock" ))] use hermit_sync::InterruptTicketMutex; use hermit_sync::without_interrupts; @@ -16,6 +16,8 @@ use volatile::VolatileRef; use crate::arch::kernel::interrupts::GIC; use crate::arch::mm::paging::{self, PageSize}; use crate::drivers::InterruptHandlerMap; +#[cfg(feature = "virtio-blk")] +use crate::drivers::blk::VirtioBlkDriver; #[cfg(feature = "virtio-fs")] use crate::drivers::fs::VirtioFsDriver; #[cfg(feature = "virtio-net")] @@ -24,6 +26,7 @@ use crate::drivers::net::virtio::VirtioNetDriver; use crate::drivers::rng::VirtioRngDriver; use crate::drivers::virtio::transport::mmio as mmio_virtio; #[cfg(any( + feature = "virtio-blk", feature = "virtio-console", feature = "virtio-fs", feature = "virtio-net", @@ -44,6 +47,8 @@ pub(crate) static MMIO_DRIVERS: InitCell> = InitCell::new(Vec::n #[allow(clippy::enum_variant_names, clippy::large_enum_variant)] #[non_exhaustive] pub(crate) enum MmioDriver { + #[cfg(feature = "virtio-blk")] + VirtioBlk(VirtioBlkDriver), #[cfg(feature = "virtio-fs")] VirtioFs(InterruptTicketMutex), #[cfg(feature = "virtio-rng")] @@ -53,6 +58,15 @@ pub(crate) enum MmioDriver { } impl MmioDriver { + #[cfg(feature = "virtio-blk")] + fn get_block_driver(&self) -> Option<&VirtioBlkDriver> { + #[allow(unreachable_patterns)] + match self { + Self::VirtioBlk(drv) => Some(drv), + _ => None, + } + } + #[cfg(feature = "virtio-fs")] fn get_filesystem_driver(&self) -> Option<&InterruptTicketMutex> { #[allow(unreachable_patterns)] @@ -82,6 +96,7 @@ impl MmioDriver { } #[cfg(any( + feature = "virtio-blk", feature = "virtio-fs", feature = "virtio-rng", feature = "virtio-vsock" @@ -101,6 +116,14 @@ pub(crate) fn get_filesystem_driver() -> Option<&'static InterruptTicketMutex Option<&'static VirtioBlkDriver> { + MMIO_DRIVERS + .get()? + .iter() + .find_map(|drv| drv.get_block_driver()) +} + #[cfg(feature = "virtio-rng")] pub(crate) fn get_rng_driver() -> Option<&'static InterruptTicketMutex> { MMIO_DRIVERS @@ -230,6 +253,10 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { match drv { #[cfg(feature = "virtio-console")] VirtioDriver::Console(drv) => crate::console::switch_to_virtio(*drv), + #[cfg(feature = "virtio-blk")] + VirtioDriver::Blk(drv) => { + register_driver(MmioDriver::VirtioBlk(*drv)); + } #[cfg(feature = "virtio-fs")] VirtioDriver::Fs(drv) => { register_driver(MmioDriver::VirtioFs(InterruptTicketMutex::new(*drv))); diff --git a/src/arch/riscv64/kernel/devicetree.rs b/src/arch/riscv64/kernel/devicetree.rs index 27cae557b6..f8affbf5de 100644 --- a/src/arch/riscv64/kernel/devicetree.rs +++ b/src/arch/riscv64/kernel/devicetree.rs @@ -12,6 +12,7 @@ use volatile::VolatileRef; use crate::arch::kernel::interrupts::init_plic; #[cfg(all( any( + feature = "virtio-blk", feature = "virtio-fs", feature = "virtio-rng", feature = "virtio-vsock", @@ -21,6 +22,7 @@ use crate::arch::kernel::interrupts::init_plic; use crate::arch::kernel::mmio::MmioDriver; #[cfg(all( any( + feature = "virtio-blk", feature = "virtio-fs", feature = "virtio-rng", feature = "virtio-vsock", @@ -36,6 +38,7 @@ use crate::drivers::net::gem; use crate::drivers::virtio::transport::mmio as mmio_virtio; #[cfg(all( any( + feature = "virtio-blk", feature = "virtio-console", feature = "virtio-fs", feature = "virtio-net", @@ -238,6 +241,10 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { hermit_sync::InterruptSpinMutex::new(*drv), )); } + #[cfg(feature = "virtio-blk")] + Ok(VirtioDriver::Blk(drv)) => { + register_driver(MmioDriver::VirtioBlk(*drv)); + } Err(DriverError::InitVirtioDevFail(VirtioError::DevNotSupported(0))) => (), Err(err) => error!("Could not initialize virtio-mmio device: {err}"), } diff --git a/src/arch/riscv64/kernel/mmio.rs b/src/arch/riscv64/kernel/mmio.rs index d7789ec84b..f3c9dfd8a2 100644 --- a/src/arch/riscv64/kernel/mmio.rs +++ b/src/arch/riscv64/kernel/mmio.rs @@ -3,10 +3,12 @@ use alloc::vec::Vec; #[cfg(any( feature = "virtio-fs", feature = "virtio-rng", - feature = "virtio-vsock", + feature = "virtio-vsock" ))] use hermit_sync::InterruptSpinMutex; +#[cfg(feature = "virtio-blk")] +use crate::drivers::blk::VirtioBlkDriver; #[cfg(feature = "virtio-fs")] use crate::drivers::fs::VirtioFsDriver; #[cfg(feature = "gem-net")] @@ -24,6 +26,8 @@ pub(crate) static MMIO_DRIVERS: InitCell> = InitCell::new(Vec::n #[allow(clippy::enum_variant_names, clippy::large_enum_variant)] #[non_exhaustive] pub(crate) enum MmioDriver { + #[cfg(feature = "virtio-blk")] + VirtioBlk(VirtioBlkDriver), #[cfg(feature = "virtio-fs")] VirtioFs(InterruptSpinMutex), #[cfg(feature = "virtio-rng")] @@ -33,6 +37,15 @@ pub(crate) enum MmioDriver { } impl MmioDriver { + #[cfg(feature = "virtio-blk")] + fn get_block_driver(&self) -> Option<&VirtioBlkDriver> { + #[allow(unreachable_patterns)] + match self { + Self::VirtioBlk(drv) => Some(drv), + _ => None, + } + } + #[cfg(feature = "virtio-fs")] fn get_filesystem_driver(&self) -> Option<&InterruptSpinMutex> { #[allow(unreachable_patterns)] @@ -62,6 +75,7 @@ impl MmioDriver { } #[cfg(any( + feature = "virtio-blk", feature = "virtio-fs", feature = "virtio-rng", feature = "virtio-vsock", @@ -84,6 +98,14 @@ pub(crate) fn get_filesystem_driver() -> Option<&'static InterruptSpinMutex Option<&'static VirtioBlkDriver> { + MMIO_DRIVERS + .get()? + .iter() + .find_map(|drv| drv.get_block_driver()) +} + #[cfg(feature = "virtio-rng")] pub(crate) fn get_rng_driver() -> Option<&'static InterruptSpinMutex> { MMIO_DRIVERS diff --git a/src/arch/x86_64/kernel/mmio.rs b/src/arch/x86_64/kernel/mmio.rs index ef95b11422..fb501e10f1 100644 --- a/src/arch/x86_64/kernel/mmio.rs +++ b/src/arch/x86_64/kernel/mmio.rs @@ -8,7 +8,7 @@ use free_list::PageRange; #[cfg(any( feature = "virtio-fs", feature = "virtio-rng", - feature = "virtio-vsock", + feature = "virtio-vsock" ))] use hermit_sync::InterruptTicketMutex; use hermit_sync::without_interrupts; @@ -18,6 +18,8 @@ use volatile::VolatileRef; use crate::arch::mm::paging; use crate::arch::mm::paging::{BasePageSize, PageSize}; use crate::drivers::InterruptHandlerMap; +#[cfg(feature = "virtio-blk")] +use crate::drivers::blk::VirtioBlkDriver; #[cfg(feature = "virtio-fs")] use crate::drivers::fs::VirtioFsDriver; #[cfg(feature = "virtio-net")] @@ -26,6 +28,7 @@ use crate::drivers::net::virtio::VirtioNetDriver; use crate::drivers::rng::VirtioRngDriver; use crate::drivers::virtio::transport::mmio as mmio_virtio; #[cfg(any( + feature = "virtio-blk", feature = "virtio-console", feature = "virtio-fs", feature = "virtio-net", @@ -46,6 +49,8 @@ static MMIO_DRIVERS: InitCell> = InitCell::new(Vec::new()); #[allow(clippy::enum_variant_names, clippy::large_enum_variant)] #[non_exhaustive] pub(crate) enum MmioDriver { + #[cfg(feature = "virtio-blk")] + VirtioBlk(VirtioBlkDriver), #[cfg(feature = "virtio-fs")] VirtioFs(InterruptTicketMutex), #[cfg(feature = "virtio-rng")] @@ -55,6 +60,15 @@ pub(crate) enum MmioDriver { } impl MmioDriver { + #[cfg(feature = "virtio-blk")] + fn get_block_driver(&self) -> Option<&VirtioBlkDriver> { + #[allow(unreachable_patterns)] + match self { + Self::VirtioBlk(drv) => Some(drv), + _ => None, + } + } + #[cfg(feature = "virtio-fs")] fn get_filesystem_driver(&self) -> Option<&InterruptTicketMutex> { #[allow(unreachable_patterns)] @@ -165,6 +179,7 @@ fn guess_device() -> impl Iterator } #[cfg(any( + feature = "virtio-blk", feature = "virtio-fs", feature = "virtio-rng", feature = "virtio-vsock", @@ -184,6 +199,14 @@ pub(crate) fn get_filesystem_driver() -> Option<&'static InterruptTicketMutex Option<&'static VirtioBlkDriver> { + MMIO_DRIVERS + .get()? + .iter() + .find_map(|drv| drv.get_block_driver()) +} + #[cfg(feature = "virtio-rng")] pub(crate) fn get_rng_driver() -> Option<&'static InterruptTicketMutex> { MMIO_DRIVERS @@ -210,6 +233,10 @@ fn register_mmio( Ok(VirtioDriver::Console(drv)) => { crate::console::switch_to_virtio(*drv); } + #[cfg(feature = "virtio-blk")] + Ok(VirtioDriver::Blk(drv)) => { + register_driver(MmioDriver::VirtioBlk(*drv)); + } #[cfg(feature = "virtio-fs")] Ok(VirtioDriver::Fs(drv)) => { register_driver(MmioDriver::VirtioFs(InterruptTicketMutex::new(*drv))); diff --git a/src/drivers/blk.rs b/src/drivers/blk.rs new file mode 100644 index 0000000000..f18aadc70e --- /dev/null +++ b/src/drivers/blk.rs @@ -0,0 +1,475 @@ +//! A virtio-blk driver. +//! +//! For details on the device, see [Block Device]. +//! +//! [Block Device]: https://docs.oasis-open.org/virtio/virtio/v1.4/cs01/virtio-v1.4-cs01.html#x1-3120002 + +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::task::Poll; + +use hermit_sync::InterruptTicketMutex; +use smallvec::SmallVec; +use virtio::blk::{ConfigVolatileFieldAccess, S, T}; +use virtio::{le32, le64}; +use volatile::VolatileRef; +use volatile::access::ReadOnly; + +use crate::arch::kernel::core_local::core_id; +use crate::config::VIRTIO_MAX_QUEUE_SIZE; +#[cfg(not(feature = "pci"))] +use crate::drivers::mmio::get_block_driver; +#[cfg(feature = "pci")] +use crate::drivers::pci::get_block_driver; +use crate::drivers::virtio::error::VirtioError; +use crate::drivers::virtio::transport::{InterruptCapability, UniCapsColl}; +use crate::drivers::virtio::virtqueue::error::VirtqError; +use crate::drivers::virtio::virtqueue::split::SplitVq; +use crate::drivers::virtio::virtqueue::{ + AvailBufferToken, BufferElem, BufferType, VirtQueue, Virtq, +}; +use crate::drivers::{Driver, InterruptHandlerMap, InterruptLine}; +use crate::errno::Errno; +use crate::mm::device_alloc::DeviceAlloc; + +/// The block device driver, if the system has one. +pub(crate) fn driver() -> Option<&'static VirtioBlkDriver> { + get_block_driver() +} + +/// Runs `f` on the block device driver, if the system has one. +pub(crate) fn with_driver(f: impl FnOnce(&VirtioBlkDriver) -> T) -> Option { + get_block_driver().map(f) +} + +/// Addressing unit of the device, fixed by the specification. +pub(crate) const SECTOR_SIZE: usize = 512; + +/// The fixed-size header that starts every request. +#[doc(alias = "virtio_blk_req")] +#[repr(C)] +struct RequestHeader { + type_: le32, + reserved: le32, + sector: le64, +} + +impl RequestHeader { + fn new(type_: T, sector: u64) -> Self { + Self { + type_: le32::from_ne(type_.into()), + reserved: le32::from_ne(0), + sector: le64::from_ne(sector), + } + } +} + +/// The number of cores that may ever issue a request. +/// +/// `get_processor_count` would return 1 during driver initialization. +fn possible_cores() -> usize { + #[cfg(feature = "smp")] + let count = crate::arch::kernel::get_possible_cpus(); + #[cfg(not(feature = "smp"))] + let count = 1; + + usize::try_from(count).unwrap() +} + +/// Wraps a single buffer element in the [`SmallVec`] the virtqueue expects. +fn single(elem: BufferElem) -> SmallVec<[BufferElem; 2]> { + let mut vec = SmallVec::new(); + vec.push(elem); + vec +} + +/// Driver for a virtio block device. +/// +/// Requests are awaited by polling the used ring, device notifications stay +/// disabled. +pub(crate) struct VirtioBlkDriver { + /// Request queues, one per core at most. The lock is held while waiting + /// for the device, so it is asynchronous. + vqs: Vec>, + /// Used by `Self::handle_interrupt` only after initialization. + caps: InterruptTicketMutex, + /// Negotiated features. + features: virtio::blk::F, + capacity: usize, + blk_size: u32, + read_only: bool, +} + +impl VirtioBlkDriver { + /// The capacity of the device in `SECTOR_SIZE`-byte sectors. + pub fn capacity(&self) -> usize { + self.capacity + } + + /// The optimal I/O size in bytes, or `SECTOR_SIZE` if the device does not + /// report one. + #[allow(dead_code)] + pub fn block_size(&self) -> u32 { + self.blk_size + } + + /// Whether the device refuses writes. + #[allow(dead_code)] + pub fn is_read_only(&self) -> bool { + self.read_only + } + + /// Reads `buf.len() / SECTOR_SIZE` sectors starting at `sector`. + pub async fn read(&self, sector: usize, buf: &mut [u8]) -> Result<(), Errno> { + let len = self.checked_len(sector, buf.len())?; + + let send = single(BufferElem::Sized(Box::new_in( + RequestHeader::new(T::In, sector.try_into().unwrap()), + DeviceAlloc, + ))); + let recv = SmallVec::from_buf([ + BufferElem::Vector(Vec::with_capacity_in(len, DeviceAlloc)), + BufferElem::Sized(Box::::new_uninit_in(DeviceAlloc)), + ]); + + let mut used = self.dispatch(send, recv).await?; + + let data = used.used_recv_buff.pop_front_vec().ok_or(Errno::Io)?; + Self::check_status(&mut used)?; + + if data.len() != len { + return Err(Errno::Io); + } + buf.copy_from_slice(&data); + + Ok(()) + } + + /// Starts a set of requests that stay in flight together. + /// + /// The queue stays locked until the batch is finished. + pub async fn batch(&self) -> Batch<'_> { + Batch { + driver: self, + vq: self.vq().lock().await, + outstanding: 0, + } + } + + /// Asks the device to write out its cache. + /// + /// Without `virtio::blk::F::FLUSH` the device has no volatile cache, so + /// this is a no-op. + pub async fn flush(&self) -> Result<(), Errno> { + if !self.features.contains(virtio::blk::F::FLUSH) { + return Ok(()); + } + + let send = single(BufferElem::Sized(Box::new_in( + RequestHeader::new(T::Flush, 0), + DeviceAlloc, + ))); + let recv = single(BufferElem::Sized(Box::::new_uninit_in(DeviceAlloc))); + + let mut used = self.dispatch(send, recv).await?; + Self::check_status(&mut used) + } + + /// Validates a transfer against the sector size and the device capacity and + /// returns its length in bytes. + fn checked_len(&self, sector: usize, len: usize) -> Result { + if len == 0 || !len.is_multiple_of(SECTOR_SIZE) { + return Err(Errno::Inval); + } + + let sectors = len / SECTOR_SIZE; + if sector + .checked_add(sectors) + .is_none_or(|end| end > self.capacity) + { + return Err(Errno::Inval); + } + + Ok(len) + } + + /// Picks the virtqueue of the current core. + fn vq(&self) -> &async_lock::Mutex { + let index = usize::try_from(core_id()).unwrap() % self.vqs.len(); + + &self.vqs[index] + } + + /// Sends a request and waits for the device to complete it. + async fn dispatch( + &self, + send: SmallVec<[BufferElem; 2]>, + recv: SmallVec<[BufferElem; 2]>, + ) -> Result { + let tkn = AvailBufferToken::new(send, recv).map_err(|_| Errno::Io)?; + let mut vq = self.vq().lock().await; + + vq.dispatch(tkn, false, BufferType::Direct) + .map_err(|_| Errno::Io)?; + + core::future::poll_fn(|cx| match vq.try_recv() { + Ok(used) => Poll::Ready(Ok(used)), + Err(VirtqError::NoNewUsed) => { + // Yield to other tasks instead of spinning. + cx.waker().wake_by_ref(); + Poll::Pending + } + Err(_) => Poll::Ready(Err(Errno::Io)), + }) + .await + } + + /// Acknowledges a device interrupt by reading the ISR status register. + pub fn handle_interrupt(&self) { + let mut caps = self.caps.lock(); + + #[cfg_attr( + not(all(feature = "pci", target_arch = "x86_64")), + expect(irrefutable_let_patterns) + )] + let InterruptCapability::IsrStatus(isr_stat) = &mut caps.int_cap else { + panic!("MSI-X vectors should be configured to the interrupt type-specific handlers.") + }; + + let _ = isr_stat.acknowledge(); + } + + /// Pops the trailing status byte and maps it onto an `Errno`. + fn check_status( + used: &mut crate::drivers::virtio::virtqueue::UsedBufferToken, + ) -> Result<(), Errno> { + // SAFETY: every request ends with a `Box` for the status. It is + // read as `u8` rather than `S`, because the device may write any value. + let status = unsafe { used.used_recv_buff.pop_front_downcast::() }; + let Some(status) = status.as_deref().copied() else { + return Err(Errno::Io); + }; + + if status == u8::from(S::Ok) { + Ok(()) + } else if status == u8::from(S::Unsupp) { + Err(Errno::Nosys) + } else { + Err(Errno::Io) + } + } +} + +/// Requests submitted together and awaited in one go. +/// +/// Must be finished even after a failed `Batch::write`, otherwise its +/// completions are left in the queue. +pub(crate) struct Batch<'a> { + driver: &'a VirtioBlkDriver, + vq: async_lock::MutexGuard<'a, VirtQueue>, + outstanding: usize, +} + +impl Batch<'_> { + /// Queues a write without waiting for it. + pub fn write(&mut self, sector: usize, buf: &[u8]) -> Result<(), Errno> { + self.driver.checked_len(sector, buf.len())?; + + if self.driver.read_only { + return Err(Errno::Rofs); + } + + let mut data = Vec::with_capacity_in(buf.len(), DeviceAlloc); + data.extend_from_slice(buf); + + let send = SmallVec::from_buf([ + BufferElem::Sized(Box::new_in( + RequestHeader::new(T::Out, sector.try_into().unwrap()), + DeviceAlloc, + )), + BufferElem::Vector(data), + ]); + let recv = single(BufferElem::Sized(Box::::new_uninit_in(DeviceAlloc))); + + let tkn = AvailBufferToken::new(send, recv).map_err(|_| Errno::Io)?; + self.vq + .dispatch(tkn, false, BufferType::Direct) + .map_err(|_| Errno::Io)?; + self.outstanding += 1; + + Ok(()) + } + + /// Waits for every queued request and reports the first failure. + pub async fn finish(mut self) -> Result<(), Errno> { + let mut result = Ok(()); + + while self.outstanding > 0 { + let vq = &mut self.vq; + let used = core::future::poll_fn(|cx| match vq.try_recv() { + Ok(used) => Poll::Ready(Ok(used)), + Err(VirtqError::NoNewUsed) => { + cx.waker().wake_by_ref(); + Poll::Pending + } + Err(_) => Poll::Ready(Err(Errno::Io)), + }) + .await; + + self.outstanding -= 1; + + let status = match used { + Ok(mut used) => VirtioBlkDriver::check_status(&mut used), + Err(err) => return Err(err), + }; + + if let Err(err) = status + && result.is_ok() + { + result = Err(err); + } + } + + result + } +} + +impl Driver for VirtioBlkDriver { + fn get_name() -> &'static str { + "virtio-blk" + } +} + +impl super::virtio::VirtioDriver for VirtioBlkDriver { + type Config = virtio::blk::Config; + type Error = error::VirtioBlkError; + type DeviceFeatures = virtio::blk::F; + + const MINIMAL_FEATURES: Self::DeviceFeatures = virtio::blk::F::VERSION_1; + const OPTIONAL_FEATURES: Self::DeviceFeatures = virtio::blk::F::SIZE_MAX + .union(virtio::blk::F::SEG_MAX) + .union(virtio::blk::F::RO) + .union(virtio::blk::F::BLK_SIZE) + .union(virtio::blk::F::FLUSH) + .union(virtio::blk::F::MQ); + + fn init_dev( + (mut caps_coll, dev_cfg_raw): (UniCapsColl, VolatileRef<'static, Self::Config, ReadOnly>), + handlers: &mut InterruptHandlerMap, + irq: Option, + ) -> Result { + let mut queues = Vec::new(); + + let dev_cfg = match caps_coll.init_caps::(dev_cfg_raw, |caps_coll, dev_cfg| { + // `num_queues` is only valid with MQ. + let features: virtio::blk::F = dev_cfg.features; + let offered = if features.contains(virtio::blk::F::MQ) { + dev_cfg.raw.as_ptr().num_queues().read() + } else { + 1 + }; + + // More queues than cores would stay idle. + let count = usize::from(offered.max(1)).min(possible_cores()); + + for index in 0..count { + queues.push(VirtQueue::Split( + SplitVq::new( + &mut caps_coll.com_cfg, + &caps_coll.notif_cfg, + VIRTIO_MAX_QUEUE_SIZE, + u16::try_from(index).unwrap(), + virtio::F::from(dev_cfg.features), + ) + .unwrap(), + )); + } + + // The ISR status has to be acknowledged on every interrupt. + match &mut caps_coll.int_cap { + InterruptCapability::IsrStatus(_) => { + let irq = irq.unwrap(); + handlers.entry(irq).or_default().push_back(|| { + if let Some(driver) = get_block_driver() { + driver.handle_interrupt(); + }; + }); + crate::arch::kernel::interrupts::add_irq_name(irq, "virtio"); + info!("Virtio interrupt handler at line {irq}"); + } + #[cfg(all(feature = "pci", target_arch = "x86_64"))] + InterruptCapability::Msix(msix_table) => { + use core::iter; + + caps_coll.com_cfg.register_msix_vectors( + msix_table, + handlers, + || { + if let Some(driver) = get_block_driver() { + driver.handle_interrupt(); + }; + }, + iter::empty::<(iter::Empty<_>, _)>(), + 0..u16::try_from(queues.len()).unwrap(), + ); + } + } + + Ok(()) + }) { + Ok(dev_cfg) => dev_cfg, + Err(err) => return Err((err, caps_coll)), + }; + + for vq in &mut queues { + vq.disable_notifs(); + } + + let cfg = dev_cfg.raw.as_ptr(); + let features: virtio::blk::F = dev_cfg.features; + + let capacity = cfg.capacity().read().to_ne(); + let blk_size = if features.contains(virtio::blk::F::BLK_SIZE) { + cfg.blk_size().read().to_ne() + } else { + u32::try_from(SECTOR_SIZE).unwrap() + }; + let read_only = features.contains(virtio::blk::F::RO); + let vqs: Vec<_> = queues.into_iter().map(async_lock::Mutex::new).collect(); + + info!( + "virtio-blk: {capacity} sectors ({} MiB), block size {blk_size}, {} request queue(s){}", + capacity * u64::try_from(SECTOR_SIZE).unwrap() / (1024 * 1024), + vqs.len(), + if read_only { ", read-only" } else { "" } + ); + + Ok(Self { + vqs, + caps: InterruptTicketMutex::new(caps_coll), + features, + capacity: capacity.try_into().unwrap(), + blk_size, + read_only, + }) + } + + #[cfg(feature = "pci")] + fn no_dev_cfg_err(dev_id: u16) -> Self::Error { + error::VirtioBlkError::NoDevCfg(dev_id) + } +} + +/// Error module of the virtio block driver. +pub mod error { + use thiserror::Error; + + #[derive(Error, Debug, Copy, Clone)] + pub enum VirtioBlkError { + #[cfg(feature = "pci")] + #[error( + "Virtio block device driver failed, for device {0:x}, due to a missing or malformed device config!" + )] + NoDevCfg(u16), + } +} diff --git a/src/drivers/mmio.rs b/src/drivers/mmio.rs index 61ac23065f..8c2701a834 100644 --- a/src/drivers/mmio.rs +++ b/src/drivers/mmio.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "virtio-blk")] +pub(crate) use crate::arch::kernel::mmio::get_block_driver; #[cfg(feature = "virtio-fs")] pub(crate) use crate::arch::kernel::mmio::get_filesystem_driver; #[cfg(feature = "virtio-rng")] diff --git a/src/drivers/mod.rs b/src/drivers/mod.rs index 416d7f61af..bc37668b43 100644 --- a/src/drivers/mod.rs +++ b/src/drivers/mod.rs @@ -1,5 +1,7 @@ //! A module containing hermit-rs driver, hermit-rs driver trait and driver specific errors. +#[cfg(feature = "virtio-blk")] +pub mod blk; #[cfg(feature = "virtio-console")] pub mod console; #[cfg(feature = "virtio-fs")] diff --git a/src/drivers/pci.rs b/src/drivers/pci.rs index b9b50a0cd2..92a70f91da 100644 --- a/src/drivers/pci.rs +++ b/src/drivers/pci.rs @@ -5,8 +5,8 @@ use core::fmt; #[cfg(any( feature = "virtio-fs", - feature = "virtio-vsock", feature = "virtio-rng", + feature = "virtio-vsock" ))] use hermit_sync::InterruptTicketMutex; use hermit_sync::without_interrupts; @@ -21,6 +21,8 @@ use crate::arch::kernel::pci::PciConfigRegion; #[allow(unused_imports)] use crate::drivers::Driver; use crate::drivers::InterruptHandlerMap; +#[cfg(feature = "virtio-blk")] +use crate::drivers::blk::VirtioBlkDriver; #[cfg(feature = "virtio-fs")] use crate::drivers::fs::VirtioFsDriver; #[cfg(feature = "rtl8139")] @@ -362,6 +364,8 @@ pub(crate) fn print_information() { #[allow(clippy::enum_variant_names)] #[non_exhaustive] pub(crate) enum PciDriver { + #[cfg(feature = "virtio-blk")] + VirtioBlk(VirtioBlkDriver), #[cfg(feature = "virtio-fs")] VirtioFs(InterruptTicketMutex), #[cfg(feature = "virtio-rng")] @@ -371,6 +375,15 @@ pub(crate) enum PciDriver { } impl PciDriver { + #[cfg(feature = "virtio-blk")] + fn get_block_driver(&self) -> Option<&VirtioBlkDriver> { + #[allow(unreachable_patterns)] + match self { + Self::VirtioBlk(drv) => Some(drv), + _ => None, + } + } + #[cfg(feature = "virtio-rng")] fn get_rng_driver(&self) -> Option<&InterruptTicketMutex> { #[allow(unreachable_patterns)] @@ -409,6 +422,14 @@ pub(crate) type NetworkDevice = VirtioNetDriver; #[cfg(feature = "rtl8139")] pub(crate) type NetworkDevice = RTL8139Driver; +#[cfg(feature = "virtio-blk")] +pub(crate) fn get_block_driver() -> Option<&'static VirtioBlkDriver> { + PCI_DRIVERS + .get()? + .iter() + .find_map(|drv| drv.get_block_driver()) +} + #[cfg(feature = "virtio-rng")] pub(crate) fn get_rng_driver() -> Option<&'static InterruptTicketMutex> { PCI_DRIVERS @@ -458,6 +479,10 @@ pub(crate) fn init(handlers: &mut InterruptHandlerMap) { Ok(VirtioDriver::Console(drv)) => { crate::console::switch_to_virtio(*drv); } + #[cfg(feature = "virtio-blk")] + Ok(VirtioDriver::Blk(drv)) => { + register_driver(PciDriver::VirtioBlk(*drv)); + } #[cfg(feature = "virtio-fs")] Ok(VirtioDriver::Fs(drv)) => { register_driver(PciDriver::VirtioFs(InterruptTicketMutex::new(*drv))); diff --git a/src/drivers/virtio/mod.rs b/src/drivers/virtio/mod.rs index feed0850c8..797b05f790 100644 --- a/src/drivers/virtio/mod.rs +++ b/src/drivers/virtio/mod.rs @@ -31,6 +31,7 @@ impl VirtioIdExt for virtio::Id { fn as_feature(&self) -> Option<&str> { let feature = match self { Self::Net => "virtio-net", + Self::Block => "virtio-blk", Self::Console => "virtio-console", Self::Fs => "virtio-fs", Self::Vsock => "virtio-vsock", @@ -271,6 +272,8 @@ where pub mod error { use thiserror::Error; + #[cfg(feature = "virtio-blk")] + pub use crate::drivers::blk::error::VirtioBlkError; #[cfg(feature = "virtio-console")] pub use crate::drivers::console::error::VirtioConsoleError; #[cfg(feature = "virtio-fs")] @@ -341,5 +344,9 @@ pub mod error { #[cfg(feature = "virtio-rng")] #[error(transparent)] RngDriver(#[from] VirtioRngError), + + #[cfg(feature = "virtio-blk")] + #[error(transparent)] + BlkDriver(#[from] VirtioBlkError), } } diff --git a/src/drivers/virtio/transport/mmio.rs b/src/drivers/virtio/transport/mmio.rs index b59f9a00e7..e64e3505a3 100644 --- a/src/drivers/virtio/transport/mmio.rs +++ b/src/drivers/virtio/transport/mmio.rs @@ -16,6 +16,8 @@ use virtio::{DeviceStatus, le32}; use volatile::access::ReadOnly; use volatile::{VolatilePtr, VolatileRef}; +#[cfg(feature = "virtio-blk")] +use crate::drivers::blk::VirtioBlkDriver; #[cfg(feature = "virtio-console")] use crate::drivers::console::VirtioConsoleDriver; use crate::drivers::error::DriverError; @@ -324,6 +326,8 @@ impl IsrStatus { } pub(crate) enum VirtioDriver { + #[cfg(feature = "virtio-blk")] + Blk(alloc::boxed::Box), #[cfg(feature = "virtio-console")] Console(alloc::boxed::Box), #[cfg(feature = "virtio-fs")] @@ -393,6 +397,17 @@ pub(crate) fn init_device( Err(DriverError::InitVirtioDevFail(virtio_error)) } }, + #[cfg(feature = "virtio-blk")] + virtio::Id::Block => match init::(registers, irq_no, handlers) { + Ok(virt_blk_drv) => { + info!("Virtio block device driver initialized."); + Ok(VirtioDriver::Blk(alloc::boxed::Box::new(virt_blk_drv))) + } + Err(virtio_error) => { + error!("Virtio block device driver could not be initialized with device"); + Err(DriverError::InitVirtioDevFail(virtio_error)) + } + }, #[cfg(feature = "virtio-rng")] virtio::Id::Rng => match init::(registers, irq_no, handlers) { Ok(virt_rng_drv) => { diff --git a/src/drivers/virtio/transport/pci.rs b/src/drivers/virtio/transport/pci.rs index 2bbadc98cb..9f39347cba 100644 --- a/src/drivers/virtio/transport/pci.rs +++ b/src/drivers/virtio/transport/pci.rs @@ -21,6 +21,8 @@ use volatile::{VolatilePtr, VolatileRef}; use crate::arch::kernel::pci::PciConfigRegion; use crate::drivers::InterruptHandlerMap; +#[cfg(feature = "virtio-blk")] +use crate::drivers::blk::VirtioBlkDriver; #[cfg(feature = "virtio-console")] use crate::drivers::console::VirtioConsoleDriver; use crate::drivers::error::DriverError; @@ -139,6 +141,11 @@ impl CapCfg for IsrStatusRaw { const TYPE: CapCfgType = CapCfgType::Isr; } +#[cfg(feature = "virtio-blk")] +impl CapCfg for virtio::blk::Config { + const TYPE: CapCfgType = CapCfgType::Device; +} + impl CapCfg for virtio::console::Config { const TYPE: CapCfgType = CapCfgType::Device; } @@ -727,6 +734,7 @@ pub(crate) fn map_caps( #[cfg_attr( not(any( feature = "virtio-console", + feature = "virtio-blk", feature = "virtio-fs", all( not(all(target_arch = "riscv64", feature = "gem-net", not(feature = "pci"))), @@ -811,6 +819,19 @@ pub(crate) fn init_device( Err(DriverError::InitVirtioDevFail(virtio_error)) } }, + #[cfg(feature = "virtio-blk")] + virtio::Id::Block => match init::(device, handlers) { + Ok(virt_blk_drv) => { + info!("Virtio block device driver initialized."); + Ok(VirtioDriver::Blk(alloc::boxed::Box::new(virt_blk_drv))) + } + Err(virtio_error) => { + error!( + "Virtio block device driver could not be initialized with device: {device_id:x}" + ); + Err(DriverError::InitVirtioDevFail(virtio_error)) + } + }, #[cfg(feature = "virtio-rng")] virtio::Id::Rng => match init::(device, handlers) { Ok(virt_rng_drv) => { @@ -851,6 +872,8 @@ pub(crate) fn init_device( } pub(crate) enum VirtioDriver { + #[cfg(feature = "virtio-blk")] + Blk(alloc::boxed::Box), #[cfg(feature = "virtio-console")] Console(alloc::boxed::Box), #[cfg(feature = "virtio-fs")] diff --git a/src/fd/delegate.rs b/src/fd/delegate.rs index c17ceb3c90..af5639d197 100644 --- a/src/fd/delegate.rs +++ b/src/fd/delegate.rs @@ -24,6 +24,8 @@ use crate::fd::{Endpoint, ListenEndpoint, SocketOption, SocketOptionValue}; use crate::fs::mem::{MemDirectoryInterface, RamFileInterface, RomFileInterface}; #[cfg(feature = "uhyve")] use crate::fs::uhyve::{UhyveDirectoryHandle, UhyveFileHandle}; +#[cfg(feature = "virtio-blk")] +use crate::fs::vfat::{VfatDirectoryHandle, VfatFileHandle}; #[cfg(feature = "virtio-fs")] use crate::fs::virtio_fs::{VirtioFsDirectoryHandle, VirtioFsFileHandle}; use crate::fs::{DirectoryReader, FileAttr, SeekWhence}; @@ -51,6 +53,10 @@ pub(crate) enum Fd { VirtioFsFileHandle(VirtioFsFileHandle), #[cfg(feature = "virtio-fs")] VirtioFsDirectoryHandle(VirtioFsDirectoryHandle), + #[cfg(feature = "virtio-blk")] + VfatFileHandle(VfatFileHandle), + #[cfg(feature = "virtio-blk")] + VfatDirectoryHandle(VfatDirectoryHandle), RomFileInterface(RomFileInterface), RamFileInterface(RamFileInterface), MemDirectoryInterface(MemDirectoryInterface), @@ -102,6 +108,10 @@ fd_from! { VirtioFsFileHandle(VirtioFsFileHandle), #[cfg(feature = "virtio-fs")] VirtioFsDirectoryHandle(VirtioFsDirectoryHandle), + #[cfg(feature = "virtio-blk")] + VfatFileHandle(VfatFileHandle), + #[cfg(feature = "virtio-blk")] + VfatDirectoryHandle(VfatDirectoryHandle), RomFileInterface(RomFileInterface), RamFileInterface(RamFileInterface), MemDirectoryInterface(MemDirectoryInterface), @@ -137,6 +147,10 @@ impl ObjectInterface for Fd { Self::VirtioFsFileHandle(fd) => fd, #[cfg(feature = "virtio-fs")] Self::VirtioFsDirectoryHandle(fd) => fd, + #[cfg(feature = "virtio-blk")] + Self::VfatFileHandle(fd) => fd, + #[cfg(feature = "virtio-blk")] + Self::VfatDirectoryHandle(fd) => fd, Self::RomFileInterface(fd) => fd, Self::RamFileInterface(fd) => fd, Self::MemDirectoryInterface(fd) => fd, diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 78d0d46270..7d029f4f83 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -2,6 +2,8 @@ pub(crate) mod dev_directory; pub(crate) mod mem; #[cfg(feature = "uhyve")] pub(crate) mod uhyve; +#[cfg(feature = "virtio-blk")] +pub(crate) mod vfat; #[cfg(feature = "virtio-fs")] pub(crate) mod virtio_fs; @@ -390,6 +392,9 @@ pub(crate) fn init() { #[cfg(feature = "virtio-fs")] virtio_fs::init(); + #[cfg(feature = "virtio-blk")] + vfat::init(); + #[cfg(feature = "uhyve")] use crate::env::UhyveStartInfo; @@ -669,3 +674,11 @@ impl Drop for File { } } } + +/// force completion of pending disk writes (flush cache) +pub(crate) fn sync() -> io::Result<()> { + #[cfg(feature = "virtio-blk")] + vfat::sync()?; + + Ok(()) +} diff --git a/src/fs/vfat.rs b/src/fs/vfat.rs new file mode 100644 index 0000000000..e6f2e4c5a8 --- /dev/null +++ b/src/fs/vfat.rs @@ -0,0 +1,1037 @@ +use alloc::boxed::Box; +use alloc::string::{String, ToString}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicU64, Ordering}; + +use hadris_fat::r#async::read::FileReader; +use hadris_fat::r#async::write::FileWriter; +use hadris_fat::r#async::{ + Error as IoError, FatDir, FatVolume, FatVolumeReadExt, FatVolumeWriteExt, FileEntry, Read, + Seek, SeekFrom, Write, +}; +use hadris_fat::time::{FatDateTime, TimeProvider}; +use hermit_sync::OnceCell; +use time::OffsetDateTime; + +#[cfg(not(feature = "pci"))] +use crate::arch::kernel::mmio::get_block_driver; +use crate::drivers::blk::{SECTOR_SIZE, driver, with_driver}; +#[cfg(feature = "pci")] +use crate::drivers::pci::get_block_driver; +use crate::errno::Errno; +use crate::executor::block_on; +use crate::fd::{AccessPermission, Fd, ObjectInterface, OpenOption}; +use crate::fs::{self, DirectoryEntry, FileAttr, FileType, NodeKind, SeekWhence, VfsNode}; +use crate::io; +use crate::syscalls::DirentFormat; +use crate::time::SystemTime; + +/// Number of sectors `FatStream` keeps in memory. +/// +/// Several times [`RUN_SECTORS`], so that a read-ahead does not evict the +/// FAT and directory sectors in use. +const CACHE_SECTORS: usize = 256; + +/// Sectors fetched per cache miss and maximum length of a merged write. +const RUN_SECTORS: usize = 64; + +/// One cached device sector. +struct CachedSector { + /// Sector index on the device. + sector: usize, + /// Sector content. + data: [u8; SECTOR_SIZE], + /// Modified, but not yet written back. + dirty: bool, + /// Last access time for LRU eviction. + used: u64, +} + +/// Sector cache providing byte-wise `Read`/`Write`/`Seek` on the block +/// device for `hadris-fat`. Writes reach the device on eviction or flush. +struct FatStream { + cache: Vec, + /// Access counter for LRU eviction. + clock: u64, + /// Byte position on the device. + pos: usize, + /// Staging buffer for read-ahead. + read_run: Vec, + /// Staging buffer for write-back. Separate from `read_run`, because an + /// eviction may flush during a read-ahead. + write_run: Vec, +} + +impl FatStream { + pub fn new() -> Self { + Self { + cache: Vec::with_capacity(CACHE_SECTORS), + clock: 0, + pos: 0, + read_run: Vec::new(), + write_run: Vec::new(), + } + } + + /// The number of sectors the device holds. + fn capacity(&self) -> Result { + with_driver(|drv| drv.capacity()).ok_or(Errno::Nodev) + } + + /// Returns the slot holding `sector` and loads it on a miss. + /// + /// With `fill == false` the sector is not read, because the caller + /// overwrites it completely. + async fn slot(&mut self, sector: usize, fill: bool) -> Result { + self.clock += 1; + + if let Some(slot) = self.find(sector) { + self.cache[slot].used = self.clock; + return Ok(slot); + } + + if !fill { + return self.install(sector, &[0u8; SECTOR_SIZE]).await; + } + + // Read ahead, bounded by the device. + let sectors = RUN_SECTORS.min(self.capacity()? - sector); + + // Take the buffer out to avoid borrowing `self` twice. + let mut run = core::mem::take(&mut self.read_run); + run.resize(RUN_SECTORS * SECTOR_SIZE, 0); + + let result = self + .fill_run(sector, &mut run[..sectors * SECTOR_SIZE]) + .await; + + self.read_run = run; + result + } + + /// Reads a run of sectors into the cache and returns the slot holding + /// `sector`. + async fn fill_run(&mut self, sector: usize, run: &mut [u8]) -> Result { + driver().ok_or(Errno::Nodev)?.read(sector, run).await?; + + let mut requested = None; + for (index, data) in run.as_chunks::().0.iter().enumerate() { + // Keep cached copies, they may be dirty. + if self + .cache + .iter() + .any(|entry| entry.sector == sector + index) + { + continue; + } + + let slot = self.install(sector + index, data).await?; + if index == 0 { + requested = Some(slot); + } + } + + match requested { + Some(slot) => Ok(slot), + None => self.find(sector).ok_or(Errno::Io), + } + } + + /// Returns the slot holding `sector`, if it is cached. + fn find(&self, sector: usize) -> Option { + self.cache.iter().position(|entry| entry.sector == sector) + } + + /// Puts `data` into a free or freshly evicted slot and returns it. + async fn install(&mut self, sector: usize, data: &[u8]) -> Result { + let entry = CachedSector { + sector, + data: data.try_into().unwrap(), + dirty: false, + used: self.clock, + }; + + if self.cache.len() < CACHE_SECTORS { + self.cache.push(entry); + return Ok(self.cache.len() - 1); + } + + // LRU eviction; sectors of the current run are never chosen. + let victim = self + .cache + .iter() + .enumerate() + .min_by_key(|(_, entry)| entry.used) + .map(|(slot, _)| slot) + .unwrap(); + + // Flush all dirty sectors at once, so that adjacent ones are merged. + if self.cache[victim].dirty { + self.flush_all().await?; + } + + self.cache[victim] = entry; + + Ok(victim) + } + + /// Writes all dirty sectors back and merges adjacent ones into one request. + async fn flush_all(&mut self) -> Result<(), Errno> { + let mut dirty: Vec = (0..self.cache.len()) + .filter(|&slot| self.cache[slot].dirty) + .collect(); + if dirty.is_empty() { + return Ok(()); + } + dirty.sort_unstable_by_key(|&slot| self.cache[slot].sector); + + // `Batch::write` copies the data, so one buffer serves every run. + let mut run = core::mem::take(&mut self.write_run); + run.resize(RUN_SECTORS * SECTOR_SIZE, 0); + + let mut batch = driver().ok_or(Errno::Nodev)?.batch().await; + let mut submitted = Ok(()); + + let mut rest = dirty.as_slice(); + while let Some((&first, _)) = rest.split_first() { + // Longest run of consecutive sectors that fits the buffer. + let len = rest + .iter() + .enumerate() + .take(RUN_SECTORS) + .take_while(|&(index, &slot)| { + self.cache[slot].sector == self.cache[first].sector + index + }) + .count(); + let (group, tail) = rest.split_at(len); + rest = tail; + + for (index, &slot) in group.iter().enumerate() { + run[index * SECTOR_SIZE..(index + 1) * SECTOR_SIZE] + .copy_from_slice(&self.cache[slot].data); + } + + submitted = batch.write(self.cache[first].sector, &run[..len * SECTOR_SIZE]); + if submitted.is_err() { + break; + } + } + + self.write_run = run; + + // Keep the sectors dirty until the device has acknowledged them. + batch.finish().await?; + submitted?; + + for &slot in &dirty { + self.cache[slot].dirty = false; + } + + Ok(()) + } +} + +impl From for IoError { + fn from(e: Errno) -> Self { + IoError::from_source(e) + } +} + +impl Read for FatStream { + type Error = Errno; + + async fn read(&mut self, buf: &mut [u8]) -> Result> { + if buf.is_empty() { + return Ok(0); + } + + let sector = self.pos / SECTOR_SIZE; + let offset = self.pos % SECTOR_SIZE; + + if sector >= self.capacity()? { + return Ok(0); + } + + let slot = self.slot(sector, true).await?; + + // Short reads are allowed, so stay within one sector. + let n = usize::min(buf.len(), SECTOR_SIZE - offset); + buf[..n].copy_from_slice(&self.cache[slot].data[offset..offset + n]); + self.pos += n; + + Ok(n) + } +} + +impl Write for FatStream { + type Error = Errno; + + async fn write(&mut self, buf: &[u8]) -> Result> { + if buf.is_empty() { + return Ok(0); + } + + let sector = self.pos / SECTOR_SIZE; + let offset = self.pos % SECTOR_SIZE; + + if sector >= self.capacity()? { + return Err(IoError::from_source(Errno::Nospc)); + } + + let n = usize::min(buf.len(), SECTOR_SIZE - offset); + + // Only a partial sector has to be read first. + let slot = self.slot(sector, n != SECTOR_SIZE).await?; + + self.cache[slot].data[offset..offset + n].copy_from_slice(&buf[..n]); + self.cache[slot].dirty = true; + self.pos += n; + + Ok(n) + } + + async fn flush(&mut self) -> Result<(), IoError> { + self.flush_all().await?; + driver().ok_or(Errno::Nodev)?.flush().await?; + + Ok(()) + } +} + +impl Seek for FatStream { + type Error = Errno; + + async fn seek(&mut self, pos: SeekFrom) -> Result> { + let end = i64::try_from(self.capacity()? * SECTOR_SIZE).unwrap(); + let cur = i64::try_from(self.pos).unwrap(); + + let new = match pos { + SeekFrom::Start(n) => i64::try_from(n).ok(), + SeekFrom::Current(offset) => cur.checked_add(offset), + SeekFrom::End(offset) => end.checked_add(offset), + }; + + // Seeking past the end is allowed, accessing it is not. + let new = new.filter(|new| *new >= 0).ok_or(Errno::Inval)?; + + self.pos = usize::try_from(new).unwrap(); + + Ok(new as u64) + } +} + +fn format_file_size(size: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = 1024 * KB; + const GB: u64 = 1024 * MB; + if size < KB { + format!("{size}B") + } else if size < MB { + format!("{}KB", size / KB) + } else if size < GB { + format!("{}MB", size / MB) + } else { + format!("{}GB", size / GB) + } +} + +pub(crate) fn init() { + debug!("Try to initialize vfat filesystem"); + + if get_block_driver().is_none() { + return; + }; + + let stream = FatStream::new(); + debug!( + "Found block device with a capacity of {}", + format_file_size( + (stream.capacity().unwrap() * SECTOR_SIZE) + .try_into() + .unwrap() + ) + ); + + match block_on( + async { + FatVolume::builder(stream) + .time_provider(&TIME_PROVIDER) + .open() + .await + .map_err(map_err) + }, + None, + ) { + Ok(volume) => { + info!("Mounting {} volume at {MOUNT_POINT}", volume.fat_type()); + + if VFAT.set(SharedVolume(volume)).is_err() { + error!("Unable to mount block device at {MOUNT_POINT}"); + return; + } + + if let Err(err) = fs::FILESYSTEM + .get() + .unwrap() + .mount(MOUNT_POINT, Box::new(VfatDirectory::new(String::new()))) + { + error!("Unable to mount block device at {MOUNT_POINT}: {err:?}"); + } + } + Err(_) => { + warn!("Unable to create FAT volume"); + } + } +} + +/// Stamps directory entries with the system's wall-clock time. +#[derive(Debug)] +struct HermitTimeProvider; + +impl HermitTimeProvider { + pub const fn new() -> Self { + Self {} + } +} + +impl TimeProvider for HermitTimeProvider { + fn now(&self) -> FatDateTime { + let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH); + let Ok(now) = OffsetDateTime::from_unix_timestamp_nanos(now.as_nanos().cast_signed()) + else { + return FatDateTime::EPOCH; + }; + + let mut date_time = FatDateTime::new( + u16::try_from(now.year()).unwrap_or(1980), + now.month() as u8, + now.day(), + now.hour(), + now.minute(), + now.second(), + ); + + // `time_tenth` holds the odd second and the 10 ms units. + date_time.time_tenth = + u8::try_from((u16::from(now.second() % 2) * 100 + now.millisecond() / 10).min(199)) + .unwrap(); + + date_time + } +} + +static TIME_PROVIDER: HermitTimeProvider = HermitTimeProvider::new(); + +// --------------------------------------------------------------------------- +// VFS integration +// --------------------------------------------------------------------------- + +/// `FatVolume` is not `Sync`, because it keeps counters in `Cell`s. +struct SharedVolume(FatVolume); + +// SAFETY: the volume is only accessed while `VFAT_LOCK` is held, see `volume`. +unsafe impl Sync for SharedVolume {} + +static VFAT: OnceCell = OnceCell::new(); + +/// Serializes file system operations, see [`volume`]. +static VFAT_LOCK: async_lock::Mutex<()> = async_lock::Mutex::new(()); + +/// Incremented on every modification, invalidates cached readers. +static GENERATION: AtomicU64 = AtomicU64::new(0); + +fn modified() { + GENERATION.fetch_add(1, Ordering::Relaxed); +} + +/// Where the FAT file system is attached in the VFS. +const MOUNT_POINT: &str = "/root"; + +/// Translates a `hadris-fat` error into the kernel's error number. +fn map_err(err: hadris_fat::error::Error) -> Errno { + use hadris_fat::error::Error; + + match err { + Error::EntryNotFound => Errno::Noent, + Error::NotADirectory => Errno::Notdir, + Error::NotAFile => Errno::Isdir, + Error::NoFreeSpace => Errno::Nospc, + Error::InvalidPath | Error::InvalidShortFilename => Errno::Inval, + _ => Errno::Io, + } +} + +/// Locks the volume for one complete file system operation. +/// +/// `hadris-fat` locks only single accesses, so a lookup and the following +/// write could otherwise race on another core. +async fn volume() -> io::Result { + let fs = &VFAT.get().ok_or(Errno::Nodev)?.0; + let _guard = VFAT_LOCK.lock().await; + + Ok(Volume { fs, _guard }) +} + +/// The mounted volume, locked for one operation. +struct Volume { + fs: &'static FatVolume, + _guard: async_lock::MutexGuard<'static, ()>, +} + +impl core::ops::Deref for Volume { + type Target = FatVolume; + + fn deref(&self) -> &Self::Target { + self.fs + } +} + +/// Joins the directory's own path with a path relative to it. +fn join(prefix: &str, path: &str) -> String { + match (prefix.trim_matches('/'), path.trim_matches('/')) { + ("", rest) => rest.to_string(), + (base, "") => base.to_string(), + (base, rest) => [base, rest].join("/"), + } +} + +/// Finds the entry called `name` directly below `dir`. +async fn find(dir: &FatDir<'_, FatStream>, name: &str) -> io::Result> { + let mut entries = dir.entries(); + while let Some(entry) = entries.next_entry().await { + let entry = entry.map_err(map_err)?; + if entry.name() == name + && let Some(entry) = entry.as_entry() + { + return Ok(Some(entry.clone())); + } + } + + Ok(None) +} + +/// Walks `path` and returns the directory holding its last component together +/// with that component's entry, if it exists. +async fn resolve<'a>( + volume: &'a FatVolume, + path: &str, +) -> io::Result<(FatDir<'a, FatStream>, Option)> { + let mut dir = volume.root_dir(); + + let mut components = path.split('/').filter(|part| !part.is_empty()).peekable(); + + while let Some(component) = components.next() { + let entry = find(&dir, component).await?; + + if components.peek().is_none() { + return Ok((dir, entry)); + } + + let entry = entry.ok_or(Errno::Noent)?; + dir = dir.open_entry(&entry).map_err(map_err)?; + } + + Ok((dir, None)) +} + +/// Resolves `path` to a directory. +async fn resolve_dir<'a>( + volume: &'a FatVolume, + path: &str, +) -> io::Result> { + let (parent, entry) = resolve(volume, path).await?; + + match entry { + Some(entry) => parent.open_entry(&entry).map_err(map_err), + None if is_root(path) => Ok(parent), + None => Err(Errno::Noent), + } +} + +/// Whether `path` names the root of the volume. +fn is_root(path: &str) -> bool { + path.split('/').all(str::is_empty) +} + +/// Resolves `path` to an existing file entry. +async fn resolve_file(volume: &FatVolume, path: &str) -> io::Result { + let (_, entry) = resolve(volume, path).await?; + let entry = entry.ok_or(Errno::Noent)?; + + if entry.is_directory() { + return Err(Errno::Isdir); + } + + Ok(entry) +} + +/// Builds a `FileAttr`; FAT has no ownership or link count. +fn file_attr(size: u64, kind: NodeKind) -> FileAttr { + let mode = match kind { + NodeKind::Directory => { + AccessPermission::from_bits(0o755).unwrap() | AccessPermission::S_IFDIR + } + NodeKind::File => AccessPermission::from_bits(0o644).unwrap() | AccessPermission::S_IFREG, + }; + + FileAttr { + st_size: i64::try_from(size).unwrap_or(i64::MAX), + st_mode: mode, + st_blksize: i64::try_from(SECTOR_SIZE).unwrap(), + st_blocks: i64::try_from(size.div_ceil(SECTOR_SIZE as u64)).unwrap_or(i64::MAX), + st_nlink: 1, + ..Default::default() + } +} + +/// Lists the entries of the directory at `path`, relative to the FAT root. +async fn readdir_at( + volume: &FatVolume, + path: String, +) -> io::Result> { + let dir = resolve_dir(volume, &path).await?; + + let mut result = Vec::new(); + let mut entries = dir.entries(); + while let Some(entry) = entries.next_entry().await { + let entry = entry.map_err(map_err)?; + result.push(DirectoryEntry::new(entry.name().into_owned())); + } + + Ok(result) +} + +/// A directory of the FAT file system, as seen by the VFS. +#[derive(Debug)] +pub(crate) struct VfatDirectory { + /// Path of this directory relative to the FAT root, without leading slash. + prefix: String, +} + +impl VfatDirectory { + pub fn new(prefix: String) -> Self { + Self { prefix } + } +} + +impl VfsNode for VfatDirectory { + fn get_kind(&self) -> NodeKind { + NodeKind::Directory + } + + fn get_file_attributes(&self) -> io::Result { + Ok(file_attr(0, NodeKind::Directory)) + } + + fn get_object(&self) -> io::Result>> { + Ok(Arc::new(async_lock::RwLock::new( + VfatDirectoryHandle::new(self.prefix.clone()).into(), + ))) + } + + fn traverse_readdir(&self, path: &str) -> io::Result> { + let path = join(&self.prefix, path); + + block_on( + async { + let volume = volume().await?; + + readdir_at(&volume, path).await + }, + None, + ) + } + + fn traverse_stat(&self, path: &str) -> io::Result { + let path = join(&self.prefix, path); + + block_on( + async { + let volume = volume().await?; + + let (_, entry) = resolve(&volume, &path).await?; + let Some(entry) = entry else { + return if is_root(&path) { + Ok(file_attr(0, NodeKind::Directory)) + } else { + Err(Errno::Noent) + }; + }; + + if entry.is_directory() { + Ok(file_attr(0, NodeKind::Directory)) + } else { + Ok(file_attr(entry.len(), NodeKind::File)) + } + }, + None, + ) + } + + fn traverse_lstat(&self, path: &str) -> io::Result { + // FAT has no symbolic links. + self.traverse_stat(path) + } + + fn traverse_mkdir(&self, path: &str, _mode: AccessPermission) -> io::Result<()> { + let path = join(&self.prefix, path); + let name = path.rsplit('/').next().ok_or(Errno::Inval)?; + + block_on( + async { + let volume = volume().await?; + + let (parent, entry) = resolve(&volume, &path).await?; + if entry.is_some() { + return Err(Errno::Exist); + } + + volume.create_dir(&parent, name).await.map_err(map_err)?; + + Ok(()) + }, + None, + ) + } + + fn traverse_rmdir(&self, path: &str) -> io::Result<()> { + block_on( + async { + let volume = volume().await?; + + let entry = resolve(&volume, &join(&self.prefix, path)) + .await? + .1 + .ok_or(Errno::Noent)?; + if !entry.is_directory() { + return Err(Errno::Notdir); + } + + modified(); + volume.delete(&entry).await.map_err(map_err) + }, + None, + ) + } + + fn traverse_unlink(&self, path: &str) -> io::Result<()> { + block_on( + async { + let volume = volume().await?; + + let entry = resolve_file(&volume, &join(&self.prefix, path)).await?; + + modified(); + volume.delete(&entry).await.map_err(map_err) + }, + None, + ) + } + + fn traverse_open( + &self, + path: &str, + opt: OpenOption, + _mode: AccessPermission, + ) -> io::Result>> { + let full = join(&self.prefix, path); + + block_on( + async { + let volume = volume().await?; + + let (parent, entry) = resolve(&volume, &full).await?; + + // Directories are always opened as directory handle. + if entry + .as_ref() + .map_or(is_root(&full), FileEntry::is_directory) + { + return Ok(Arc::new(async_lock::RwLock::new( + VfatDirectoryHandle::new(full.clone()).into(), + ))); + } + + let entry = match entry { + Some(entry) => { + if opt.contains(OpenOption::O_CREAT | OpenOption::O_EXCL) { + return Err(Errno::Exist); + } + if opt.contains(OpenOption::O_TRUNC) { + modified(); + volume.truncate(&entry, 0).await.map_err(map_err)?; + } + entry + } + None => { + if !opt.contains(OpenOption::O_CREAT) { + return Err(Errno::Noent); + } + let name = full.rsplit('/').next().ok_or(Errno::Inval)?; + volume.create_file(&parent, name).await.map_err(map_err)? + } + }; + + let pos = if opt.contains(OpenOption::O_APPEND) { + entry.len() + } else { + 0 + }; + + Ok(Arc::new(async_lock::RwLock::new( + VfatFileHandle::new(full.clone(), pos).into(), + ))) + }, + None, + ) + } +} + +/// An open file of the FAT file system. +/// +/// Keeps the reader of the last `read`, so that sequential reads continue +/// at the current cluster instead of resolving the path and walking the +/// FAT chain again. +pub(crate) struct VfatFileHandle { + path: String, + state: async_lock::Mutex, +} + +struct FileState { + pos: u64, + /// Reader and the [`GENERATION`] it was created in. + reader: Option<(u64, FileReader<'static, FatStream>)>, +} + +// SAFETY: the reader borrows the volume, but is only used in +// `VfatFileHandle::read` while `VFAT_LOCK` is held, and has no `Drop`. +unsafe impl Send for FileState {} + +impl core::fmt::Debug for VfatFileHandle { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("VfatFileHandle") + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl VfatFileHandle { + pub fn new(path: String, pos: u64) -> Self { + Self { + path, + state: async_lock::Mutex::new(FileState { pos, reader: None }), + } + } +} + +impl ObjectInterface for VfatFileHandle { + async fn read(&self, buf: &mut [u8]) -> io::Result { + let volume = volume().await?; + let mut state = self.state.lock().await; + let generation = GENERATION.load(Ordering::Relaxed); + + let mut reader = match state.reader.take() { + Some((created, reader)) if created == generation && reader.position() == state.pos => { + reader + } + _ => { + let entry = resolve_file(volume.fs, &self.path).await?; + let mut reader = volume + .fs + .read_file(&entry) + .map_err(map_err)? + .with_cached_chain() + .await + .map_err(map_err)?; + reader + .seek(SeekFrom::Start(state.pos)) + .await + .map_err(map_err)?; + reader + } + }; + + let read = reader.read(buf).await.map_err(map_err)?; + state.pos += u64::try_from(read).unwrap(); + state.reader = Some((generation, reader)); + + Ok(read) + } + + async fn write(&self, buf: &[u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + + let volume = volume().await?; + let mut entry = resolve_file(&volume, &self.path).await?; + let mut state = self.state.lock().await; + state.reader = None; + modified(); + + let start = usize::try_from(state.pos).map_err(|_| Errno::Fbig)?; + let end = start.checked_add(buf.len()).ok_or(Errno::Fbig)?; + let old_len = usize::try_from(entry.len()).map_err(|_| Errno::Fbig)?; + + // `FileWriter` only appends, so data behind `start` is cut off and + // written again after `buf`. + let mut tail = Vec::new(); + let mut gap = Vec::new(); + if start < old_len { + if end < old_len { + let mut reader = volume.read_file(&entry).map_err(map_err)?; + reader + .seek(SeekFrom::Start(u64::try_from(end).unwrap())) + .await + .map_err(map_err)?; + tail = reader.read_to_vec().await.map_err(map_err)?; + } + + volume.truncate(&entry, start).await.map_err(map_err)?; + entry = resolve_file(&volume, &self.path).await?; + } else { + gap.resize(start - old_len, 0); + } + + let mut writer = FileWriter::new_append(&volume, &entry) + .await + .map_err(map_err)?; + for part in [gap.as_slice(), buf, tail.as_slice()] { + if !part.is_empty() { + writer.write(part).await.map_err(map_err)?; + } + } + writer.finish().await.map_err(map_err)?; + + state.pos = u64::try_from(end).unwrap(); + + Ok(buf.len()) + } + + async fn lseek(&self, offset: isize, whence: SeekWhence) -> io::Result { + let volume = volume().await?; + let mut state = self.state.lock().await; + let offset = i64::try_from(offset).unwrap(); + + let new = match whence { + SeekWhence::Set => offset, + SeekWhence::Cur => i64::try_from(state.pos).unwrap() + offset, + SeekWhence::End => { + let entry = resolve_file(&volume, &self.path).await?; + i64::try_from(entry.len()).unwrap() + offset + } + _ => return Err(Errno::Inval), + }; + + if new < 0 { + return Err(Errno::Inval); + } + + state.pos = u64::try_from(new).unwrap(); + + Ok(isize::try_from(new).unwrap()) + } + + async fn fstat(&self) -> io::Result { + let volume = volume().await?; + let entry = resolve_file(&volume, &self.path).await?; + + Ok(file_attr(entry.len(), NodeKind::File)) + } + + async fn truncate(&self, size: usize) -> io::Result<()> { + let volume = volume().await?; + let entry = resolve_file(&volume, &self.path).await?; + + modified(); + volume.truncate(&entry, size).await.map_err(map_err) + } + + async fn isatty(&self) -> io::Result { + Ok(false) + } + + async fn fsync(&self) -> io::Result<()> { + volume().await?.sync().await.map_err(map_err) + } +} + +/// An open directory of the FAT file system. +#[derive(Debug)] +pub(crate) struct VfatDirectoryHandle { + prefix: String, + /// Index of the next entry `getdents` will report. + next: async_lock::Mutex, +} + +impl VfatDirectoryHandle { + pub fn new(prefix: String) -> Self { + Self { + prefix, + next: async_lock::Mutex::new(0), + } + } +} + +impl ObjectInterface for VfatDirectoryHandle { + async fn getdents( + &self, + buf: &mut [MaybeUninit], + format: DirentFormat, + ) -> io::Result { + let entries = { + let volume = volume().await?; + readdir_at(&volume, self.prefix.clone()).await? + }; + + let mut next = self.next.lock().await; + let mut offset = 0; + + while let Some(entry) = entries.get(*next) { + let Some(next_offset) = + format.write_entry(buf, offset, 0, FileType::Unknown, entry.name.as_bytes()) + else { + if offset == 0 { + return Err(Errno::Inval); + } + break; + }; + + offset = next_offset; + *next += 1; + } + + Ok(offset) + } + + async fn lseek(&self, offset: isize, whence: SeekWhence) -> io::Result { + if whence != SeekWhence::Set || offset != 0 { + return Err(Errno::Inval); + } + *self.next.lock().await = 0; + + Ok(0) + } + + async fn fstat(&self) -> io::Result { + Ok(file_attr(0, NodeKind::Directory)) + } + + async fn fsync(&self) -> io::Result<()> { + volume().await?.sync().await.map_err(map_err) + } +} + +pub(crate) fn sync() -> io::Result<()> { + block_on( + async { + match volume().await { + Ok(v) => v.sync().await.map_err(map_err), + Err(_) => Ok(()), + } + }, + None, + ) +} diff --git a/src/init_cell.rs b/src/init_cell.rs index c2646e0134..36d2ed79de 100644 --- a/src/init_cell.rs +++ b/src/init_cell.rs @@ -2,7 +2,8 @@ not(any( feature = "virtio-vsock", feature = "virtio-fs", - feature = "virtio-rng" + feature = "virtio-rng", + feature = "virtio-blk" )), expect(dead_code) )] diff --git a/src/syscalls/mod.rs b/src/syscalls/mod.rs index b8d904d5e3..6441601ee8 100644 --- a/src/syscalls/mod.rs +++ b/src/syscalls/mod.rs @@ -99,6 +99,11 @@ pub(crate) fn get_application_parameters() -> (i32, *const *const u8, *const *co } pub(crate) fn shutdown(arg: i32) -> ! { + // force completion of pending disk writes + if fs::sync().is_err() { + error!("Unable to synchronize file system!"); + } + // print some performance statistics crate::arch::kernel::print_statistics(); From 1303a9465814417ddaa31dad2d0b84d7157c5188 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 2 Nov 2026 22:10:08 +0100 Subject: [PATCH 2/2] remove obsolete helper function `possible_cores` --- src/drivers/blk.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/drivers/blk.rs b/src/drivers/blk.rs index f18aadc70e..394cf3a2c5 100644 --- a/src/drivers/blk.rs +++ b/src/drivers/blk.rs @@ -16,6 +16,7 @@ use volatile::VolatileRef; use volatile::access::ReadOnly; use crate::arch::kernel::core_local::core_id; +use crate::arch::kernel::get_processor_count; use crate::config::VIRTIO_MAX_QUEUE_SIZE; #[cfg(not(feature = "pci"))] use crate::drivers::mmio::get_block_driver; @@ -64,18 +65,6 @@ impl RequestHeader { } } -/// The number of cores that may ever issue a request. -/// -/// `get_processor_count` would return 1 during driver initialization. -fn possible_cores() -> usize { - #[cfg(feature = "smp")] - let count = crate::arch::kernel::get_possible_cpus(); - #[cfg(not(feature = "smp"))] - let count = 1; - - usize::try_from(count).unwrap() -} - /// Wraps a single buffer element in the [`SmallVec`] the virtqueue expects. fn single(elem: BufferElem) -> SmallVec<[BufferElem; 2]> { let mut vec = SmallVec::new(); @@ -370,7 +359,7 @@ impl super::virtio::VirtioDriver for VirtioBlkDriver { }; // More queues than cores would stay idle. - let count = usize::from(offered.max(1)).min(possible_cores()); + let count = usize::from(offered.max(1)).min(get_processor_count().try_into().unwrap()); for index in 0..count { queues.push(VirtQueue::Split(