From 6abc7baef0a8f560e4c41db3f7bd4137270c91cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=88=CE=BB=CE=BB=CE=B5=CE=BD=20=CE=95=CE=BC=CE=AF=CE=BB?= =?UTF-8?q?=CE=B9=CE=B1=20=CE=86=CE=BD=CE=BD=CE=B1=20Zscheile?= Date: Tue, 28 Apr 2026 17:17:45 +0200 Subject: [PATCH 1/2] feat(fs/mem): tarfs implementation This implements the internal Hermit image mode. The decompressed Hermit image gets passed from the bootloader (hermit-loader) or hypervisor (uhyve) to the kernel, the location is announced via the FDT property `/chosen/linux,initrd-*`, and the kernel parses the contained tar (expected in ustar format) file, placing its content at the filesystem root, similar to an initrd. The access permissions get copied from the tar archive, directories get default permissions. --- Cargo.lock | 12 +++++++ Cargo.toml | 8 +++++ src/fs/mem.rs | 88 +++++++++++++++++++++++++++++++++++++-------- src/fs/mod.rs | 16 ++++++++- src/mm/mod.rs | 74 +++++++++++++++++++++++++++++++++++--- xtask/src/clippy.rs | 2 +- 6 files changed, 179 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 488f6f647a..a74750c77c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -887,6 +887,7 @@ dependencies = [ "smoltcp", "take-static", "talc", + "tar-no-std", "thiserror", "time", "tock-registers 0.10.1", @@ -1859,6 +1860,17 @@ dependencies = [ "xattr", ] +[[package]] +name = "tar-no-std" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715f9a4586706a61c571cb5ee1c3ac2bbb2cf63e15bce772307b95befef5f5ee" +dependencies = [ + "bitflags 2.13.0", + "log", + "num-traits", +] + [[package]] name = "thiserror" version = "2.0.18" diff --git a/Cargo.toml b/Cargo.toml index 467138ff40..5d8b4d1676 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,6 +156,13 @@ semihosting = ["dep:semihosting"] ## _application processor_s (AP) from the _boot-strap processor_ (BSP). smp = ["acpi"] +## Enables (internal) hermit image support. +## +## This allows packaging a kernel together with all its static environment files into a .tar.gz archive, +## whose decompressed version gets handled by the kernel +## (the decompression gets handled by the bootloader / hypervisor). +hermit-image = ["dep:tar-no-std"] + #! ### Network Features ## Enables TCP support. @@ -346,6 +353,7 @@ simple-shell = { version = "0.0.1", optional = true } smallvec = { version = "1", features = ["const_new"] } take-static = "0.1" talc = { version = "5" } +tar-no-std = { version = "0.4", optional = true, features = ["alloc"] } thiserror = { version = "2", default-features = false } time = { version = "0.3", default-features = false } volatile = "0.6" diff --git a/src/fs/mem.rs b/src/fs/mem.rs index dde335f13c..91a888ec10 100644 --- a/src/fs/mem.rs +++ b/src/fs/mem.rs @@ -122,15 +122,6 @@ pub(crate) struct RamFileInner { pub attr: FileAttr, } -impl RamFileInner { - pub fn new(attr: FileAttr) -> Self { - Self { - data: Vec::new(), - attr, - } - } -} - pub struct RamFileInterface { /// Position within the file pos: Mutex, @@ -354,6 +345,10 @@ impl VfsNode for RamFile { impl RamFile { pub fn new(mode: AccessPermission) -> Self { + Self::new_with_data(Vec::new(), mode) + } + + fn new_with_data(data: Vec, mode: AccessPermission) -> Self { let microseconds = arch::kernel::systemtime::now_micros(); let t = timespec::from_usec(microseconds as i64); let attr = FileAttr { @@ -365,7 +360,7 @@ impl RamFile { }; Self { - data: Arc::new(RwLock::new(RamFileInner::new(attr))), + data: Arc::new(RwLock::new(RamFileInner { data, attr })), } } } @@ -468,6 +463,64 @@ impl MemDirectory { } } + #[cfg(feature = "hermit-image")] + pub fn try_from_image(image: &'static [u8]) -> io::Result { + let this = Self::new(AccessPermission::S_IRUSR); + + let tar_archive_ref = tar_no_std::TarArchiveRef::new(image).map_err(|e| { + error!("[Hermit image] Tar file has invalid format: {e:?}"); + Errno::Inval + })?; + + for entry in tar_archive_ref.entries() { + let filename = entry.filename(); + let filename = filename.as_str().map_err(|e| { + error!( + "[Hermit image] Tar entry has not supported filename (non UTF-8): {filename:?}; {e}", + ); + Errno::Inval + })?; + if filename.is_empty() { + continue; + } + debug!("[Hermit image] Processing tar entry: {filename}"); + + let mode = entry.posix_header().mode.to_flags().map_err(|e| { + error!( + "[Hermit image] Tar entry {filename:?} has invalid mode: {:?}; {e}", + entry.posix_header().mode, + ); + Errno::Inval + })?; + let mode = AccessPermission::from_bits(mode.bits() as u32).ok_or_else(|| { + error!("[Hermit image] Tar entry {filename:?} has invalid mode: {mode:?}"); + Errno::Inval + })?; + + for (i, _) in filename.match_indices("/") { + let part = &filename[..i]; + if this.traverse_lstat(part).is_err() { + this.traverse_mkdir( + part, + AccessPermission::S_IRUSR + | AccessPermission::S_IWUSR + | AccessPermission::S_IRGRP, + ) + .inspect_err(|e| { + error!("[Hermit image] Unable to mkdir {part:?}: {e}"); + })?; + } + } + + this.traverse_create_file(filename, entry.data(), mode) + .inspect_err(|e| { + error!("[Hermit image] Unable to write entry {filename:?}: {e}"); + })?; + } + + Ok(this) + } + async fn async_traverse_open( &self, path: &str, @@ -701,11 +754,16 @@ impl VfsNode for MemDirectory { return directory.traverse_create_file(rest, data, mode); } - let file = RomFile::new(data, mode); - self.inner - .write() - .await - .insert(component.to_owned(), Box::new(file)); + let file: Box = if mode.contains(AccessPermission::S_IWUSR) + || mode.contains(AccessPermission::S_IWGRP) + || mode.contains(AccessPermission::S_IWOTH) + { + Box::new(RamFile::new_with_data(data.to_vec(), mode)) + } else { + Box::new(RomFile::new(data, mode)) + }; + + self.inner.write().await.insert(component.to_owned(), file); Ok(()) }, None, diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 8ef50ad0c7..e34da34856 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -307,7 +307,21 @@ pub(crate) fn init() { const VERSION: &str = env!("CARGO_PKG_VERSION"); const UTC_BUILT_TIME: &str = build_time::build_time_utc!(); - let root_filesystem = Filesystem::new(); + #[cfg_attr(not(feature = "hermit-image"), expect(unused_mut))] + let mut root_filesystem = Filesystem::new(); + + // Handle optional Hermit Image specified in FDT. + #[cfg(feature = "hermit-image")] + if let Some(tar_image) = crate::mm::hermit_tar_image() { + root_filesystem.root = + MemDirectory::try_from_image(tar_image).expect("Unable to parse Hermit Image"); + } + + if crate::mm::hermit_tar_image().is_some() { + error!( + "Kernel built without Hermit image support, but a Hermit image was supplied: ignoring" + ); + } root_filesystem .mkdir("/tmp", AccessPermission::from_bits(0o777).unwrap()) diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 9f16f4104f..ab6c46b860 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -46,6 +46,7 @@ mod physicalmem; mod virtualmem; use core::alloc::Layout; +use core::cmp; use core::mem::MaybeUninit; use core::ops::Range; @@ -110,9 +111,56 @@ pub(crate) fn claim_initial_heap() { } } +/// Physical and virtual address range of the Hermit image, in case it is present +/// (indicated via FDT). +static HERMIT_IMAGE_START_AND_LEN: Lazy> = Lazy::new(|| { + let fdt = env::fdt()?; + + // per FDT specification, /chosen always exists + let chosen = fdt.find_node("/chosen").unwrap(); + + let fdt::node::NodeProperty { value: addr, .. } = chosen.property("linux,initrd-start")?; + + let fdt::node::NodeProperty { + value: addr_end, .. + } = chosen.property("linux,initrd-end")?; + + let (addr_, addr_end_) = (addr.try_into(), addr_end.try_into()); + + let ret = if let (Ok(addr), Ok(addr_end)) = (addr_, addr_end_) { + let addr = usize::from_be_bytes(addr); + let addr_end = usize::from_be_bytes(addr_end); + info!("Hermit image at {addr:x} - {addr_end:x}"); + addr_end.checked_sub(addr).map(|len| { + ( + VirtAddr::from_ptr(core::ptr::with_exposed_provenance::(addr)), + len, + ) + }) + } else { + None + }; + + if ret.is_none() { + error!( + "Hermit image supplied with invalid address range (#addr = {}, #addr_end = {})", + addr.len(), + addr_end.len(), + ); + } + + ret +}); + +pub(crate) fn hermit_tar_image() -> Option<&'static [u8]> { + // technically, the following is UB, because the kernel might be contained within... + HERMIT_IMAGE_START_AND_LEN + .map(|(addr, len)| unsafe { core::slice::from_raw_parts(addr.as_ptr(), len) }) +} + #[cfg(target_os = "none")] pub(crate) fn init() { - use crate::arch::mm::paging; + use arch::mm::paging; Lazy::force(&KERNEL_ADDR_RANGE); @@ -120,12 +168,30 @@ pub(crate) fn init() { arch::mm::init(); } + Lazy::force(&HERMIT_IMAGE_START_AND_LEN); + let total_mem = physicalmem::total_memory_size(); let kernel_addr_range = KERNEL_ADDR_RANGE.clone(); + let reserved_addr_range = if let Some((image_start, image_len)) = *HERMIT_IMAGE_START_AND_LEN { + cmp::min( + kernel_addr_range.start, + image_start.align_down(LargePageSize::SIZE), + ) + ..cmp::max( + kernel_addr_range.end, + (image_start + image_len).align_up(LargePageSize::SIZE), + ) + } else { + kernel_addr_range.clone() + }; info!("Total memory size: {} MiB", total_mem >> 20); info!( "Kernel region: {:p}..{:p}", - kernel_addr_range.start, kernel_addr_range.end + kernel_addr_range.start, kernel_addr_range.end, + ); + info!( + "Locally reserved region: {:p}..{:p}", + reserved_addr_range.start, reserved_addr_range.end, ); // we reserve physical memory for the required page tables @@ -146,7 +212,7 @@ pub(crate) fn init() { // On UEFI, the given memory is guaranteed free memory and the kernel is located before the given memory reserved_space } else { - (kernel_addr_range.end.as_u64() - env::get_ram_address().unwrap().as_u64() + (reserved_addr_range.end.as_u64() - env::get_ram_address().unwrap().as_u64() + reserved_space as u64) as usize }; info!("Minimum memory size: {} MiB", min_mem >> 20); @@ -166,7 +232,7 @@ pub(crate) fn init() { // we reserve at least 75% of the memory for the user space let reserve: usize = (avail_mem * 75) / 100; // 64 MB is enough as kernel heap - let reserve = core::cmp::min(reserve, 0x0400_0000); + let reserve = cmp::min(reserve, 0x0400_0000); let virt_size: usize = reserve.align_down(LargePageSize::SIZE as usize); let layout = PageLayout::from_size_align(virt_size, LargePageSize::SIZE as usize).unwrap(); diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs index de1a6997e5..4448e910a6 100644 --- a/xtask/src/clippy.rs +++ b/xtask/src/clippy.rs @@ -21,7 +21,7 @@ impl Clippy { clippy().run()?; clippy().arg("--features=common-os").run()?; clippy() - .arg("--features=acpi,dns,fsgsbase,pci,smp,vga") + .arg("--features=acpi,dns,fsgsbase,pci,smp,vga,hermit-image") .run()?; clippy().arg("--no-default-features").run()?; clippy().arg("--all-features").run()?; From d1e0ec4b0d41395e197022dd73518754d581d99c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=88=CE=BB=CE=BB=CE=B5=CE=BD=20=CE=95=CE=BC=CE=AF=CE=BB?= =?UTF-8?q?=CE=B9=CE=B1=20=CE=86=CE=BD=CE=BD=CE=B1=20Zscheile?= Date: Tue, 14 Jul 2026 13:41:08 +0200 Subject: [PATCH 2/2] feat(xtask): Hermit Image building --- Cargo.lock | 277 +++++++++++++++++++++++++++++++++++++++++++++ xtask/Cargo.toml | 21 +++- xtask/src/ci/rs.rs | 87 +++++++++++++- 3 files changed, 381 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a74750c77c..1bee8e452d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,12 @@ dependencies = [ "tock-registers 0.10.1", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -168,6 +174,12 @@ dependencies = [ "paste", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-executor" version = "1.14.0" @@ -284,6 +296,17 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "byte-unit" +version = "5.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a813de7f2bbedb7dce265b64f1cf5908ebe4d56281ece8d847e98113788b9b0" +dependencies = [ + "rust_decimal", + "serde", + "utf8-width", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -302,6 +325,39 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a57a50948117a233b27f9bf73ab74709ab90d245216c4707cc16eea067a50bb" +[[package]] +name = "camino" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.2.56" @@ -446,6 +502,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -611,6 +676,12 @@ dependencies = [ "log", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -676,6 +747,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "float-cmp" version = "0.10.0" @@ -829,8 +910,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77e69cdfe59e70c3c9b9b361e1fa0052aa8c08a746b6837a0258b7cfab46b67c" dependencies = [ "align-address 0.4.0", + "byte-unit", "const_parse", + "goblin", + "log", + "plain", + "serde", + "tar-no-std", "time", + "toml", ] [[package]] @@ -981,6 +1069,16 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "interrupt-mutex" version = "0.1.0" @@ -1174,6 +1272,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "nix" version = "0.28.0" @@ -1603,6 +1711,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3f2ad9f15a07f4a0e1677124f9120ce7e83ab7e1ca7186af0ca9da529b62e80" +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +dependencies = [ + "arrayvec", + "num-traits", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1665,6 +1783,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "sbi-rt" version = "0.0.4" @@ -1715,6 +1842,68 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8e4abf97879f4e80db69a9fba7bd64998e9bdad25f58ef045a778e191172fd4" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "sha2" version = "0.11.0" @@ -1738,6 +1927,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "simple-shell" version = "0.0.1" @@ -1921,6 +2116,45 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d2d250f87fb3fb6f225c907cf54381509f47b40b74b1d1f12d2dccbc915bdfe" +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "trapframe" version = "0.11.0" @@ -2000,6 +2234,12 @@ dependencies = [ "log", ] +[[package]] +name = "utf8-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "159a7cadce548703edd50d24069bc294c5415ecab0a480e0cd1ca06d112dc94a" + [[package]] name = "utf8-zero" version = "0.8.1" @@ -2087,6 +2327,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2172,6 +2422,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2370,6 +2629,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -2419,15 +2684,21 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", + "cargo_metadata", "clap", + "flate2", "goblin", + "hermit-entry", "home", "libc", "ovmf-prebuilt", "sysinfo", + "tar", + "toml", "ureq", "vsock", "wait-timeout", + "walkdir", "xshell", ] @@ -2456,3 +2727,9 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 9a144eae7b..453ed7fe71 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -5,19 +5,38 @@ edition = "2024" [features] default = ["ci"] -ci = ["dep:libc", "dep:ovmf-prebuilt", "dep:sysinfo", "dep:ureq", "dep:vsock", "dep:wait-timeout"] +ci = [ + "dep:cargo_metadata", + "dep:flate2", + "dep:hermit-entry", + "dep:libc", + "dep:ovmf-prebuilt", + "dep:sysinfo", + "dep:tar", + "dep:toml", + "dep:ureq", + "dep:vsock", + "dep:wait-timeout", + "dep:walkdir", +] [dependencies] anyhow = "1.0" +cargo_metadata = { version = "0.23", optional = true } clap = { version = "4", features = ["derive"] } +flate2 = { version = "1.1", optional = true } goblin = { version = "0.10", default-features = false, features = ["archive", "elf32", "elf64", "std"] } +hermit-entry = { version = "0.10", features = ["loader"], optional = true } home = "0.5" libc = { version = "0.2", optional = true } ovmf-prebuilt = { version = "0.2", optional = true } sysinfo = { version = "0.39", optional = true } +tar = { version = "0.4", optional = true } +toml = { version = "1.1", optional = true } ureq = { version = "3", default-features = false, features = ["rustls"], optional = true } vsock = { version = "0.5", optional = true } wait-timeout = { version = "0.2", optional = true } +walkdir = { version = "2.5", optional = true } xshell = "0.2" [lints] diff --git a/xtask/src/ci/rs.rs b/xtask/src/ci/rs.rs index 3be615aa16..ed2ba8e553 100644 --- a/xtask/src/ci/rs.rs +++ b/xtask/src/ci/rs.rs @@ -1,7 +1,9 @@ -use std::path::PathBuf; +use std::fs; +use std::path::{Path, PathBuf}; use anyhow::Result; use clap::{Args, Subcommand}; +use hermit_entry::config; use crate::cargo_build::CargoBuild; @@ -58,6 +60,7 @@ impl Rs { } let mut cargo = crate::cargo(); + let parent_root = super::parent_root(); if self.package.contains("rftrace") { cargo.env( @@ -67,7 +70,7 @@ impl Rs { }; cargo - .current_dir(super::parent_root()) + .current_dir(parent_root) .arg("build") .args(self.cargo_build.artifact.arch.ci_cargo_args()) .args(self.cargo_build.cargo_build_args()) @@ -77,10 +80,88 @@ impl Rs { let status = cargo.status()?; assert!(status.success()); + // discover Hermit Images case + let manifest_dir = { + let cur_package = cargo_metadata::PackageName::new(self.package.clone()); + let mut cargo = cargo_metadata::MetadataCommand::new(); + cargo + .current_dir(parent_root) + .no_deps() + .verbose(true) + .exec()? + .packages + .iter() + .find(|i| i.name == cur_package) + .expect("unable to find current package in `cargo metadata` output") + // this path points to `Cargo.toml` + .manifest_path + .parent() + .unwrap() + .to_path_buf() + }; + eprintln!("MANIFEST_DIR = {manifest_dir}"); + let hermit_image_root = { + let image_root = manifest_dir.join("image-root"); + if image_root.is_dir() { + Some(image_root) + } else { + None + } + }; + + let mut build_artifact = self.cargo_build.artifact.ci_image(&self.package); + + // handle Hermit Images + if let Some(image_root) = hermit_image_root { + eprintln!("discovered image-root, creating Hermit Image."); + // find kernel name + let konfig = fs::read_to_string(image_root.join(config::Config::DEFAULT_PATH))?; + let konfig: config::Config<'_> = toml::from_str(&konfig)?; + let kernel_name: &str = match &konfig { + config::Config::V1 { kernel, .. } => kernel, + }; + + let tar_artifact_path = build_artifact.with_extension("tar.gz"); + let mut tar_artifact = tar::Builder::new(flate2::write::GzEncoder::new( + fs::File::create(&tar_artifact_path)?, + flate2::Compression::default(), + )); + tar_artifact.mode(tar::HeaderMode::Deterministic); + + // NOTE: use tar ustar to create the image. + + // add kernel + eprintln!("- {kernel_name}"); + { + let mut header = tar::Header::new_ustar(); + let kernel_meta = fs::metadata(&build_artifact)?; + header.set_path(kernel_name).unwrap(); + header.set_size(kernel_meta.len()); + header.set_cksum(); + + tar_artifact.append(&header, fs::File::open(&build_artifact)?)?; + } + + // add rest + for entry in walkdir::WalkDir::new(&image_root) { + let entry = entry?; + let entry_rel_path = entry.path().strip_prefix(&image_root)?; + eprintln!("- {}", entry_rel_path.display()); + if entry_rel_path == Path::new(kernel_name) || entry.metadata()?.is_dir() { + continue; + } + + tar_artifact.append_path_with_name(entry.path(), entry_rel_path)?; + } + + tar_artifact.into_inner()?.finish()?.sync_all()?; + build_artifact = tar_artifact_path; + } + if super::in_ci() { eprintln!("::endgroup::"); } - Ok(self.cargo_build.artifact.ci_image(&self.package)) + Ok(build_artifact) } }