From d0455106f75d60688db830572646a92b6c969d35 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 9 Nov 2024 20:23:19 +0000 Subject: [PATCH] wip: Install with fsverity Signed-off-by: Colin Walters --- .github/workflows/ci.yml | 10 ++- Cargo.lock | 74 ++++++++++++++-- ci/Containerfile.install-fsverity | 10 +++ lib/Cargo.toml | 2 + lib/src/cli.rs | 26 ++++++ lib/src/fsck.rs | 128 +++++++++++++++++++++++++++ lib/src/install.rs | 32 +++++-- lib/src/install/baseline.rs | 20 +++-- lib/src/install/config.rs | 59 ++++++++++++ lib/src/lib.rs | 1 + lib/src/store/mod.rs | 2 + ostree-ext/src/ostree_prepareroot.rs | 6 +- tests-integration/src/install.rs | 1 + 13 files changed, 348 insertions(+), 23 deletions(-) create mode 100644 ci/Containerfile.install-fsverity create mode 100644 lib/src/fsck.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a727a4ab6..2cf535640 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,10 +69,17 @@ jobs: uses: actions/checkout@v4 - name: Free up disk space on runner run: sudo ./ci/clean-gha-runner.sh + - name: Enable fsverity for / + run: sudo tune2fs -O verity $(findmnt -vno SOURCE /) + - name: Install utils + run: sudo apt -y install fsverity - name: Integration tests run: | set -xeu + # Build images to test; TODO investigate doing single container builds + # via GHA and pushing to a temporary registry to share among workflows? sudo podman build -t localhost/bootc -f hack/Containerfile . + sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity export CARGO_INCREMENTAL=0 # because we aren't caching the test runner bits cargo build --release -p tests-integration df -h / @@ -84,8 +91,9 @@ jobs: -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/ostree-ext/ci/priv-integration.sh # Nondestructive but privileged tests sudo bootc-integration-tests host-privileged localhost/bootc - # Finally the install-alongside suite + # Install tests sudo bootc-integration-tests install-alongside localhost/bootc + sudo bootc-integration-tests install-fsverity localhost/bootc-fsverity docs: if: ${{ contains(github.event.pull_request.labels.*.name, 'documentation') }} runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index bf55dcb5f..ab4d35fdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,19 @@ version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +[[package]] +name = "async-compression" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -177,6 +190,7 @@ dependencies = [ "clap", "clap_mangen", "comfy-table", + "composefs", "fn-error-context", "hex", "indicatif", @@ -423,6 +437,29 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "composefs" +version = "0.2.0" +source = "git+https://github.com/containers/composefs-rs?rev=55ae2e9ba72f6afda4887d746e6b98f0a1875ac4#55ae2e9ba72f6afda4887d746e6b98f0a1875ac4" +dependencies = [ + "anyhow", + "async-compression", + "clap", + "containers-image-proxy", + "hex", + "indicatif", + "oci-spec", + "regex-automata 0.4.9", + "rustix", + "sha2", + "tar", + "tempfile", + "thiserror 2.0.6", + "tokio", + "zerocopy 0.8.13", + "zstd", +] + [[package]] name = "console" version = "0.15.8" @@ -1025,6 +1062,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", + "tokio", "unicode-width 0.2.0", "web-time", ] @@ -1318,7 +1356,7 @@ dependencies = [ "serde_json", "strum", "strum_macros", - "thiserror 2.0.3", + "thiserror 2.0.6", ] [[package]] @@ -1552,7 +1590,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -2123,11 +2161,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.6", ] [[package]] @@ -2143,9 +2181,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", @@ -2691,7 +2729,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67914ab451f3bfd2e69e5e9d2ef3858484e7074d63f204fd166ec391b54de21d" +dependencies = [ + "zerocopy-derive 0.8.13", ] [[package]] @@ -2705,6 +2752,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7988d73a4303ca289df03316bc490e934accf371af6bc745393cf3c2c5c4f25d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/ci/Containerfile.install-fsverity b/ci/Containerfile.install-fsverity new file mode 100644 index 000000000..7dc34f545 --- /dev/null +++ b/ci/Containerfile.install-fsverity @@ -0,0 +1,10 @@ +# Enable fsverity at install time +FROM localhost/bootc +RUN < /usr/lib/bootc/install/30-fsverity.toml < All .file objects have fsverity + /// "disabled" => No .file objects have fsverity + /// "inconsistent" => Mixed state + OstreeVerity, +} + /// Hidden, internal only options #[derive(Debug, clap::Subcommand, PartialEq, Eq)] pub(crate) enum InternalsOpts { @@ -344,6 +354,8 @@ pub(crate) enum InternalsOpts { FixupEtcFstab, /// Should only be used by `make update-generated` PrintJsonSchema, + /// Perform consistency checking. + Fsck, /// Perform cleanup actions Cleanup, /// Proxy frontend for the `ostree-ext` CLI. @@ -1051,6 +1063,20 @@ async fn run_from_opt(opt: Opt) -> Result<()> { ) .await } + InternalsOpts::Fsck => { + let storage = get_storage().await?; + let r = crate::fsck::fsck(&storage).await?; + match r.errors.as_slice() { + [] => {} + errs => { + for err in errs { + eprintln!("error: {err}"); + } + anyhow::bail!("fsck found errors"); + } + } + Ok(()) + } InternalsOpts::FixupEtcFstab => crate::deploy::fixup_etc_fstab(&root), InternalsOpts::PrintJsonSchema => { let schema = schema_for!(crate::spec::Host); diff --git a/lib/src/fsck.rs b/lib/src/fsck.rs new file mode 100644 index 000000000..64df5528b --- /dev/null +++ b/lib/src/fsck.rs @@ -0,0 +1,128 @@ +//! # Write deployments merging image with configmap +//! +//! Create a merged filesystem tree with the image and mounted configmaps. + +use std::os::fd::AsFd; +use std::str::FromStr as _; + +use anyhow::Ok; +use anyhow::{Context, Result}; +use camino::Utf8PathBuf; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use fn_error_context::context; +use ostree_ext::keyfileext::KeyFileExt; +use ostree_ext::ostree; +use serde::{Deserialize, Serialize}; + +use crate::install::config::Tristate; +use crate::store::{self, Storage}; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum VerityState { + Enabled, + Disabled, + Inconsistent, +} + +#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub(crate) struct FsckResult { + pub(crate) notices: Vec, + pub(crate) errors: Vec, + pub(crate) verity: Option, +} + +/// Check the fsverity state of all regular files in this object directory. +#[context("Computing verity state")] +fn verity_state_of_objects(d: &Dir) -> Result<(u64, u64)> { + let mut enabled = 0; + let mut disabled = 0; + for ent in d.entries()? { + let ent = ent?; + if !ent.file_type()?.is_file() { + continue; + } + let name = ent.file_name(); + let name = name + .into_string() + .map(Utf8PathBuf::from) + .map_err(|_| anyhow::anyhow!("Invalid UTF-8"))?; + let Some("file") = name.extension() else { + continue; + }; + let f = d + .open(&name) + .with_context(|| format!("Failed to open {name}"))?; + let r: Option = + composefs::fsverity::ioctl::fs_ioc_measure_verity(f.as_fd())?; + drop(f); + if r.is_some() { + enabled += 1; + } else { + disabled += 1; + } + } + Ok((enabled, disabled)) +} + +async fn verity_state_of_all_objects(repo: &ostree::Repo) -> Result<(u64, u64)> { + const MAX_CONCURRENT: usize = 3; + + let repodir = Dir::reopen_dir(&repo.dfd_borrow())?; + + let mut joinset = tokio::task::JoinSet::new(); + let mut results = Vec::new(); + + for ent in repodir.read_dir("objects")? { + while joinset.len() >= MAX_CONCURRENT { + results.push(joinset.join_next().await.unwrap()??); + } + let ent = ent?; + if !ent.file_type()?.is_dir() { + continue; + } + let objdir = ent.open_dir()?; + joinset.spawn_blocking(move || verity_state_of_objects(&objdir)); + } + + while let Some(output) = joinset.join_next().await { + results.push(output??); + } + let r = results.into_iter().fold((0, 0), |mut acc, v| { + acc.0 += v.0; + acc.1 += v.1; + acc + }); + Ok(r) +} + +pub(crate) async fn fsck(storage: &Storage) -> Result { + let mut r = FsckResult::default(); + + let repo_config = storage.repo().config(); + let verity_state = { + let (k, v) = store::REPO_VERITY_CONFIG.split_once('.').unwrap(); + repo_config + .optional_string(k, v)? + .map(|v| Tristate::from_str(&v)) + .transpose()? + .unwrap_or_default() + }; + + r.verity = match verity_state_of_all_objects(&storage.repo()).await? { + (0, 0) => None, + (_, 0) => Some(VerityState::Enabled), + (0, _) => Some(VerityState::Disabled), + _ => Some(VerityState::Inconsistent), + }; + if matches!(&r.verity, &Some(VerityState::Inconsistent)) { + let inconsistent = "Inconsistent fsverity state".to_string(); + match verity_state { + Tristate::Disabled | Tristate::Maybe => r.notices.push(inconsistent), + Tristate::Enabled => r.errors.push(inconsistent), + } + } + serde_json::to_writer(std::io::stdout().lock(), &r)?; + Ok(r) +} diff --git a/lib/src/install.rs b/lib/src/install.rs index 548c3e8d3..7cc00760d 100644 --- a/lib/src/install.rs +++ b/lib/src/install.rs @@ -33,6 +33,7 @@ use cap_std_ext::cmdext::CapStdExtCommandExt; use cap_std_ext::prelude::CapStdExtDirExt; use chrono::prelude::*; use clap::ValueEnum; +use config::Tristate; use fn_error_context::context; use ostree::gio; use ostree_ext::container as ostree_container; @@ -71,6 +72,15 @@ const SELINUXFS: &str = "/sys/fs/selinux"; const EFIVARFS: &str = "/sys/firmware/efi/efivars"; pub(crate) const ARCH_USES_EFI: bool = cfg!(any(target_arch = "x86_64", target_arch = "aarch64")); +const DEFAULT_REPO_CONFIG: &[(&str, &str)] = &[ + // Default to avoiding grub2-mkconfig etc. + ("sysroot.bootloader", "none"), + // Always flip this one on because we need to support alongside installs + // to systems without a separate boot partition. + ("sysroot.bootprefix", "true"), + ("sysroot.readonly", "true"), +]; + /// Kernel argument used to specify we want the rootfs mounted read-write by default const RW_KARG: &str = "rw"; @@ -600,6 +610,7 @@ pub(crate) fn print_configuration() -> Result<()> { #[context("Creating ostree deployment")] async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result<(Storage, bool)> { + let install_config = state.install_config.as_ref(); let sepolicy = state.load_policy()?; let sepolicy = sepolicy.as_ref(); // Load a fd for the mounted target physical root @@ -632,14 +643,19 @@ async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result crate::lsm::ensure_dir_labeled(rootfs_dir, "boot", None, 0o755.into(), sepolicy)?; } - for (k, v) in [ - // Default to avoiding grub2-mkconfig etc. - ("sysroot.bootloader", "none"), - // Always flip this one on because we need to support alongside installs - // to systems without a separate boot partition. - ("sysroot.bootprefix", "true"), - ("sysroot.readonly", "true"), - ] { + let fsverity = install_config + .and_then(|c| c.fsverity.clone()) + .unwrap_or_default(); + let fsverity_ostree_key = crate::store::REPO_VERITY_CONFIG; + let fsverity_ostree_opt = match fsverity { + Tristate::Disabled => None, + Tristate::Maybe => Some((fsverity_ostree_key, "maybe")), + Tristate::Enabled => Some((fsverity_ostree_key, "yes")), + }; + for (k, v) in DEFAULT_REPO_CONFIG + .iter() + .chain(fsverity_ostree_opt.as_ref()) + { Command::new("ostree") .args(["config", "--repo", "ostree/repo", "set", k, v]) .cwd_dir(rootfs_dir.try_clone()?) diff --git a/lib/src/install/baseline.rs b/lib/src/install/baseline.rs index 5db2e221b..8a4b471e6 100644 --- a/lib/src/install/baseline.rs +++ b/lib/src/install/baseline.rs @@ -24,6 +24,7 @@ use serde::{Deserialize, Serialize}; use super::MountSpec; use super::RootSetup; use super::State; +use super::Tristate; use super::RUN_BOOTC; use super::RW_KARG; use crate::mount; @@ -147,13 +148,12 @@ pub(crate) fn install_create_rootfs( state: &State, opts: InstallBlockDeviceOpts, ) -> Result { + let install_config = state.install_config.as_ref(); let luks_name = "root"; // Ensure we have a root filesystem upfront let root_filesystem = opts .filesystem - .or(state - .install_config - .as_ref() + .or(install_config .and_then(|c| c.filesystem_root()) .and_then(|r| r.fstype)) .ok_or_else(|| anyhow::anyhow!("No root filesystem specified"))?; @@ -192,7 +192,7 @@ pub(crate) fn install_create_rootfs( } // Use the install configuration to find the block setup, if we have one - let block_setup = if let Some(config) = state.install_config.as_ref() { + let block_setup = if let Some(config) = install_config { config.get_block_setup(opts.block_setup.as_ref().copied())? } else if opts.filesystem.is_some() { // Otherwise, if a filesystem is specified then we default to whatever was @@ -371,8 +371,18 @@ pub(crate) fn install_create_rootfs( None }; + let fsverity = install_config + .and_then(|c| c.fsverity.clone()) + .unwrap_or_default(); + let mkfs_options = match (root_filesystem, fsverity) { + (Filesystem::Ext4, Tristate::Enabled | Tristate::Maybe) => ["-O", "verity"].as_slice(), + _ => [].as_slice(), + } + .iter() + .copied(); + // Initialize rootfs - let root_uuid = mkfs(&rootdev, root_filesystem, "root", opts.wipe, [])?; + let root_uuid = mkfs(&rootdev, root_filesystem, "root", opts.wipe, mkfs_options)?; let rootarg = format!("root=UUID={root_uuid}"); let bootsrc = boot_uuid.as_ref().map(|uuid| format!("UUID={uuid}")); let bootarg = bootsrc.as_deref().map(|bootsrc| format!("boot={bootsrc}")); diff --git a/lib/src/install/config.rs b/lib/src/install/config.rs index b0fe3e76a..1b1a7916d 100644 --- a/lib/src/install/config.rs +++ b/lib/src/install/config.rs @@ -41,6 +41,36 @@ pub(crate) struct BasicFilesystems { // pub(crate) esp: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub(crate) enum Tristate { + #[default] + // The feature is disabled + #[serde(alias = "no", alias = "false")] + Disabled, + // The feature is enabled if supported + #[serde(alias = "maybe")] + Maybe, + // The feature is enabled + #[serde(alias = "yes", alias = "true")] + Enabled, +} + +impl std::str::FromStr for Tristate { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let r = match s { + // Keep this in sync with ot_keyfile_get_tristate_with_default from ostree + "yes" | "true" | "1" => Tristate::Enabled, + "no" | "false" | "0" => Tristate::Disabled, + "maybe" => Tristate::Maybe, + o => anyhow::bail!("Invalid tristate value: {o}"), + }; + Ok(r) + } +} + /// The serialized [install] section #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename = "install", rename_all = "kebab-case", deny_unknown_fields)] @@ -50,6 +80,8 @@ pub(crate) struct InstallConfiguration { /// Enabled block storage configurations pub(crate) block: Option>, pub(crate) filesystem: Option, + /// How we should use fsverity. + pub(crate) fsverity: Option, /// Kernel arguments, applied at installation time #[serde(skip_serializing_if = "Option::is_none")] pub(crate) kargs: Option>, @@ -113,6 +145,7 @@ impl Mergeable for InstallConfiguration { { merge_basic(&mut self.root_fs_type, other.root_fs_type, env); merge_basic(&mut self.block, other.block, env); + merge_basic(&mut self.fsverity, other.fsverity, env); self.filesystem.merge(other.filesystem, env); if let Some(other_kargs) = other.kargs { self.kargs @@ -550,3 +583,29 @@ root-fs-type = "xfs" ) ); } + +#[test] +/// Test parsing fsverity +fn test_fsverity() { + let env = EnvProperties { + sys_arch: "aarch64".to_string(), + }; + let mut c: InstallConfigurationToplevel = toml::from_str( + r##"[install] +root-fs-type = "xfs" +fsverity = "enabled" +"##, + ) + .unwrap(); + let install = c.install.as_ref().unwrap(); + assert_eq!(install.fsverity.as_ref().unwrap(), &Tristate::Enabled); + let o: InstallConfigurationToplevel = toml::from_str( + r##"[install] +fsverity = "optional" +"##, + ) + .unwrap(); + c.install.merge(o.install, &env); + let install = c.install.as_ref().unwrap(); + assert_eq!(install.fsverity.as_ref().unwrap(), &Tristate::Maybe); +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 1f0c263b5..007cc0f0e 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -7,6 +7,7 @@ mod boundimage; pub mod cli; pub(crate) mod deploy; +pub(crate) mod fsck; pub(crate) mod generator; mod image; pub(crate) mod journal; diff --git a/lib/src/store/mod.rs b/lib/src/store/mod.rs index 56e142e8b..06ccaf260 100644 --- a/lib/src/store/mod.rs +++ b/lib/src/store/mod.rs @@ -15,6 +15,8 @@ use crate::spec::ImageStatus; mod ostree_container; +pub(crate) const REPO_VERITY_CONFIG: &str = "ex-integrity.fsverity"; + pub(crate) struct Storage { pub sysroot: SysrootLock, run: Dir, diff --git a/ostree-ext/src/ostree_prepareroot.rs b/ostree-ext/src/ostree_prepareroot.rs index bd2dc3f1b..d84ae654e 100644 --- a/ostree-ext/src/ostree_prepareroot.rs +++ b/ostree-ext/src/ostree_prepareroot.rs @@ -47,10 +47,14 @@ pub(crate) fn overlayfs_root_enabled(root: &ostree::RepoFile) -> Result { } } +/// An option which can be enabled, disabled, or possibly enabled. #[derive(Debug, PartialEq, Eq)] -enum Tristate { +pub enum Tristate { + /// Enabled Enabled, + /// Disabled Disabled, + /// Maybe Maybe, } diff --git a/tests-integration/src/install.rs b/tests-integration/src/install.rs index c4eb31dd9..ad84df3b5 100644 --- a/tests-integration/src/install.rs +++ b/tests-integration/src/install.rs @@ -138,6 +138,7 @@ pub(crate) fn run_alongside(image: &str, mut testargs: libtest_mimic::Arguments) "grep authorized_keys etc/tmpfiles.d/bootc-root-ssh.conf" ) .run()?; + drop(cwd); Ok(()) },