Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@cgwalters
cgwalters committed Oct 20, 2023
1 parent be85997 commit ce9da51
Showing 1 changed file with 113 additions and 68 deletions.
181 changes: 113 additions & 68 deletions lib/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,20 @@ use anyhow::{anyhow, Context, Result};
use camino::Utf8Path;
use cap_std::fs::Dir;
use cap_std_ext::cap_std;
use containers_image_proxy::ImageProxy;
use fn_error_context::context;
use ostree_ext::container as ostree_container;
use ostree_ext::oci_spec;
use ostree_ext::prelude::{Cast, FileExt, InputStreamExtManual, ToVariant};
use ostree_ext::{gio, glib, ostree};
use ostree_ext::{ostree::Deployment, sysroot::SysrootLock};
use rustix::fd::AsRawFd;
use tokio::io::AsyncReadExt;

/// The media type of a configmap stored in a registry as an OCI artifact
const MEDIA_TYPE_CONFIGMAP: &str = "application/containers.configmap+json";

const CONFIGMAP_SIZE_LIMIT: u32 = 1_048_576;

/// The prefix used to store configmaps
const REF_PREFIX: &str = "bootc/config";
Expand All @@ -35,11 +42,16 @@ const ORIGIN_BOOTC_CONFIG_PREFIX: &str = "bootc.config.";
/// The serialized metadata about configmaps attached to a deployment
pub(crate) struct ConfigSpec {
pub(crate) name: String,
pub(crate) url: String,
pub(crate) imgref: ostree_container::ImageReference,
}

pub(crate) struct ConfigMapObject {
manifest: oci_spec::image::ImageManifest,
config: ConfigMap,
}

impl ConfigSpec {
const KEY_URL: &str = "url";
const KEY_IMAGE: &str = "imageref";

/// Return the keyfile group name
fn group(name: &str) -> String {
Expand All @@ -50,9 +62,10 @@ impl ConfigSpec {
#[context("Parsing config spec")]
fn from_keyfile(kf: &glib::KeyFile, name: &str) -> Result<Self> {
let group = Self::group(name);
let url = kf.string(&group, Self::KEY_URL)?.to_string();
let imgref = kf.string(&group, Self::KEY_IMAGE)?;
let imgref = imgref.as_str().try_into()?;
Ok(Self {
url,
imgref,
name: name.to_string(),
})
}
Expand All @@ -62,7 +75,7 @@ impl ConfigSpec {
let group = &Self::group(&self.name);
// Ignore errors if the group didn't exist
let _ = kf.remove_group(group);
kf.set_string(group, Self::KEY_URL, &self.url);
kf.set_string(group, Self::KEY_IMAGE, &self.imgref.to_string());
}

/// Remove this config from the target; returns `true` if the value was present
Expand All @@ -80,9 +93,13 @@ impl ConfigSpec {
#[derive(Debug, clap::Subcommand)]
pub(crate) enum ConfigOpts {
/// Add a remote configmap
AddFromURL {
/// Remote URL for configmap
url: String,
Add {
/// Container registry pull specification; this must refer to an OCI artifact
imgref: String,

/// The transport; e.g. oci, oci-archive. Defaults to `registry`.
#[clap(long, default_value = "registry")]
transport: String,

#[clap(long)]
/// Provide an explicit name for the map
Expand Down Expand Up @@ -112,15 +129,26 @@ pub(crate) async fn run(opts: ConfigOpts) -> Result<()> {
crate::cli::prepare_for_write().await?;
let sysroot = &crate::cli::get_locked_sysroot().await?;
match opts {
ConfigOpts::AddFromURL { url, name } => add_from_url(sysroot, &url, name.as_deref()).await,
ConfigOpts::Add {
imgref,
transport,
name,
} => {
let transport = ostree_container::Transport::try_from(transport.as_str())?;
let imgref = ostree_container::ImageReference {
transport,
name: imgref,
};
add(sysroot, &imgref, name.as_deref()).await
}
ConfigOpts::Remove { name } => remove(sysroot, name.as_str()).await,
ConfigOpts::Update { names } => update(sysroot, names.into_iter()).await,
ConfigOpts::Show { name } => show(sysroot, &name).await,
ConfigOpts::List => list(sysroot).await,
}
}

async fn new_proxy() -> Result<containers_image_proxy::ImageProxy> {
async fn new_proxy() -> Result<ImageProxy> {
let mut config = containers_image_proxy::ImageProxyConfig::default();
ostree_container::merge_default_container_proxy_opts(&mut config);
containers_image_proxy::ImageProxy::new_with_config(config).await
Expand All @@ -147,9 +175,7 @@ async fn list(sysroot: &SysrootLock) -> Result<()> {
println!("No dynamic ConfigMap objects attached");
} else {
for config in configs {
let name = config.name;
let url = config.url;
println!("{name} {url}");
println!("{} {}", config.name.as_str(), config.imgref);
}
}
Ok(())
Expand Down Expand Up @@ -226,8 +252,7 @@ fn write_configmap(
sysroot: &SysrootLock,
sepolicy: Option<&ostree::SePolicy>,
spec: &ConfigSpec,
manifest: &oci_spec::image::ImageManifest,
map: &ConfigMap,
cfgobj: ConfigMapObject,
etag: Option<&str>,
cancellable: Option<&gio::Cancellable>,
) -> Result<()> {
Expand All @@ -241,7 +266,7 @@ fn write_configmap(
let dirmeta =
create_and_commit_dirmeta(&repo, "/etc/some-unshipped-config-file".into(), sepolicy)?;
{
let serialized = serde_json::to_string(map).context("Serializing")?;
let serialized = serde_json::to_string(&cfgobj.config).context("Serializing")?;
write_file(
repo,
tree,
Expand All @@ -253,9 +278,11 @@ fn write_configmap(
)?;
}
let mut metadata = HashMap::new();
let serialized_manifest = serde_json::to_string(manifest).context("Serializing manifest")?;
let serialized_manifest =
serde_json::to_string(&cfgobj.manifest).context("Serializing manifest")?;
metadata.insert(CONFIGMAP_MANIFEST_KEY, serialized_manifest.to_variant());
let timestamp = manifest
let timestamp = cfgobj
.manifest
.annotations()
.and_then(|m| m.get(oci_spec::image::ANNOTATION_CREATED))
.map(|v| chrono::DateTime::parse_from_rfc3339(v))
Expand All @@ -282,49 +309,71 @@ fn write_configmap(
Ok(())
}

#[context("Fetching configmap from {url}")]
/// Download a configmap, honoring an optional ETag. If the server says the resource
/// is unmodified, this returns `Ok(None)`.
async fn fetch_configmap(
client: &reqwest::Client,
url: &str,
etag: Option<&str>,
) -> Result<Option<HttpCachableReply<ConfigMap>>> {
tracing::debug!("Fetching {url}");
let mut req = client.get(url);
if let Some(etag) = etag {
tracing::trace!("Providing etag {etag}");
let val = HeaderValue::from_str(etag).context("Parsing etag")?;
req = req.header(reqwest::header::IF_NONE_MATCH, val);
/// Parse a manifest, returning the single configmap descriptor (layer)
fn configmap_object_from_manifest(
manifest: &oci_spec::image::ImageManifest,
) -> Result<&oci_spec::image::Descriptor> {
let l = match manifest.layers().as_slice() {
[] => anyhow::bail!("No layers in configmap manifest"),
[l] => l,
o => anyhow::bail!(
"Expected exactly one layer in configmap manifest, found: {}",
o.len()
),
};
match l.media_type() {
oci_spec::image::MediaType::Other(o) if o.as_str() == MEDIA_TYPE_CONFIGMAP => Ok(l),
o => anyhow::bail!("Expected media type {MEDIA_TYPE_CONFIGMAP} but found: {o}"),
}
let reply = req.send().await?;
if reply.status() == StatusCode::NOT_MODIFIED {
tracing::debug!("Server returned NOT_MODIFIED");
}

#[context("Fetching configmap from {imgref}")]
/// Download a configmap, honoring a previous manifest digest. If the digest
/// hasn't changed, then this function will return None.
async fn fetch_configmap(
proxy: &ImageProxy,
imgref: &ostree_container::ImageReference,
previous_manifest_digest: Option<&str>,
) -> Result<Option<Box<ConfigMapObject>>> {
tracing::debug!("Fetching {imgref}");
let imgref = imgref.to_string();
let oimg = proxy.open_image(&imgref).await?;
let (digest, manifest) = proxy.fetch_manifest(&oimg).await?;
if previous_manifest_digest == Some(digest.as_str()) {
return Ok(None);
}
let etag = reply
.headers()
.get(reqwest::header::ETAG)
.map(|v| v.to_str())
.transpose()
.context("Parsing etag")?
.map(ToOwned::to_owned);
// TODO: streaming deserialize
let buf = reply.bytes().await?;
tracing::trace!("Parsing server reply of {} bytes", buf.len());
serde_yaml::from_slice(&buf)
.context("Deserializing configmap")
.map(|v| Some(HttpCachableReply { content: v, etag }))
let layer = configmap_object_from_manifest(&manifest)?;
// Layer sizes shouldn't be negative
let layer_size = u64::try_from(layer.size()).unwrap();
let layer_size = u32::try_from(layer_size)?;
if layer_size > CONFIGMAP_SIZE_LIMIT {
anyhow::bail!(
"configmap size limit is {CONFIGMAP_SIZE_LIMIT} bytes, found: {}",
glib::format_size(layer_size.into())
)
}
let (mut configmap_reader, driver) = proxy
.get_blob(&oimg, layer.digest(), layer_size.into())
.await?;
let mut configmap_blob = String::new();
let reader = configmap_reader.read_to_string(&mut configmap_blob);
let (reader, driver) = tokio::join!(reader, driver);
let _ = reader?;
driver?;

let config: ConfigMap = serde_json::from_str(&configmap_blob).context("Parsing configmap")?;
Ok(Some(Box::new(ConfigMapObject { manifest, config })))
}

/// Download a configmap.
async fn fetch_required_configmap(
client: &reqwest::Client,
url: &str,
) -> Result<HttpCachableReply<ConfigMap>> {
fetch_configmap(client, url, None)
.await?
.ok_or_else(|| anyhow::anyhow!("Server unexpectedly returned unmodified status"))
proxy: &containers_image_proxy::ImageProxy,
imgref: &ostree_container::ImageReference,
) -> Result<Box<ConfigMapObject>> {
// SAFETY: We must get a new configmap here
fetch_configmap(proxy, imgref, None)
.await
.map(|v| v.expect("internal error: expected configmap"))
}

/// Return the attached configmaps for a deployment.
Expand All @@ -349,14 +398,17 @@ pub(crate) fn configs_for_deployment(
})
}

async fn add_from_url(sysroot: &SysrootLock, url: &str, name: Option<&str>) -> Result<()> {
async fn add(
sysroot: &SysrootLock,
imgref: &ostree_container::ImageReference,
name: Option<&str>,
) -> Result<()> {
let cancellable = gio::Cancellable::NONE;
let repo = &sysroot.repo();
let merge_deployment = &crate::cli::target_deployment(sysroot)?;
let stateroot = merge_deployment.osname();
let client = crate::utils::new_http_client().build()?;
let reply = fetch_required_configmap(&client, url).await?;
let configmap = reply.content;
let importer = new_proxy().await?;
let cfgobj = fetch_required_configmap(&importer, imgref).await?;
let origin = merge_deployment
.origin()
.ok_or_else(|| anyhow::anyhow!("Deployment is missing an origin"))?;
Expand All @@ -367,28 +419,21 @@ async fn add_from_url(sysroot: &SysrootLock, url: &str, name: Option<&str>) -> R
.with_context(|| format!("Opening deployment directory {dirpath:?}"))?;
let sepolicy = ostree::SePolicy::new_at(deployment_fd.as_raw_fd(), cancellable)?;
let name = name
.or_else(|| configmap.metadata.name.as_deref())
.or_else(|| cfgobj.config.metadata.name.as_deref())
.ok_or_else(|| anyhow!("Missing metadata.name and no name provided"))?;
let configs = configs_for_deployment(sysroot, merge_deployment)?;
if configs.iter().any(|v| v.name == name) {
anyhow::bail!("Already have a config with name {name}");
}
let spec = ConfigSpec {
name: name.to_owned(),
url: url.to_owned(),
imgref: imgref.clone(),
};
let oref = name_to_ostree_ref(name)?;
tracing::trace!("configmap {name} => {oref}");
// TODO use ostree_ext::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| {
// once https://github.com/ostreedev/ostree/pull/2824 lands
write_configmap(
sysroot,
Some(&sepolicy),
&spec,
&configmap,
reply.etag.as_deref(),
cancellable,
)?;
write_configmap(sysroot, Some(&sepolicy), &spec, &configmap, cancellable)?;
println!("Stored configmap: {name}");

spec.store(&origin);
Expand Down

0 comments on commit ce9da51

Please sign in to comment.