deploy: Don't recompute verity checksums if not enabled

This fixes a truly horrific performance bug when
composefs is enabled, but fsverity is not supported
by the filesystem. We'd fall back to doing *userspace*
checksumming of all files at deployment time which was absolutely
not expected or required.

There's really an immense amount of technical debt
here, such as the confusion between `ex-integity.composefs`
vs the prepare-root config, how we handle "torn" states
where some objects don't have verity enabled but some do,
etc.

The ostree composefs state has two modes:

- signed: We need to enforce fsverity
- unsigned: Best effort resilience

So we fix this by making the deploy path to make verity
"opportunistic" - if the ioctl gives us the data, then we
add it to the composefs.

However, this code path is also invoked when we're
computing the expected composefs digest to inject
as commit metadata, and *that* API must work regardless
of whether the target repo has fsverity enabled as
it may operate on a build server.

One lucky thing in all of this: When I went to add
the "checkout composefs" API I added a stub `GVariant`
for options extensibility, which we now use.

Signed-off-by: Colin Walters <[email protected]>

man/ostree-checkout.xml

@@ -211,6 +211,24 @@ License along with this library. If not, see <https://www.gnu.org/licenses/>.
                     (may be /). This implies <literal>--force-copy</literal>.
                 </para></listitem>
             </varlistentry>
+
+            <varlistentry>
+                <term><option>--composefs</option></term>
+
+                <listitem><para>
+                    Only generate a composefs, not a directory.
+                </para></listitem>
+            </varlistentry>
+
+            <varlistentry>
+                <term><option>--composefs-noverity</option></term>
+
+                <listitem><para>
+                    Only generate a composefs, not a directory; fsverity digests
+                    will not be included. This is best used for "opportunistic"
+                    use of composefs.
+                </para></listitem>
+            </varlistentry>
         </variablelist>
     </refsect1>
 

src/libostree/ostree-repo-checkout.c

@@ -1273,23 +1273,50 @@ compare_verity_digests (GVariant *metadata_composefs, const guchar *fsverity_dig
 /**
  * ostree_repo_checkout_composefs:
  * @self: A repo
- * @options: (nullable): Future expansion space; must currently be %NULL
+ * @options: (nullable): If non-NULL, must be a GVariant of type a{sv}. See below.
  * @destination_dfd: Parent directory fd
  * @destination_path: Filename
  * @checksum: OStree commit digest
  * @cancellable: Cancellable
  * @error: Error
  *
- * Create a composefs filesystem metadata blob from an OSTree commit.
+ * Create a composefs filesystem metadata blob from an OSTree commit. Supported
+ * options:
+ *
+ *  - verity: `u`: 0 = disabled, 1 = set if present on file, 2 = enabled; any other value is a fatal
+ * error
  */
 gboolean
 ostree_repo_checkout_composefs (OstreeRepo *self, GVariant *options, int destination_dfd,
                                 const char *destination_path, const char *checksum,
                                 GCancellable *cancellable, GError **error)
 {
 #ifdef HAVE_COMPOSEFS
-  /* Force this for now */
-  g_assert (options == NULL);
+  OtTristate verity = OT_TRISTATE_YES;
+
+  if (options != NULL)
+    {
+      g_auto (GVariantDict) options_dict;
+      g_variant_dict_init (&options_dict, options);
+      guint32 verity_v = 0;
+      if (g_variant_dict_lookup (&options_dict, "verity", "u", &verity_v))
+        {
+          switch (verity_v)
+            {
+            case 0:
+              verity = OT_TRISTATE_NO;
+              break;
+            case 1:
+              verity = OT_TRISTATE_MAYBE;
+              break;
+            case 2:
+              verity = OT_TRISTATE_YES;
+              break;
+            default:
+              g_assert_not_reached ();
+            }
+        }
+    }
 
   g_auto (GLnxTmpfile) tmpf = {
     0,
@@ -1311,8 +1338,8 @@ ostree_repo_checkout_composefs (OstreeRepo *self, GVariant *options, int destina
 
   g_autoptr (OstreeComposefsTarget) target = ostree_composefs_target_new ();
 
-  if (!_ostree_repo_checkout_composefs (self, target, (OstreeRepoFile *)commit_root, cancellable,
-                                        error))
+  if (!_ostree_repo_checkout_composefs (self, verity, target, (OstreeRepoFile *)commit_root,
+                                        cancellable, error))
     return FALSE;
 
   g_autofree guchar *fsverity_digest = NULL;

src/libostree/ostree-repo-composefs.c

@@ -265,9 +265,9 @@ _ostree_composefs_set_xattrs (struct lcfs_node_s *node, GVariant *xattrs, GCance
 }
 
 static gboolean
-checkout_one_composefs_file_at (OstreeRepo *repo, const char *checksum, struct lcfs_node_s *parent,
-                                const char *destination_name, GCancellable *cancellable,
-                                GError **error)
+checkout_one_composefs_file_at (OstreeRepo *repo, OtTristate verity, const char *checksum,
+                                struct lcfs_node_s *parent, const char *destination_name,
+                                GCancellable *cancellable, GError **error)
 {
   g_autoptr (GInputStream) input = NULL;
   g_autoptr (GVariant) xattrs = NULL;
@@ -320,32 +320,38 @@ checkout_one_composefs_file_at (OstreeRepo *repo, const char *checksum, struct l
       if (lcfs_node_set_payload (node, loose_path_buf) != 0)
         return glnx_throw_errno (error);
 
-      guchar *known_digest = NULL;
-
-#ifdef HAVE_LINUX_FSVERITY_H
-      /* First try to get the digest directly from the bare repo file.
-       * This is the typical case when we're pulled into the target
-       * system repo with verity on and are recreating the composefs
-       * image during deploy. */
-      char buf[sizeof (struct fsverity_digest) + OSTREE_SHA256_DIGEST_LEN];
-
-      if (G_IS_UNIX_INPUT_STREAM (input))
+      if (verity != OT_TRISTATE_NO)
         {
-          int content_fd = g_unix_input_stream_get_fd (G_UNIX_INPUT_STREAM (input));
-          struct fsverity_digest *d = (struct fsverity_digest *)&buf;
-          d->digest_size = OSTREE_SHA256_DIGEST_LEN;
-
-          if (ioctl (content_fd, FS_IOC_MEASURE_VERITY, d) == 0
-              && d->digest_size == OSTREE_SHA256_DIGEST_LEN
-              && d->digest_algorithm == FS_VERITY_HASH_ALG_SHA256)
-            known_digest = d->digest;
-        }
+#ifdef HAVE_LINUX_FSVERITY_H
+          /* First try to get the digest directly from the bare repo file.
+           * This is the typical case when we're pulled into the target
+           * system repo with verity on and are recreating the composefs
+           * image during deploy. */
+          char buf[sizeof (struct fsverity_digest) + OSTREE_SHA256_DIGEST_LEN];
+          guchar *known_digest = NULL;
+
+          if (G_IS_UNIX_INPUT_STREAM (input))
+            {
+              int content_fd = g_unix_input_stream_get_fd (G_UNIX_INPUT_STREAM (input));
+              struct fsverity_digest *d = (struct fsverity_digest *)&buf;
+              d->digest_size = OSTREE_SHA256_DIGEST_LEN;
+
+              if (ioctl (content_fd, FS_IOC_MEASURE_VERITY, d) == 0
+                  && d->digest_size == OSTREE_SHA256_DIGEST_LEN
+                  && d->digest_algorithm == FS_VERITY_HASH_ALG_SHA256)
+                known_digest = d->digest;
+            }
 #endif
 
-      if (known_digest)
-        lcfs_node_set_fsverity_digest (node, known_digest);
-      else if (lcfs_node_set_fsverity_from_content (node, input, _composefs_read_cb) != 0)
-        return glnx_throw_errno (error);
+          if (known_digest)
+            lcfs_node_set_fsverity_digest (node, known_digest);
+          else if (verity == OT_TRISTATE_YES)
+            {
+              // Only fall back to userspace computation if explicitly requested
+              if (lcfs_node_set_fsverity_from_content (node, input, _composefs_read_cb) != 0)
+                return glnx_throw_errno (error);
+            }
+        }
     }
 
   if (xattrs)
@@ -360,7 +366,7 @@ checkout_one_composefs_file_at (OstreeRepo *repo, const char *checksum, struct l
 }
 
 static gboolean
-checkout_composefs_recurse (OstreeRepo *self, const char *dirtree_checksum,
+checkout_composefs_recurse (OstreeRepo *self, OtTristate verity, const char *dirtree_checksum,
                             const char *dirmeta_checksum, struct lcfs_node_s *parent,
                             const char *name, GCancellable *cancellable, GError **error)
 {
@@ -422,8 +428,8 @@ checkout_composefs_recurse (OstreeRepo *self, const char *dirtree_checksum,
         char tmp_checksum[OSTREE_SHA256_STRING_LEN + 1];
         _ostree_checksum_inplace_from_bytes_v (contents_csum_v, tmp_checksum);
 
-        if (!checkout_one_composefs_file_at (self, tmp_checksum, directory, fname, cancellable,
-                                             error))
+        if (!checkout_one_composefs_file_at (self, verity, tmp_checksum, directory, fname,
+                                             cancellable, error))
           return glnx_prefix_error (error, "Processing %s", tmp_checksum);
       }
     contents_csum_v = NULL; /* iter_loop freed it */
@@ -453,8 +459,8 @@ checkout_composefs_recurse (OstreeRepo *self, const char *dirtree_checksum,
         _ostree_checksum_inplace_from_bytes_v (subdirtree_csum_v, subdirtree_checksum);
         char subdirmeta_checksum[OSTREE_SHA256_STRING_LEN + 1];
         _ostree_checksum_inplace_from_bytes_v (subdirmeta_csum_v, subdirmeta_checksum);
-        if (!checkout_composefs_recurse (self, subdirtree_checksum, subdirmeta_checksum, directory,
-                                         dname, cancellable, error))
+        if (!checkout_composefs_recurse (self, verity, subdirtree_checksum, subdirmeta_checksum,
+                                         directory, dname, cancellable, error))
           return FALSE;
       }
     /* Freed by iter-loop */
@@ -467,8 +473,9 @@ checkout_composefs_recurse (OstreeRepo *self, const char *dirtree_checksum,
 
 /* Begin a checkout process */
 static gboolean
-checkout_composefs_tree (OstreeRepo *self, OstreeComposefsTarget *target, OstreeRepoFile *source,
-                         GFileInfo *source_info, GCancellable *cancellable, GError **error)
+checkout_composefs_tree (OstreeRepo *self, OtTristate verity, OstreeComposefsTarget *target,
+                         OstreeRepoFile *source, GFileInfo *source_info, GCancellable *cancellable,
+                         GError **error)
 {
   if (g_file_info_get_file_type (source_info) != G_FILE_TYPE_DIRECTORY)
     return glnx_throw (error, "Root checkout of composefs must be directory");
@@ -483,8 +490,8 @@ checkout_composefs_tree (OstreeRepo *self, OstreeComposefsTarget *target, Ostree
 
   const char *dirtree_checksum = ostree_repo_file_tree_get_contents_checksum (source);
   const char *dirmeta_checksum = ostree_repo_file_tree_get_metadata_checksum (source);
-  return checkout_composefs_recurse (self, dirtree_checksum, dirmeta_checksum, target->dest, "root",
-                                     cancellable, error);
+  return checkout_composefs_recurse (self, verity, dirtree_checksum, dirmeta_checksum, target->dest,
+                                     "root", cancellable, error);
 }
 
 static struct lcfs_node_s *
@@ -515,6 +522,7 @@ ensure_lcfs_dir (struct lcfs_node_s *parent, const char *name, GError **error)
  * _ostree_repo_checkout_composefs:
  * @self: Repo
  * @target: A target for the checkout
+ * @verity: Use fsverity
  * @source: Source tree
  * @cancellable: Cancellable
  * @error: Error
@@ -530,7 +538,7 @@ ensure_lcfs_dir (struct lcfs_node_s *parent, const char *name, GError **error)
  * Returns: %TRUE on success, %FALSE on failure
  */
 gboolean
-_ostree_repo_checkout_composefs (OstreeRepo *self, OstreeComposefsTarget *target,
+_ostree_repo_checkout_composefs (OstreeRepo *self, OtTristate verity, OstreeComposefsTarget *target,
                                  OstreeRepoFile *source, GCancellable *cancellable, GError **error)
 {
 #ifdef HAVE_COMPOSEFS
@@ -545,7 +553,7 @@ _ostree_repo_checkout_composefs (OstreeRepo *self, OstreeComposefsTarget *target
   if (!target_info)
     return glnx_prefix_error (error, "Failed to query");
 
-  if (!checkout_composefs_tree (self, target, source, target_info, cancellable, error))
+  if (!checkout_composefs_tree (self, verity, target, source, target_info, cancellable, error))
     return FALSE;
 
   /* We need a root dir */
@@ -593,7 +601,11 @@ ostree_repo_commit_add_composefs_metadata (OstreeRepo *self, guint format_versio
 
   g_autoptr (OstreeComposefsTarget) target = ostree_composefs_target_new ();
 
-  if (!_ostree_repo_checkout_composefs (self, target, repo_root, cancellable, error))
+  // We unconditionally add the expected verity digest. Note that for repositories
+  // on filesystems without fsverity, this operation currently requires re-checksumming
+  // all objects.
+  if (!_ostree_repo_checkout_composefs (self, OT_TRISTATE_YES, target, repo_root, cancellable,
+                                        error))
     return FALSE;
 
   g_autofree guchar *fsverity_digest = NULL;

src/libostree/ostree-repo-private.h

@@ -473,9 +473,9 @@ gboolean ostree_composefs_target_write (OstreeComposefsTarget *target, int fd,
                                         guchar **out_fsverity_digest, GCancellable *cancellable,
                                         GError **error);
 
-gboolean _ostree_repo_checkout_composefs (OstreeRepo *self, OstreeComposefsTarget *target,
-                                          OstreeRepoFile *source, GCancellable *cancellable,
-                                          GError **error);
+gboolean _ostree_repo_checkout_composefs (OstreeRepo *self, OtTristate verity,
+                                          OstreeComposefsTarget *target, OstreeRepoFile *source,
+                                          GCancellable *cancellable, GError **error);
 static inline gboolean
 composefs_not_supported (GError **error)
 {

src/libostree/ostree-sysroot-deploy.c

@@ -606,8 +606,8 @@ merge_configuration_from (OstreeSysroot *sysroot, OstreeDeployment *merge_deploy
  */
 static gboolean
 checkout_deployment_tree (OstreeSysroot *sysroot, OstreeRepo *repo, OstreeDeployment *deployment,
-                          const char *revision, int *out_deployment_dfd, GCancellable *cancellable,
-                          GError **error)
+                          const char *revision, int *out_deployment_dfd, guint64 *checkout_elapsed,
+                          guint64 *composefs_elapsed, GCancellable *cancellable, GError **error)
 {
   GLNX_AUTO_PREFIX_ERROR ("Checking out deployment tree", error);
   /* Find the directory with deployments for this stateroot */
@@ -630,14 +630,18 @@ checkout_deployment_tree (OstreeSysroot *sysroot, OstreeRepo *repo, OstreeDeploy
   /* Generate hardlink farm, then opendir it */
   OstreeRepoCheckoutAtOptions checkout_opts = { .process_passthrough_whiteouts = TRUE };
 
+  guint64 checkout_start_time = g_get_monotonic_time ();
   if (!ostree_repo_checkout_at (repo, &checkout_opts, osdeploy_dfd, checkout_target_name, csum,
                                 cancellable, error))
     return FALSE;
+  guint64 checkout_end_time = g_get_monotonic_time ();
 
   glnx_autofd int ret_deployment_dfd = -1;
   if (!glnx_opendirat (osdeploy_dfd, checkout_target_name, TRUE, &ret_deployment_dfd, error))
     return FALSE;
 
+  guint64 composefs_start_time = 0;
+  guint64 composefs_end_time = 0;
 #ifdef HAVE_COMPOSEFS
   /* TODO: Consider changing things in the future to parse the deployment config from memory, and
    * if composefs is enabled, then we can check out in "user mode" (i.e. only have suid binaries
@@ -665,14 +669,35 @@ checkout_deployment_tree (OstreeSysroot *sysroot, OstreeRepo *repo, OstreeDeploy
     composefs_enabled = repo->composefs_wanted;
   if (composefs_enabled == OT_TRISTATE_YES)
     {
-      if (!ostree_repo_checkout_composefs (repo, NULL, ret_deployment_dfd, OSTREE_COMPOSEFS_NAME,
-                                           csum, cancellable, error))
+      composefs_start_time = g_get_monotonic_time ();
+      // TODO: Clean up our mess around composefs/fsverity...we have duplication
+      // between the repo config and the sysroot config, *and* we need to better
+      // handle skew between repo config and repo state (e.g. "post-copy" should
+      // support transitioning verity on and off in general).
+      // For now we configure things such that the fsverity digest is only added
+      // if present on disk in the unsigned case, and in the signed case unconditionally
+      // require it.
+      g_auto (GVariantBuilder) cfs_checkout_opts_builder
+          = G_VARIANT_BUILDER_INIT (G_VARIANT_TYPE_VARDICT);
+      guint32 composefs_requested = 1;
+      if (composefs_config->is_signed)
+        composefs_requested = 2;
+      g_variant_builder_add (&cfs_checkout_opts_builder, "{sv}", "verity",
+                             g_variant_new_uint32 (composefs_requested));
+      g_debug ("composefs requested: %u", composefs_requested);
+      g_autoptr (GVariant) cfs_checkout_opts
+          = g_variant_ref_sink (g_variant_builder_end (&cfs_checkout_opts_builder));
+      if (!ostree_repo_checkout_composefs (repo, cfs_checkout_opts, ret_deployment_dfd,
+                                           OSTREE_COMPOSEFS_NAME, csum, cancellable, error))
         return FALSE;
+      composefs_end_time = g_get_monotonic_time ();
     }
   else
     g_debug ("not using composefs");
 #endif
 
+  *checkout_elapsed = (checkout_end_time - checkout_start_time);
+  *composefs_elapsed = (composefs_end_time - composefs_start_time);
   if (out_deployment_dfd)
     *out_deployment_dfd = glnx_steal_fd (&ret_deployment_dfd);
   return TRUE;
@@ -3176,8 +3201,10 @@ sysroot_initialize_deployment (OstreeSysroot *self, const char *osname, const ch
 
   /* Check out the userspace tree onto the filesystem */
   glnx_autofd int deployment_dfd = -1;
-  if (!checkout_deployment_tree (self, repo, new_deployment, revision, &deployment_dfd, cancellable,
-                                 error))
+  guint64 checkout_elapsed = 0;
+  guint64 composefs_elapsed = 0;
+  if (!checkout_deployment_tree (self, repo, new_deployment, revision, &deployment_dfd,
+                                 &checkout_elapsed, &composefs_elapsed, cancellable, error))
     return FALSE;
 
   g_autoptr (OstreeKernelLayout) kernel_layout = NULL;
@@ -3189,12 +3216,20 @@ sysroot_initialize_deployment (OstreeSysroot *self, const char *osname, const ch
                                                 opts ? opts->override_kernel_argv : NULL);
   _ostree_deployment_set_overlay_initrds (new_deployment, opts ? opts->overlay_initrds : NULL);
 
+  guint64 etc_start_time = g_get_monotonic_time ();
   if (!prepare_deployment_etc (self, repo, new_deployment, deployment_dfd, cancellable, error))
     return FALSE;
+  guint64 etc_elapsed = g_get_monotonic_time () - etc_start_time;
 
   if (!prepare_deployment_var (self, new_deployment, deployment_dfd, cancellable, error))
     return FALSE;
 
+  g_autofree char *checkout_elapsed_str = ot_format_human_duration (checkout_elapsed);
+  g_autofree char *composefs_elapsed_str = ot_format_human_duration (composefs_elapsed);
+  g_autofree char *etc_elapsed_str = ot_format_human_duration (etc_elapsed);
+  ot_journal_print (LOG_INFO, "Created deployment; subtasks: checkout=%s composefs=%s etc=%s",
+                    checkout_elapsed_str, composefs_elapsed_str, etc_elapsed_str);
+
   ot_transfer_out_value (out_new_deployment, &new_deployment);
   return TRUE;
 }

src/libotutil/ot-gio-utils.c

@@ -170,3 +170,21 @@ ot_file_get_path_cached (GFile *file)
 
   return path;
 }
+
+/* Format the provided nanoseconds for human consumption;
+ * currently only suitable for tasks on the order of seconds.
+ */
+char *
+ot_format_human_duration (guint64 nanos)
+{
+  guint64 ms = nanos / 1000;
+  if (ms == 0)
+    return g_strdup_printf ("%" G_GUINT64_FORMAT "ns", nanos);
+  else if (ms < 1000)
+    return g_strdup_printf ("%" G_GUINT64_FORMAT "ms", ms);
+  else
+    {
+      double secs = ((double)ms) / 1000;
+      return g_strdup_printf ("%0.1fs", secs);
+    }
+}