0608-libxl-add-pcidevs-to-stubdomain-earlier.patch

From 87acd6cc196c9633fe102e329c4c41c8a3a94818 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 30 Dec 2021 21:12:00 +0100
Subject: [PATCH] libxl: add pcidevs to stubdomain earlier
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When stubdomain is constructed, PCI devices were added during qemu
startup (via hotplug path). While it makes the code (slightly) simpler,
as it's more similar to no-stubdomain case, it also causes problems:
 - race condition during stubdomain startup (QEMU trying to access sysfs
   entries before xen-pcifront materialize them): QubesOS/qubes-issues#6203
 - absent 0000:00:00.0 (host bridge) device during QEMU startup, which
   breaks `igd-passthru=on` case: QubesOS/qubes-issues#7164

While the first issue has alternative solution (retrying on failure),
the second does not. In this case, QEMU tries to copy few parameters
from the host bridge (apparently necessary for some drivers, although
doesn't seem to affect Linux), even though only IGD is assigned. While
0000:00:00.0 in (PV) stubdomain is not even a host bridge, it is the
Intel graphics card (when it's assigned) and share the same values in
config space offsets that QEMU reads (see hw/pci-host/xen_igd_pt.c in
QEMU).

So, to make QEMU happy and fix both issues at the same time, assign PCI
devices before unpausing the stubdomain. This requires rather careful
handling, as some steps need to be done only once (assigning to
xen-pciback for example).

Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
---
 tools/libs/light/libxl_dm.c       | 34 +++++++++++++++++++++++++++++--
 tools/libs/light/libxl_internal.h |  3 +++
 tools/libs/light/libxl_pci.c      | 20 ++++++++++--------
 3 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/tools/libs/light/libxl_dm.c b/tools/libs/light/libxl_dm.c
index bcbaac8d8f1d..160d4e065d08 100644
--- a/tools/libs/light/libxl_dm.c
+++ b/tools/libs/light/libxl_dm.c
@@ -2232,6 +2232,9 @@ static void spawn_stubdom_pvqemu_cb(libxl__egc *egc,
                                 libxl__dm_spawn_state *stubdom_dmss,
                                 int rc);
 
+static void spawn_stub_pcis_dm(libxl__egc *egc,
+                               libxl__multidev *multidev, int ret);
+
 static void spawn_stub_launch_dm(libxl__egc *egc,
                                  libxl__multidev *aodevs, int ret);
 
@@ -2447,7 +2450,7 @@ retry_transaction:
             goto retry_transaction;
 
     libxl__multidev_begin(ao, &sdss->multidev);
-    sdss->multidev.callback = spawn_stub_launch_dm;
+    sdss->multidev.callback = spawn_stub_pcis_dm;
     libxl__add_disks(egc, ao, dm_domid, dm_config, &sdss->multidev);
     libxl__multidev_prepared(egc, &sdss->multidev, 0);
 
@@ -2458,6 +2461,33 @@ out:
     spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, ret);
 }
 
+static void spawn_stub_pcis_dm(libxl__egc *egc,
+                               libxl__multidev *multidev, int ret)
+{
+    libxl__stub_dm_spawn_state *sdss = CONTAINER_OF(multidev, *sdss, multidev);
+    STATE_AO_GC(sdss->dm.spawn.ao);
+
+    /* convenience aliases */
+    libxl_domain_config *const guest_config = sdss->dm.guest_config;
+    const int guest_domid = sdss->dm.guest_domid;
+    uint32_t dm_domid = sdss->pvqemu.guest_domid;
+
+    if (ret) {
+        LOGD(ERROR, guest_domid, "error connecting disk devices");
+        goto out;
+    }
+
+    libxl__multidev_begin(ao, &sdss->multidev);
+    sdss->multidev.callback = spawn_stub_launch_dm;
+    libxl__add_pcis(egc, ao, dm_domid, guest_config, &sdss->multidev);
+    libxl__multidev_prepared(egc, &sdss->multidev, 0);
+    return;
+
+out:
+    assert(ret);
+    spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, ret);
+}
+
 static void spawn_stub_launch_dm(libxl__egc *egc,
                                  libxl__multidev *multidev, int ret)
 {
@@ -2477,7 +2507,7 @@ static void spawn_stub_launch_dm(libxl__egc *egc,
     int need_qemu;
 
     if (ret) {
-        LOGD(ERROR, guest_domid, "error connecting disk devices");
+        LOGD(ERROR, guest_domid, "error connecting pci devices");
         goto out;
      }
 
diff --git a/tools/libs/light/libxl_internal.h b/tools/libs/light/libxl_internal.h
index e455d46dc5ef..6e2363d03196 100644
--- a/tools/libs/light/libxl_internal.h
+++ b/tools/libs/light/libxl_internal.h
@@ -1731,6 +1731,9 @@ _hidden int libxl__device_pci_setdefault(libxl__gc *gc, uint32_t domid,
                                          libxl_device_pci *pci, bool hotplug);
 _hidden bool libxl__is_igd_vga_passthru(libxl__gc *gc,
                                         const libxl_domain_config *d_config);
+_hidden void libxl__add_pcis(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
+                             libxl_domain_config *d_config,
+                             libxl__multidev *multidev);
 
 /* from libxl_dtdev */
 
diff --git a/tools/libs/light/libxl_pci.c b/tools/libs/light/libxl_pci.c
index 38bffed56238..bf4acb46d6e8 100644
--- a/tools/libs/light/libxl_pci.c
+++ b/tools/libs/light/libxl_pci.c
@@ -1434,10 +1434,6 @@ static void pci_add_dm_done(libxl__egc *egc,
 
     if (rc) goto out;
 
-    /* stubdomain is always running by now, even at create time */
-    if (isstubdom)
-        starting = false;
-
     sysfs_path = GCSPRINTF(SYSFS_PCI_DEV"/"PCI_BDF"/resource", pci->domain,
                            pci->bus, pci->dev, pci->func);
     f = fopen(sysfs_path, "r");
@@ -1670,6 +1666,13 @@ void libxl__device_pci_add(libxl__egc *egc, uint32_t domid,
     rc = libxl__device_pci_setdefault(gc, domid, pci, !starting);
     if (rc) goto out;
 
+    stubdomid = libxl_get_stubdom_id(ctx, domid);
+    if (stubdomid != 0 && starting) {
+        /* Initial work already done when attaching to the stubdom */
+        device_pci_add_stubdom_done(egc, pas, 0); /* must be last */
+        return;
+    }
+
     if (pci->seize && !pciback_dev_is_assigned(gc, pci)) {
         rc = libxl__device_pci_assignable_add(gc, pci, 1);
         if ( rc )
@@ -1688,8 +1691,7 @@ void libxl__device_pci_add(libxl__egc *egc, uint32_t domid,
 
     libxl__device_pci_reset(gc, pci->domain, pci->bus, pci->dev, pci->func);
 
-    stubdomid = libxl_get_stubdom_id(ctx, domid);
-    if (stubdomid != 0) {
+    if (stubdomid != 0 && !starting) {
         pas->callback = device_pci_add_stubdom_wait;
 
         do_pci_add(egc, stubdomid, pas); /* must be last */
@@ -1831,9 +1833,9 @@ typedef struct {
 
 static void add_pcis_done(libxl__egc *, libxl__multidev *, int rc);
 
-static void libxl__add_pcis(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
-                            libxl_domain_config *d_config,
-                            libxl__multidev *multidev)
+void libxl__add_pcis(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
+                     libxl_domain_config *d_config,
+                     libxl__multidev *multidev)
 {
     AO_GC;
     add_pcis_state *apds;
-- 
2.44.0