CP-52131/CP-53474: Reorder operations during pci_add (#6426)

Reorder the operations in _pci_add:

1) So that Xen can verify calls to grant ioport and iomem permissions,
reorder the calls so that the device is assigned to the domain before
granting permissions for the resources. When Secure Boot is enabled, Xen
will enforce that ioport/iomem permissions can be granted to a domain
only when the corresponding device is assigned to that domain.

2) Add the device to QEMU after assigning to the domain. Rather than
accessing the PCI config space through dom0 sysfs which is blocked when
Secure Boot is enabled, QEMU in XS9 has been updated to use a hypercall
to access the PCI config space of a device assigned to a domain.
Therefore, add the device to QEMU after assigning it to the domain
rather than before so that the config space accesses it performs during
the QMP call succeed.

Author: edwintorok

ocaml/xenopsd/xc/device.ml

@@ -1215,30 +1215,6 @@ module PCI = struct
       |> String.trim
       |> int_of_string
     in
-    if hvm && qmp_add then
-      if Service.Qemu.is_running ~xs domid then
-        let id =
-          Printf.sprintf "pci-pt-%02x_%02x.%01x" host.bus host.dev host.fn
-        in
-        let _qmp_result =
-          qmp_send_cmd domid
-            (Qmp.Device_add
-               {
-                 driver= "xen-pci-passthrough"
-               ; device=
-                   Qmp.Device.PCI
-                     {
-                       id
-                     ; devfn
-                     ; hostaddr= string_of_address host
-                     ; permissive= false
-                     }
-               }
-            )
-        in
-        ()
-      else
-        raise (Domain_not_running (host, domid)) ;
     let addresses =
       sysfs_pci_dev ^ string_of_address host ^ "/resource"
       |> Unixext.string_of_file
@@ -1264,15 +1240,39 @@ module PCI = struct
             in
             Xenctrl.domain_iomem_permission xc domid scan_start scan_size true
     in
-    List.iteri apply_io_permission addresses ;
     let xcext = Xenctrlext.get_handle () in
+    ignore (quarantine host) ;
+    Xenctrlext.assign_device xcext domid (encode_bdf host)
+      _xen_domctl_dev_rdm_relaxed ;
+    List.iteri apply_io_permission addresses ;
     ( if irq > 0 then
         Xenctrlext.physdev_map_pirq xcext domid irq |> fun x ->
         Xenctrl.domain_irq_permission xc domid x true
     ) ;
-    ignore (quarantine host) ;
-    Xenctrlext.assign_device xcext domid (encode_bdf host)
-      _xen_domctl_dev_rdm_relaxed
+    if hvm && qmp_add then
+      if Service.Qemu.is_running ~xs domid then
+        let id =
+          Printf.sprintf "pci-pt-%02x_%02x.%01x" host.bus host.dev host.fn
+        in
+        let _qmp_result =
+          qmp_send_cmd domid
+            (Qmp.Device_add
+               {
+                 driver= "xen-pci-passthrough"
+               ; device=
+                   Qmp.Device.PCI
+                     {
+                       id
+                     ; devfn
+                     ; hostaddr= string_of_address host
+                     ; permissive= false
+                     }
+               }
+            )
+        in
+        ()
+      else
+        raise (Domain_not_running (host, domid))
 
   let add ~xc ~xs ~hvm pcidevs domid =
     let host_addr {host; guest= _; _} = host in