nodedev: Export NUMA node locality for PCI devices

A PCI device can be associated with a specific NUMA node. Later, when
a guest is pinned to one NUMA node the PCI device can be assigned on
different NUMA node. This makes DMA transfers travel across nodes and
thus results in suboptimal performance. We should expose the NUMA node
locality for PCI devices so management applications can make better
decisions.
Signed-off-by: NMichal Privoznik <mprivozn@redhat.com>

docs/formatnode.html.in

@@ -110,6 +110,13 @@
                 have a list of <code>address</code> subelements, one
                 for each VF on this PF.
               </dd>
+              <dt><code>numa</code></dt>
+              <dd>
+                This optional element contains information on the PCI device
+                with respect to NUMA. For example, the optional
+                <code>node</code> attribute tells which NUMA node is the PCI
+                device associated with.
+              </dd>
             </dl>
           </dd>
           <dt><code>usb_device</code></dt>

docs/schemas/nodedev.rng

@@ -158,6 +158,16 @@
       </element>
     </optional>
 
+    <optional>
+      <element name='numa'>
+        <optional>
+          <attribute name='node'>
+            <data type='int'/>
+          </attribute>
+        </optional>
+      </element>
+    </optional>
+
   </define>
 
   <define name='capusbdev'>

src/conf/node_device_conf.c

@@ -346,6 +346,9 @@ char *virNodeDeviceDefFormat(const virNodeDeviceDef *def)
                 virBufferAdjustIndent(&buf, -2);
                 virBufferAddLit(&buf, "</iommuGroup>\n");
             }
+            if (data->pci_dev.numa_node >= 0)
+                virBufferAsprintf(&buf, "<numa node='%d'/>\n",
+                                  data->pci_dev.numa_node);
             break;
         case VIR_NODE_DEV_CAP_USB_DEV:
             virBufferAsprintf(&buf, "<bus>%d</bus>\n", data->usb_dev.bus);
@@ -520,6 +523,41 @@ char *virNodeDeviceDefFormat(const virNodeDeviceDef *def)
     return NULL;
 }
 
+/**
+ * virNodeDevCapsDefParseIntOptional:
+ * @xpath:  XPath to evaluate
+ * @ctxt:   Context
+ * @value:  Where to store parsed value
+ * @def:    Node device which is parsed
+ * @invalid_error_fmt:  error message to print on invalid format
+ *
+ * Returns: -1 on error (invalid int format under @xpath)
+ *           0 if @xpath was not found (@value is untouched)
+ *           1 on success
+ */
+static int
+virNodeDevCapsDefParseIntOptional(const char *xpath,
+                                  xmlXPathContextPtr ctxt,
+                                  int *value,
+                                  virNodeDeviceDefPtr def,
+                                  const char *invalid_error_fmt)
+{
+    int ret;
+    int val;
+
+    ret = virXPathInt(xpath, ctxt, &val);
+    if (ret < -1) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       invalid_error_fmt,
+                       def->name);
+        return -1;
+    } else if (ret == -1) {
+        return 0;
+    }
+    *value = val;
+    return 1;
+}
+
 static int
 virNodeDevCapsDefParseULong(const char *xpath,
                             xmlXPathContextPtr ctxt,
@@ -1101,6 +1139,12 @@ virNodeDevCapPCIDevParseXML(xmlXPathContextPtr ctxt,
             goto out;
         }
     }
+
+    if (virNodeDevCapsDefParseIntOptional("number(./numa[1]/@node)", ctxt,
+                                          &data->pci_dev.numa_node, def,
+                                          _("invalid NUMA node ID supplied for '%s'")) < 0)
+        goto out;
+
     ret = 0;
  out:
     ctxt->node = orignode;

src/conf/node_device_conf.h

@@ -115,6 +115,7 @@ struct _virNodeDevCapsDef {
             virPCIDeviceAddressPtr *iommuGroupDevices;
             size_t nIommuGroupDevices;
             unsigned int iommuGroupNumber;
+            int numa_node;
         } pci_dev;
         struct {
             unsigned int bus;

src/node_device/node_device_udev.c

@@ -493,6 +493,18 @@ static int udevProcessPCI(struct udev_device *device,
         goto out;
     }
 
+    rc = udevGetIntSysfsAttr(device,
+                            "numa_node",
+                            &data->pci_dev.numa_node,
+                            10);
+    if (rc == PROPERTY_ERROR) {
+        goto out;
+    } else if (rc == PROPERTY_MISSING) {
+        /* The default value is -1, because it can't be 0
+         * as zero is valid node number. */
+        data->pci_dev.numa_node = -1;
+    }
+
     if (!virPCIGetPhysicalFunction(syspath, &data->pci_dev.physical_function))
         data->pci_dev.flags |= VIR_NODE_DEV_CAP_FLAG_PCI_PHYSICAL_FUNCTION;
 

tests/nodedevschemadata/pci_1002_71c4.xml

@@ -8,5 +8,6 @@
     <function>0</function>
     <product id='0x71c4'>M56GL [Mobility FireGL V5200]</product>
     <vendor id='0x1002'>ATI Technologies Inc</vendor>
+    <numa node='1'/>
   </capability>
 </device>

tests/nodedevschemadata/pci_8086_10c9_sriov_pf.xml

@@ -12,5 +12,6 @@
       <address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
       <address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/>
     </iommuGroup>
+    <numa node='0'/>
   </capability>
 </device>