From 5f6723e71e3765d1d43bfa9ba1c66e0e05e11a48 Mon Sep 17 00:00:00 2001 Message-Id: <5f6723e71e3765d1d43bfa9ba1c66e0e05e11a48@dist-git> From: Michal Privoznik Date: Mon, 9 Nov 2020 17:22:32 +0100 Subject: [PATCH] Allow NUMA nodes without vCPUs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QEMU allows creating NUMA nodes that have memory only. These are somehow important for HMAT. With check done in qemuValidateDomainDef() for QEMU 2.7 or newer (checked via QEMU_CAPS_NUMA), we can be sure that the vCPUs are fully assigned to NUMA nodes in domain XML. Signed-off-by: Michal Privoznik Reviewed-by: Daniel Henrique Barboza (cherry picked from commit a26f61ee0cffa421b87ef568002b684dd8025432) Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1749518 Conflicts: - src/qemu/qemu_validate.c: This file doesn't exist in downstream yet, so I've moved the change that original patch would do to qemu_domain.c where the validator lives. Signed-off-by: Michal Privoznik Message-Id: <365508c75e579e9037ad555d6c372068ccd50c95.1604938867.git.mprivozn@redhat.com> Reviewed-by: Ján Tomko --- docs/formatdomain.html.in | 2 + docs/schemas/cputypes.rng | 8 ++- src/conf/numa_conf.c | 59 ++++++++++--------- src/libxl/xen_xl.c | 10 ++-- src/qemu/qemu_command.c | 26 ++++---- src/qemu/qemu_domain.c | 22 +++---- tests/qemuxml2argvdata/numatune-no-vcpu.args | 33 +++++++++++ tests/qemuxml2argvdata/numatune-no-vcpu.xml | 42 +++++++++++++ tests/qemuxml2argvtest.c | 1 + tests/qemuxml2xmloutdata/numatune-no-vcpu.xml | 1 + tests/qemuxml2xmltest.c | 1 + 11 files changed, 149 insertions(+), 56 deletions(-) create mode 100644 tests/qemuxml2argvdata/numatune-no-vcpu.args create mode 100644 tests/qemuxml2argvdata/numatune-no-vcpu.xml create mode 120000 tests/qemuxml2xmloutdata/numatune-no-vcpu.xml diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index 76799f5ffc..4b8d312596 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -1783,6 +1783,8 @@ cpus specifies the CPU or range of CPUs that are part of the node. memory specifies the node memory in kibibytes (i.e. blocks of 1024 bytes). + Since 6.6.0 the cpus attribute + is optional and if omitted a CPU-less NUMA node is created. Since 1.2.11 one can use an additional unit attribute to define units in which memory is specified. diff --git a/docs/schemas/cputypes.rng b/docs/schemas/cputypes.rng index e2744acad3..a1682a1003 100644 --- a/docs/schemas/cputypes.rng +++ b/docs/schemas/cputypes.rng @@ -115,9 +115,11 @@ - - - + + + + + diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c index c9cc8ac22e..a805336d16 100644 --- a/src/conf/numa_conf.c +++ b/src/conf/numa_conf.c @@ -889,32 +889,28 @@ virDomainNumaDefParseXML(virDomainNumaPtr def, } VIR_FREE(tmp); - if (def->mem_nodes[cur_cell].cpumask) { + if (def->mem_nodes[cur_cell].mem) { virReportError(VIR_ERR_XML_ERROR, _("Duplicate NUMA cell info for cell id '%u'"), cur_cell); goto cleanup; } - if (!(tmp = virXMLPropString(nodes[i], "cpus"))) { - virReportError(VIR_ERR_XML_ERROR, "%s", - _("Missing 'cpus' attribute in NUMA cell")); - goto cleanup; - } + if ((tmp = virXMLPropString(nodes[i], "cpus"))) { + g_autoptr(virBitmap) cpumask = NULL; - if (virBitmapParse(tmp, &def->mem_nodes[cur_cell].cpumask, - VIR_DOMAIN_CPUMASK_LEN) < 0) - goto cleanup; + if (virBitmapParse(tmp, &cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0) + goto cleanup; - if (virBitmapIsAllClear(def->mem_nodes[cur_cell].cpumask)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("NUMA cell %d has no vCPUs assigned"), cur_cell); - goto cleanup; + if (!virBitmapIsAllClear(cpumask)) + def->mem_nodes[cur_cell].cpumask = g_steal_pointer(&cpumask); + VIR_FREE(tmp); } - VIR_FREE(tmp); for (j = 0; j < n; j++) { - if (j == cur_cell || !def->mem_nodes[j].cpumask) + if (j == cur_cell || + !def->mem_nodes[j].cpumask || + !def->mem_nodes[cur_cell].cpumask) continue; if (virBitmapOverlaps(def->mem_nodes[j].cpumask, @@ -976,7 +972,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf, { virDomainMemoryAccess memAccess; virTristateBool discard; - char *cpustr; size_t ncells = virDomainNumaGetNodeCount(def); size_t i; @@ -986,17 +981,22 @@ virDomainNumaDefFormatXML(virBufferPtr buf, virBufferAddLit(buf, "\n"); virBufferAdjustIndent(buf, 2); for (i = 0; i < ncells; i++) { + virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def, i); int ndistances; memAccess = virDomainNumaGetNodeMemoryAccessMode(def, i); discard = virDomainNumaGetNodeDiscard(def, i); - if (!(cpustr = virBitmapFormat(virDomainNumaGetNodeCpumask(def, i)))) - return -1; - virBufferAddLit(buf, "\n"); } - - VIR_FREE(cpustr); } virBufferAdjustIndent(buf, -2); virBufferAddLit(buf, "\n"); @@ -1048,8 +1046,12 @@ virDomainNumaGetCPUCountTotal(virDomainNumaPtr numa) size_t i; unsigned int ret = 0; - for (i = 0; i < numa->nmem_nodes; i++) - ret += virBitmapCountBits(virDomainNumaGetNodeCpumask(numa, i)); + for (i = 0; i < numa->nmem_nodes; i++) { + virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i); + + if (cpumask) + ret += virBitmapCountBits(cpumask); + } return ret; } @@ -1061,11 +1063,14 @@ virDomainNumaGetMaxCPUID(virDomainNumaPtr numa) unsigned int ret = 0; for (i = 0; i < numa->nmem_nodes; i++) { + virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i); int bit; - bit = virBitmapLastSetBit(virDomainNumaGetNodeCpumask(numa, i)); - if (bit > ret) - ret = bit; + if (cpumask) { + bit = virBitmapLastSetBit(cpumask); + if (bit > ret) + ret = bit; + } } return ret; diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c index edea30a86a..752fa925ec 100644 --- a/src/libxl/xen_xl.c +++ b/src/libxl/xen_xl.c @@ -1443,19 +1443,21 @@ xenFormatXLVnuma(virConfValuePtr list, { int ret = -1; size_t i; - virBuffer buf = VIR_BUFFER_INITIALIZER; virConfValuePtr numaVnode, tmp; - + virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, node); size_t nodeSize = virDomainNumaGetNodeMemorySize(numa, node) / 1024; - char *nodeVcpus = virBitmapFormat(virDomainNumaGetNodeCpumask(numa, node)); + g_autofree char *nodeVcpus = NULL; - if (VIR_ALLOC(numaVnode) < 0) + if (!cpumask || + VIR_ALLOC(numaVnode) < 0) goto cleanup; numaVnode->type = VIR_CONF_LIST; numaVnode->list = NULL; + nodeVcpus = virBitmapFormat(cpumask); + /* pnode */ virBufferAsprintf(&buf, "pnode=%zu", node); xenFormatXLVnode(numaVnode, &buf); diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 1a573c2817..ac63d18a42 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7364,8 +7364,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, size_t i, j; virQEMUCapsPtr qemuCaps = priv->qemuCaps; g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; - char *cpumask = NULL; - char *tmpmask = NULL; char *next = NULL; virBufferPtr nodeBackends = NULL; bool needBackend = false; @@ -7400,9 +7398,7 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, goto cleanup; for (i = 0; i < ncells; i++) { - VIR_FREE(cpumask); - if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i)))) - goto cleanup; + virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i); if (needBackend) { virCommandAddArg(cmd, "-object"); @@ -7412,11 +7408,19 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, virCommandAddArg(cmd, "-numa"); virBufferAsprintf(&buf, "node,nodeid=%zu", i); - for (tmpmask = cpumask; tmpmask; tmpmask = next) { - if ((next = strchr(tmpmask, ','))) - *(next++) = '\0'; - virBufferAddLit(&buf, ",cpus="); - virBufferAdd(&buf, tmpmask, -1); + if (cpumask) { + g_autofree char *cpumaskStr = NULL; + char *tmpmask; + + if (!(cpumaskStr = virBitmapFormat(cpumask))) + goto cleanup; + + for (tmpmask = cpumaskStr; tmpmask; tmpmask = next) { + if ((next = strchr(tmpmask, ','))) + *(next++) = '\0'; + virBufferAddLit(&buf, ",cpus="); + virBufferAdd(&buf, tmpmask, -1); + } } if (needBackend) @@ -7447,8 +7451,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, ret = 0; cleanup: - VIR_FREE(cpumask); - if (nodeBackends) { for (i = 0; i < ncells; i++) virBufferFreeAndReset(&nodeBackends[i]); diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 35b536868a..be25790f12 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -5373,7 +5373,7 @@ qemuDomainDefValidateNuma(const virDomainDef *def, } for (i = 0; i < ncells; i++) { - g_autofree char * cpumask = NULL; + virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i); if (!hasMemoryCap && virDomainNumaGetNodeMemoryAccessMode(def->numa, i)) { @@ -5383,17 +5383,19 @@ qemuDomainDefValidateNuma(const virDomainDef *def, return -1; } - if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i)))) - return -1; + if (cpumask) { + g_autofree char * cpumaskStr = NULL; + if (!(cpumaskStr = virBitmapFormat(cpumask))) + return -1; - if (strchr(cpumask, ',') && - !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("disjoint NUMA cpu ranges are not supported " - "with this QEMU")); - return -1; + if (strchr(cpumaskStr, ',') && + !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("disjoint NUMA cpu ranges are not supported " + "with this QEMU")); + return -1; + } } - } if (virDomainNumaNodesDistancesAreBeingSet(def->numa) && diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.args b/tests/qemuxml2argvdata/numatune-no-vcpu.args new file mode 100644 index 0000000000..a1f1ee044e --- /dev/null +++ b/tests/qemuxml2argvdata/numatune-no-vcpu.args @@ -0,0 +1,33 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/tmp/lib/domain--1-QEMUGuest \ +USER=test \ +LOGNAME=test \ +XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \ +XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \ +XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \ +QEMU_AUDIO_DRV=none \ +/usr/bin/qemu-system-x86_64 \ +-name QEMUGuest \ +-S \ +-machine pc,accel=tcg,usb=off,dump-guest-core=off \ +-m 12288 \ +-realtime mlock=off \ +-smp 12,sockets=12,cores=1,threads=1 \ +-numa node,nodeid=0,cpus=0-3,mem=2048 \ +-numa node,nodeid=1,cpus=4-7,mem=2048 \ +-numa node,nodeid=2,cpus=8-11,mem=2048 \ +-numa node,nodeid=3,mem=2048 \ +-numa node,nodeid=4,mem=2048 \ +-numa node,nodeid=5,mem=2048 \ +-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \ +-display none \ +-no-user-config \ +-nodefaults \ +-chardev socket,id=charmonitor,path=/tmp/lib/domain--1-QEMUGuest/monitor.sock,\ +server,nowait \ +-mon chardev=charmonitor,id=monitor,mode=control \ +-rtc base=utc \ +-no-shutdown \ +-usb \ +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.xml b/tests/qemuxml2argvdata/numatune-no-vcpu.xml new file mode 100644 index 0000000000..f25a07d7ed --- /dev/null +++ b/tests/qemuxml2argvdata/numatune-no-vcpu.xml @@ -0,0 +1,42 @@ + + QEMUGuest + c7a5fdb2-cdaf-9455-926a-d65c16db1809 + 12582912 + 12582912 + 12 + + hvm + + + + + + + + + + + + + + + + + + + destroy + restart + restart + + /usr/bin/qemu-system-x86_64 + +
+ + + + + +
+ + + diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index ff92af606d..49699e495d 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1812,6 +1812,7 @@ mymain(void) DO_TEST_PARSE_ERROR("numatune-memnode-no-memory", NONE); DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST); + DO_TEST("numatune-no-vcpu", NONE); DO_TEST("numatune-auto-nodeset-invalid", NONE); DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM, diff --git a/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml new file mode 120000 index 0000000000..f213032685 --- /dev/null +++ b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml @@ -0,0 +1 @@ +../qemuxml2argvdata/numatune-no-vcpu.xml \ No newline at end of file diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c index 6c3f5c4a9e..1ddeba30f0 100644 --- a/tests/qemuxml2xmltest.c +++ b/tests/qemuxml2xmltest.c @@ -1105,6 +1105,7 @@ mymain(void) DO_TEST("numatune-memnode", QEMU_CAPS_NUMA, QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST); + DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA); DO_TEST("bios-nvram", NONE); DO_TEST("bios-nvram-os-interleave", NONE); -- 2.29.2