986 lines
33 KiB
Diff
986 lines
33 KiB
Diff
From 50c5a71ce4a4400d08deff481c8781f3fca755c8 Mon Sep 17 00:00:00 2001
|
|
Message-Id: <50c5a71ce4a4400d08deff481c8781f3fca755c8@dist-git>
|
|
From: Pavel Hrdina <phrdina@redhat.com>
|
|
Date: Mon, 1 Jul 2019 17:07:53 +0200
|
|
Subject: [PATCH] vircgroup: add support for hybrid configuration
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain; charset=UTF-8
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
This enables to use both cgroup v1 and v2 at the same time together
|
|
with libvirt. It is supported by kernel and there is valid use-case,
|
|
not all controllers are implemented in cgroup v2 so there might be
|
|
configurations where administrator would enable these missing
|
|
controllers in cgroup v1.
|
|
|
|
Signed-off-by: Pavel Hrdina <phrdina@redhat.com>
|
|
(cherry picked from commit b79d858518ed15b1a4271457fc9f39463dd99230)
|
|
|
|
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1689297
|
|
|
|
Signed-off-by: Pavel Hrdina <phrdina@redhat.com>
|
|
Message-Id: <c84d8624633d438daa6b4de651eb6bb9e0afa25c.1561993100.git.phrdina@redhat.com>
|
|
Reviewed-by: Ján Tomko <jtomko@redhat.com>
|
|
---
|
|
src/util/vircgroup.c | 351 ++++++++++++++++++++++++++----------
|
|
src/util/vircgroupbackend.c | 20 ++
|
|
src/util/vircgroupbackend.h | 16 +-
|
|
src/util/vircgrouppriv.h | 2 +-
|
|
4 files changed, 291 insertions(+), 98 deletions(-)
|
|
|
|
diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c
|
|
index a859628241..069f1ae396 100644
|
|
--- a/src/util/vircgroup.c
|
|
+++ b/src/util/vircgroup.c
|
|
@@ -232,6 +232,7 @@ virCgroupDetectMounts(virCgroupPtr group)
|
|
struct mntent entry;
|
|
char buf[CGROUP_MAX_VAL];
|
|
int ret = -1;
|
|
+ size_t i;
|
|
|
|
mounts = fopen("/proc/mounts", "r");
|
|
if (mounts == NULL) {
|
|
@@ -240,11 +241,14 @@ virCgroupDetectMounts(virCgroupPtr group)
|
|
}
|
|
|
|
while (getmntent_r(mounts, &entry, buf, sizeof(buf)) != NULL) {
|
|
- if (group->backend->detectMounts(group,
|
|
- entry.mnt_type,
|
|
- entry.mnt_opts,
|
|
- entry.mnt_dir) < 0) {
|
|
- goto cleanup;
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->detectMounts(group,
|
|
+ entry.mnt_type,
|
|
+ entry.mnt_opts,
|
|
+ entry.mnt_dir) < 0) {
|
|
+ goto cleanup;
|
|
+ }
|
|
}
|
|
}
|
|
|
|
@@ -307,6 +311,7 @@ virCgroupDetectPlacement(virCgroupPtr group,
|
|
}
|
|
|
|
while (fgets(line, sizeof(line), mapping) != NULL) {
|
|
+ size_t i;
|
|
char *controllers = strchr(line, ':');
|
|
char *selfpath = controllers ? strchr(controllers + 1, ':') : NULL;
|
|
char *nl = selfpath ? strchr(selfpath, '\n') : NULL;
|
|
@@ -321,9 +326,12 @@ virCgroupDetectPlacement(virCgroupPtr group,
|
|
controllers++;
|
|
selfpath++;
|
|
|
|
- if (group->backend->detectPlacement(group, path, controllers,
|
|
- selfpath) < 0) {
|
|
- goto cleanup;
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->detectPlacement(group, path, controllers,
|
|
+ selfpath) < 0) {
|
|
+ goto cleanup;
|
|
+ }
|
|
}
|
|
}
|
|
|
|
@@ -342,8 +350,9 @@ virCgroupDetect(virCgroupPtr group,
|
|
const char *path,
|
|
virCgroupPtr parent)
|
|
{
|
|
- int rc;
|
|
size_t i;
|
|
+ bool backendAvailable = false;
|
|
+ int controllersAvailable = 0;
|
|
virCgroupBackendPtr *backends = virCgroupBackendGetAll();
|
|
|
|
VIR_DEBUG("group=%p controllers=%d path=%s parent=%p",
|
|
@@ -354,31 +363,40 @@ virCgroupDetect(virCgroupPtr group,
|
|
|
|
for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
if (backends[i] && backends[i]->available()) {
|
|
- group->backend = backends[i];
|
|
- break;
|
|
+ group->backends[i] = backends[i];
|
|
+ backendAvailable = true;
|
|
}
|
|
}
|
|
|
|
- if (!group->backend) {
|
|
+ if (!backendAvailable) {
|
|
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
|
_("no cgroup backend available"));
|
|
return -1;
|
|
}
|
|
|
|
if (parent) {
|
|
- if (group->backend->copyMounts(group, parent) < 0)
|
|
- return -1;
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->copyMounts(group, parent) < 0) {
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
} else {
|
|
if (virCgroupDetectMounts(group) < 0)
|
|
return -1;
|
|
}
|
|
|
|
- rc = group->backend->detectControllers(group, controllers);
|
|
- if (rc < 0)
|
|
- return -1;
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i]) {
|
|
+ int rc = group->backends[i]->detectControllers(group, controllers);
|
|
+ if (rc < 0)
|
|
+ return -1;
|
|
+ controllersAvailable |= rc;
|
|
+ }
|
|
+ }
|
|
|
|
/* Check that at least 1 controller is available */
|
|
- if (rc == 0) {
|
|
+ if (controllersAvailable == 0) {
|
|
virReportSystemError(ENXIO, "%s",
|
|
_("At least one cgroup controller is required"));
|
|
return -1;
|
|
@@ -387,17 +405,26 @@ virCgroupDetect(virCgroupPtr group,
|
|
/* In some cases we can copy part of the placement info
|
|
* based on the parent cgroup...
|
|
*/
|
|
- if ((parent || path[0] == '/') &&
|
|
- group->backend->copyPlacement(group, path, parent) < 0)
|
|
- return -1;
|
|
+ if (parent || path[0] == '/') {
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->copyPlacement(group, path, parent) < 0) {
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
|
|
/* ... but use /proc/cgroups to fill in the rest */
|
|
if (virCgroupDetectPlacement(group, pid, path) < 0)
|
|
return -1;
|
|
|
|
/* Check that for every mounted controller, we found our placement */
|
|
- if (group->backend->validatePlacement(group, pid) < 0)
|
|
- return -1;
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->validatePlacement(group, pid) < 0) {
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
|
|
return 0;
|
|
}
|
|
@@ -603,9 +630,14 @@ virCgroupMakeGroup(virCgroupPtr parent,
|
|
bool create,
|
|
unsigned int flags)
|
|
{
|
|
- if (group->backend->makeGroup(parent, group, create, flags) < 0) {
|
|
- virCgroupRemove(group);
|
|
- return -1;
|
|
+ size_t i;
|
|
+
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->makeGroup(parent, group, create, flags) < 0) {
|
|
+ virCgroupRemove(group);
|
|
+ return -1;
|
|
+ }
|
|
}
|
|
|
|
return 0;
|
|
@@ -666,6 +698,24 @@ virCgroupNew(pid_t pid,
|
|
}
|
|
|
|
|
|
+static int
|
|
+virCgroupAddTaskInternal(virCgroupPtr group,
|
|
+ pid_t pid,
|
|
+ unsigned int flags)
|
|
+{
|
|
+ size_t i;
|
|
+
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->addTask(group, pid, flags) < 0) {
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
+
|
|
/**
|
|
* virCgroupAddProcess:
|
|
*
|
|
@@ -680,7 +730,7 @@ virCgroupNew(pid_t pid,
|
|
int
|
|
virCgroupAddProcess(virCgroupPtr group, pid_t pid)
|
|
{
|
|
- return group->backend->addTask(group, pid, VIR_CGROUP_TASK_PROCESS);
|
|
+ return virCgroupAddTaskInternal(group, pid, VIR_CGROUP_TASK_PROCESS);
|
|
}
|
|
|
|
/**
|
|
@@ -697,9 +747,9 @@ virCgroupAddProcess(virCgroupPtr group, pid_t pid)
|
|
int
|
|
virCgroupAddMachineProcess(virCgroupPtr group, pid_t pid)
|
|
{
|
|
- return group->backend->addTask(group, pid,
|
|
- VIR_CGROUP_TASK_PROCESS |
|
|
- VIR_CGROUP_TASK_SYSTEMD);
|
|
+ return virCgroupAddTaskInternal(group, pid,
|
|
+ VIR_CGROUP_TASK_PROCESS |
|
|
+ VIR_CGROUP_TASK_SYSTEMD);
|
|
}
|
|
|
|
/**
|
|
@@ -717,7 +767,7 @@ int
|
|
virCgroupAddThread(virCgroupPtr group,
|
|
pid_t pid)
|
|
{
|
|
- return group->backend->addTask(group, pid, VIR_CGROUP_TASK_THREAD);
|
|
+ return virCgroupAddTaskInternal(group, pid, VIR_CGROUP_TASK_THREAD);
|
|
}
|
|
|
|
|
|
@@ -971,17 +1021,24 @@ virCgroupNewDetectMachine(const char *name,
|
|
char *machinename,
|
|
virCgroupPtr *group)
|
|
{
|
|
+ size_t i;
|
|
+
|
|
if (virCgroupNewDetect(pid, controllers, group) < 0) {
|
|
if (virCgroupNewIgnoreError())
|
|
return 0;
|
|
return -1;
|
|
}
|
|
|
|
- if (!(*group)->backend->validateMachineGroup(*group, name, drivername, machinename)) {
|
|
- VIR_DEBUG("Failed to validate machine name for '%s' driver '%s'",
|
|
- name, drivername);
|
|
- virCgroupFree(group);
|
|
- return 0;
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if ((*group)->backends[i] &&
|
|
+ !(*group)->backends[i]->validateMachineGroup(*group, name,
|
|
+ drivername,
|
|
+ machinename)) {
|
|
+ VIR_DEBUG("Failed to validate machine name for '%s' driver '%s'",
|
|
+ name, drivername);
|
|
+ virCgroupFree(group);
|
|
+ return 0;
|
|
+ }
|
|
}
|
|
|
|
return 0;
|
|
@@ -1059,6 +1116,7 @@ virCgroupNewMachineSystemd(const char *name,
|
|
int rv;
|
|
virCgroupPtr init;
|
|
VIR_AUTOFREE(char *) path = NULL;
|
|
+ size_t i;
|
|
|
|
VIR_DEBUG("Trying to setup machine '%s' via systemd", name);
|
|
if ((rv = virSystemdCreateMachine(name,
|
|
@@ -1081,7 +1139,12 @@ virCgroupNewMachineSystemd(const char *name,
|
|
&init) < 0)
|
|
return -1;
|
|
|
|
- path = init->backend->stealPlacement(init);
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (init->backends[i] &&
|
|
+ (path = init->backends[i]->stealPlacement(init))) {
|
|
+ break;
|
|
+ }
|
|
+ }
|
|
virCgroupFree(&init);
|
|
|
|
if (!path || STREQ(path, "/") || path[0] != '/') {
|
|
@@ -1260,12 +1323,21 @@ virCgroupFree(virCgroupPtr *group)
|
|
bool
|
|
virCgroupHasController(virCgroupPtr cgroup, int controller)
|
|
{
|
|
+ size_t i;
|
|
+
|
|
if (!cgroup)
|
|
return false;
|
|
if (controller < 0 || controller >= VIR_CGROUP_CONTROLLER_LAST)
|
|
return false;
|
|
|
|
- return cgroup->backend->hasController(cgroup, controller);
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (cgroup->backends[i] &&
|
|
+ cgroup->backends[i]->hasController(cgroup, controller)) {
|
|
+ return true;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return false;
|
|
}
|
|
|
|
|
|
@@ -1281,7 +1353,8 @@ virCgroupPathOfController(virCgroupPtr group,
|
|
return -1;
|
|
}
|
|
|
|
- return group->backend->pathOfController(group, controller, key, path);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, controller, pathOfController, -1,
|
|
+ controller, key, path);
|
|
}
|
|
|
|
|
|
@@ -1303,7 +1376,8 @@ virCgroupGetBlkioIoServiced(virCgroupPtr group,
|
|
long long *requests_read,
|
|
long long *requests_write)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioIoServiced, -1,
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioIoServiced, -1,
|
|
bytes_read, bytes_write,
|
|
requests_read, requests_write);
|
|
}
|
|
@@ -1329,7 +1403,8 @@ virCgroupGetBlkioIoDeviceServiced(virCgroupPtr group,
|
|
long long *requests_read,
|
|
long long *requests_write)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioIoDeviceServiced, -1,
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioIoDeviceServiced, -1,
|
|
path, bytes_read, bytes_write,
|
|
requests_read, requests_write);
|
|
}
|
|
@@ -1346,7 +1421,8 @@ virCgroupGetBlkioIoDeviceServiced(virCgroupPtr group,
|
|
int
|
|
virCgroupSetBlkioWeight(virCgroupPtr group, unsigned int weight)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setBlkioWeight, -1, weight);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ setBlkioWeight, -1, weight);
|
|
}
|
|
|
|
|
|
@@ -1361,7 +1437,8 @@ virCgroupSetBlkioWeight(virCgroupPtr group, unsigned int weight)
|
|
int
|
|
virCgroupGetBlkioWeight(virCgroupPtr group, unsigned int *weight)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioWeight, -1, weight);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioWeight, -1, weight);
|
|
}
|
|
|
|
/**
|
|
@@ -1377,7 +1454,8 @@ virCgroupSetBlkioDeviceReadIops(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned int riops)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setBlkioDeviceReadIops, -1, path, riops);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ setBlkioDeviceReadIops, -1, path, riops);
|
|
}
|
|
|
|
|
|
@@ -1394,7 +1472,8 @@ virCgroupSetBlkioDeviceWriteIops(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned int wiops)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setBlkioDeviceWriteIops, -1, path, wiops);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ setBlkioDeviceWriteIops, -1, path, wiops);
|
|
}
|
|
|
|
|
|
@@ -1411,7 +1490,8 @@ virCgroupSetBlkioDeviceReadBps(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned long long rbps)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setBlkioDeviceReadBps, -1, path, rbps);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ setBlkioDeviceReadBps, -1, path, rbps);
|
|
}
|
|
|
|
/**
|
|
@@ -1427,7 +1507,8 @@ virCgroupSetBlkioDeviceWriteBps(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned long long wbps)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setBlkioDeviceWriteBps, -1, path, wbps);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ setBlkioDeviceWriteBps, -1, path, wbps);
|
|
}
|
|
|
|
|
|
@@ -1445,7 +1526,8 @@ virCgroupSetBlkioDeviceWeight(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned int weight)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setBlkioDeviceWeight, -1, path, weight);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ setBlkioDeviceWeight, -1, path, weight);
|
|
}
|
|
|
|
/**
|
|
@@ -1461,7 +1543,8 @@ virCgroupGetBlkioDeviceReadIops(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned int *riops)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioDeviceReadIops, -1, path, riops);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioDeviceReadIops, -1, path, riops);
|
|
}
|
|
|
|
/**
|
|
@@ -1477,7 +1560,8 @@ virCgroupGetBlkioDeviceWriteIops(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned int *wiops)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioDeviceWriteIops, -1, path, wiops);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioDeviceWriteIops, -1, path, wiops);
|
|
}
|
|
|
|
/**
|
|
@@ -1493,7 +1577,8 @@ virCgroupGetBlkioDeviceReadBps(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned long long *rbps)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioDeviceReadBps, -1, path, rbps);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioDeviceReadBps, -1, path, rbps);
|
|
}
|
|
|
|
/**
|
|
@@ -1509,7 +1594,8 @@ virCgroupGetBlkioDeviceWriteBps(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned long long *wbps)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioDeviceWriteBps, -1, path, wbps);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioDeviceWriteBps, -1, path, wbps);
|
|
}
|
|
|
|
/**
|
|
@@ -1525,7 +1611,8 @@ virCgroupGetBlkioDeviceWeight(virCgroupPtr group,
|
|
const char *path,
|
|
unsigned int *weight)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getBlkioDeviceWeight, -1, path, weight);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_BLKIO,
|
|
+ getBlkioDeviceWeight, -1, path, weight);
|
|
}
|
|
|
|
|
|
@@ -1540,7 +1627,8 @@ virCgroupGetBlkioDeviceWeight(virCgroupPtr group,
|
|
int
|
|
virCgroupSetMemory(virCgroupPtr group, unsigned long long kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setMemory, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ setMemory, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1566,7 +1654,8 @@ virCgroupGetMemoryStat(virCgroupPtr group,
|
|
unsigned long long *inactiveFile,
|
|
unsigned long long *unevictable)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getMemoryStat, -1, cache,
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ getMemoryStat, -1, cache,
|
|
activeAnon, inactiveAnon,
|
|
activeFile, inactiveFile,
|
|
unevictable);
|
|
@@ -1584,7 +1673,8 @@ virCgroupGetMemoryStat(virCgroupPtr group,
|
|
int
|
|
virCgroupGetMemoryUsage(virCgroupPtr group, unsigned long *kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getMemoryUsage, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ getMemoryUsage, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1599,7 +1689,8 @@ virCgroupGetMemoryUsage(virCgroupPtr group, unsigned long *kb)
|
|
int
|
|
virCgroupSetMemoryHardLimit(virCgroupPtr group, unsigned long long kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setMemoryHardLimit, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ setMemoryHardLimit, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1614,7 +1705,8 @@ virCgroupSetMemoryHardLimit(virCgroupPtr group, unsigned long long kb)
|
|
int
|
|
virCgroupGetMemoryHardLimit(virCgroupPtr group, unsigned long long *kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getMemoryHardLimit, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ getMemoryHardLimit, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1629,7 +1721,8 @@ virCgroupGetMemoryHardLimit(virCgroupPtr group, unsigned long long *kb)
|
|
int
|
|
virCgroupSetMemorySoftLimit(virCgroupPtr group, unsigned long long kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setMemorySoftLimit, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ setMemorySoftLimit, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1644,7 +1737,8 @@ virCgroupSetMemorySoftLimit(virCgroupPtr group, unsigned long long kb)
|
|
int
|
|
virCgroupGetMemorySoftLimit(virCgroupPtr group, unsigned long long *kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getMemorySoftLimit, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ getMemorySoftLimit, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1659,7 +1753,8 @@ virCgroupGetMemorySoftLimit(virCgroupPtr group, unsigned long long *kb)
|
|
int
|
|
virCgroupSetMemSwapHardLimit(virCgroupPtr group, unsigned long long kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setMemSwapHardLimit, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ setMemSwapHardLimit, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1674,7 +1769,8 @@ virCgroupSetMemSwapHardLimit(virCgroupPtr group, unsigned long long kb)
|
|
int
|
|
virCgroupGetMemSwapHardLimit(virCgroupPtr group, unsigned long long *kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getMemSwapHardLimit, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ getMemSwapHardLimit, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1689,7 +1785,8 @@ virCgroupGetMemSwapHardLimit(virCgroupPtr group, unsigned long long *kb)
|
|
int
|
|
virCgroupGetMemSwapUsage(virCgroupPtr group, unsigned long long *kb)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getMemSwapUsage, -1, kb);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_MEMORY,
|
|
+ getMemSwapUsage, -1, kb);
|
|
}
|
|
|
|
|
|
@@ -1704,7 +1801,8 @@ virCgroupGetMemSwapUsage(virCgroupPtr group, unsigned long long *kb)
|
|
int
|
|
virCgroupSetCpusetMems(virCgroupPtr group, const char *mems)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setCpusetMems, -1, mems);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUSET,
|
|
+ setCpusetMems, -1, mems);
|
|
}
|
|
|
|
|
|
@@ -1719,7 +1817,8 @@ virCgroupSetCpusetMems(virCgroupPtr group, const char *mems)
|
|
int
|
|
virCgroupGetCpusetMems(virCgroupPtr group, char **mems)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpusetMems, -1, mems);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUSET,
|
|
+ getCpusetMems, -1, mems);
|
|
}
|
|
|
|
|
|
@@ -1734,7 +1833,8 @@ virCgroupGetCpusetMems(virCgroupPtr group, char **mems)
|
|
int
|
|
virCgroupSetCpusetMemoryMigrate(virCgroupPtr group, bool migrate)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setCpusetMemoryMigrate, -1, migrate);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUSET,
|
|
+ setCpusetMemoryMigrate, -1, migrate);
|
|
}
|
|
|
|
|
|
@@ -1749,7 +1849,8 @@ virCgroupSetCpusetMemoryMigrate(virCgroupPtr group, bool migrate)
|
|
int
|
|
virCgroupGetCpusetMemoryMigrate(virCgroupPtr group, bool *migrate)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpusetMemoryMigrate, -1, migrate);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUSET,
|
|
+ getCpusetMemoryMigrate, -1, migrate);
|
|
}
|
|
|
|
|
|
@@ -1764,7 +1865,8 @@ virCgroupGetCpusetMemoryMigrate(virCgroupPtr group, bool *migrate)
|
|
int
|
|
virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setCpusetCpus, -1, cpus);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUSET,
|
|
+ setCpusetCpus, -1, cpus);
|
|
}
|
|
|
|
|
|
@@ -1779,7 +1881,8 @@ virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus)
|
|
int
|
|
virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpusetCpus, -1, cpus);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUSET,
|
|
+ getCpusetCpus, -1, cpus);
|
|
}
|
|
|
|
|
|
@@ -1793,7 +1896,8 @@ virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus)
|
|
int
|
|
virCgroupDenyAllDevices(virCgroupPtr group)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, denyAllDevices, -1);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_DEVICES,
|
|
+ denyAllDevices, -1);
|
|
}
|
|
|
|
/**
|
|
@@ -1813,7 +1917,8 @@ virCgroupDenyAllDevices(virCgroupPtr group)
|
|
int
|
|
virCgroupAllowAllDevices(virCgroupPtr group, int perms)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, allowAllDevices, -1, perms);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_DEVICES,
|
|
+ allowAllDevices, -1, perms);
|
|
}
|
|
|
|
|
|
@@ -1832,7 +1937,8 @@ int
|
|
virCgroupAllowDevice(virCgroupPtr group, char type, int major, int minor,
|
|
int perms)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, allowDevice, -1, type, major, minor, perms);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_DEVICES,
|
|
+ allowDevice, -1, type, major, minor, perms);
|
|
}
|
|
|
|
|
|
@@ -1871,7 +1977,8 @@ virCgroupAllowDevicePath(virCgroupPtr group,
|
|
if (!S_ISCHR(sb.st_mode) && !S_ISBLK(sb.st_mode))
|
|
return 1;
|
|
|
|
- VIR_CGROUP_BACKEND_CALL(group, allowDevice, -1,
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_DEVICES,
|
|
+ allowDevice, -1,
|
|
S_ISCHR(sb.st_mode) ? 'c' : 'b',
|
|
major(sb.st_rdev),
|
|
minor(sb.st_rdev),
|
|
@@ -1894,7 +2001,8 @@ int
|
|
virCgroupDenyDevice(virCgroupPtr group, char type, int major, int minor,
|
|
int perms)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, denyDevice, -1, type, major, minor, perms);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_DEVICES,
|
|
+ denyDevice, -1, type, major, minor, perms);
|
|
}
|
|
|
|
|
|
@@ -1933,7 +2041,8 @@ virCgroupDenyDevicePath(virCgroupPtr group,
|
|
if (!S_ISCHR(sb.st_mode) && !S_ISBLK(sb.st_mode))
|
|
return 1;
|
|
|
|
- VIR_CGROUP_BACKEND_CALL(group, denyDevice, -1,
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_DEVICES,
|
|
+ denyDevice, -1,
|
|
S_ISCHR(sb.st_mode) ? 'c' : 'b',
|
|
major(sb.st_rdev),
|
|
minor(sb.st_rdev),
|
|
@@ -2176,14 +2285,16 @@ virCgroupGetDomainTotalCpuStats(virCgroupPtr group,
|
|
int
|
|
virCgroupSetCpuShares(virCgroupPtr group, unsigned long long shares)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setCpuShares, -1, shares);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPU,
|
|
+ setCpuShares, -1, shares);
|
|
}
|
|
|
|
|
|
int
|
|
virCgroupGetCpuShares(virCgroupPtr group, unsigned long long *shares)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpuShares, -1, shares);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPU,
|
|
+ getCpuShares, -1, shares);
|
|
}
|
|
|
|
|
|
@@ -2198,7 +2309,8 @@ virCgroupGetCpuShares(virCgroupPtr group, unsigned long long *shares)
|
|
int
|
|
virCgroupSetCpuCfsPeriod(virCgroupPtr group, unsigned long long cfs_period)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setCpuCfsPeriod, -1, cfs_period);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPU,
|
|
+ setCpuCfsPeriod, -1, cfs_period);
|
|
}
|
|
|
|
|
|
@@ -2213,7 +2325,8 @@ virCgroupSetCpuCfsPeriod(virCgroupPtr group, unsigned long long cfs_period)
|
|
int
|
|
virCgroupGetCpuCfsPeriod(virCgroupPtr group, unsigned long long *cfs_period)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpuCfsPeriod, -1, cfs_period);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPU,
|
|
+ getCpuCfsPeriod, -1, cfs_period);
|
|
}
|
|
|
|
|
|
@@ -2229,14 +2342,16 @@ virCgroupGetCpuCfsPeriod(virCgroupPtr group, unsigned long long *cfs_period)
|
|
int
|
|
virCgroupSetCpuCfsQuota(virCgroupPtr group, long long cfs_quota)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setCpuCfsQuota, -1, cfs_quota);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPU,
|
|
+ setCpuCfsQuota, -1, cfs_quota);
|
|
}
|
|
|
|
|
|
int
|
|
virCgroupGetCpuacctPercpuUsage(virCgroupPtr group, char **usage)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpuacctPercpuUsage, -1, usage);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUACCT,
|
|
+ getCpuacctPercpuUsage, -1, usage);
|
|
}
|
|
|
|
|
|
@@ -2303,7 +2418,16 @@ virCgroupRemoveRecursively(char *grppath)
|
|
int
|
|
virCgroupRemove(virCgroupPtr group)
|
|
{
|
|
- return group->backend->remove(group);
|
|
+ size_t i;
|
|
+
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->remove(group) < 0) {
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
}
|
|
|
|
|
|
@@ -2312,11 +2436,16 @@ virCgroupPathOfAnyController(virCgroupPtr group,
|
|
const char *name,
|
|
char **keypath)
|
|
{
|
|
+ size_t i;
|
|
int controller;
|
|
|
|
- controller = group->backend->getAnyController(group);
|
|
- if (controller >= 0)
|
|
- return virCgroupPathOfController(group, controller, name, keypath);
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i]) {
|
|
+ controller = group->backends[i]->getAnyController(group);
|
|
+ if (controller >= 0)
|
|
+ return virCgroupPathOfController(group, controller, name, keypath);
|
|
+ }
|
|
+ }
|
|
|
|
virReportSystemError(ENOSYS, "%s",
|
|
_("No controllers are mounted"));
|
|
@@ -2552,14 +2681,16 @@ virCgroupKillPainfully(virCgroupPtr group)
|
|
int
|
|
virCgroupGetCpuCfsQuota(virCgroupPtr group, long long *cfs_quota)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpuCfsQuota, -1, cfs_quota);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPU,
|
|
+ getCpuCfsQuota, -1, cfs_quota);
|
|
}
|
|
|
|
|
|
int
|
|
virCgroupGetCpuacctUsage(virCgroupPtr group, unsigned long long *usage)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpuacctUsage, -1, usage);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUACCT,
|
|
+ getCpuacctUsage, -1, usage);
|
|
}
|
|
|
|
|
|
@@ -2567,21 +2698,24 @@ int
|
|
virCgroupGetCpuacctStat(virCgroupPtr group, unsigned long long *user,
|
|
unsigned long long *sys)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getCpuacctStat, -1, user, sys);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_CPUACCT,
|
|
+ getCpuacctStat, -1, user, sys);
|
|
}
|
|
|
|
|
|
int
|
|
virCgroupSetFreezerState(virCgroupPtr group, const char *state)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, setFreezerState, -1, state);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_FREEZER,
|
|
+ setFreezerState, -1, state);
|
|
}
|
|
|
|
|
|
int
|
|
virCgroupGetFreezerState(virCgroupPtr group, char **state)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(group, getFreezerState, -1, state);
|
|
+ VIR_CGROUP_BACKEND_CALL(group, VIR_CGROUP_CONTROLLER_FREEZER,
|
|
+ getFreezerState, -1, state);
|
|
}
|
|
|
|
|
|
@@ -2589,7 +2723,16 @@ int
|
|
virCgroupBindMount(virCgroupPtr group, const char *oldroot,
|
|
const char *mountopts)
|
|
{
|
|
- return group->backend->bindMount(group, oldroot, mountopts);
|
|
+ size_t i;
|
|
+
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->bindMount(group, oldroot, mountopts) < 0) {
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
}
|
|
|
|
|
|
@@ -2598,7 +2741,16 @@ int virCgroupSetOwner(virCgroupPtr cgroup,
|
|
gid_t gid,
|
|
int controllers)
|
|
{
|
|
- return cgroup->backend->setOwner(cgroup, uid, gid, controllers);
|
|
+ size_t i;
|
|
+
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (cgroup->backends[i] &&
|
|
+ cgroup->backends[i]->setOwner(cgroup, uid, gid, controllers) < 0) {
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
}
|
|
|
|
|
|
@@ -2612,13 +2764,24 @@ int virCgroupSetOwner(virCgroupPtr cgroup,
|
|
bool
|
|
virCgroupSupportsCpuBW(virCgroupPtr cgroup)
|
|
{
|
|
- VIR_CGROUP_BACKEND_CALL(cgroup, supportsCpuBW, false);
|
|
+ VIR_CGROUP_BACKEND_CALL(cgroup, VIR_CGROUP_CONTROLLER_CPU,
|
|
+ supportsCpuBW, false);
|
|
}
|
|
|
|
int
|
|
virCgroupHasEmptyTasks(virCgroupPtr cgroup, int controller)
|
|
{
|
|
- return cgroup->backend->hasEmptyTasks(cgroup, controller);
|
|
+ size_t i;
|
|
+
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (cgroup->backends[i]) {
|
|
+ int rc = cgroup->backends[i]->hasEmptyTasks(cgroup, controller);
|
|
+ if (rc <= 0)
|
|
+ return rc;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return 1;
|
|
}
|
|
|
|
bool
|
|
diff --git a/src/util/vircgroupbackend.c b/src/util/vircgroupbackend.c
|
|
index 7ee39ac8ca..2e90781dc3 100644
|
|
--- a/src/util/vircgroupbackend.c
|
|
+++ b/src/util/vircgroupbackend.c
|
|
@@ -20,6 +20,9 @@
|
|
#include <config.h>
|
|
|
|
#include "vircgroupbackend.h"
|
|
+#define __VIR_CGROUP_ALLOW_INCLUDE_PRIV_H__
|
|
+#include "vircgrouppriv.h"
|
|
+#undef __VIR_CGROUP_ALLOW_INCLUDE_PRIV_H__
|
|
#include "vircgroupv1.h"
|
|
#include "vircgroupv2.h"
|
|
#include "virerror.h"
|
|
@@ -67,3 +70,20 @@ virCgroupBackendGetAll(void)
|
|
}
|
|
return virCgroupBackends;
|
|
}
|
|
+
|
|
+
|
|
+virCgroupBackendPtr
|
|
+virCgroupBackendForController(virCgroupPtr group,
|
|
+ unsigned int controller)
|
|
+{
|
|
+ size_t i;
|
|
+
|
|
+ for (i = 0; i < VIR_CGROUP_BACKEND_TYPE_LAST; i++) {
|
|
+ if (group->backends[i] &&
|
|
+ group->backends[i]->hasController(group, controller)) {
|
|
+ return group->backends[i];
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return NULL;
|
|
+}
|
|
diff --git a/src/util/vircgroupbackend.h b/src/util/vircgroupbackend.h
|
|
index 86d1539e07..bc60b44643 100644
|
|
--- a/src/util/vircgroupbackend.h
|
|
+++ b/src/util/vircgroupbackend.h
|
|
@@ -436,12 +436,22 @@ virCgroupBackendRegister(virCgroupBackendPtr backend);
|
|
virCgroupBackendPtr *
|
|
virCgroupBackendGetAll(void);
|
|
|
|
-# define VIR_CGROUP_BACKEND_CALL(group, func, ret, ...) \
|
|
- if (!group->backend->func) { \
|
|
+virCgroupBackendPtr
|
|
+virCgroupBackendForController(virCgroupPtr group,
|
|
+ unsigned int controller);
|
|
+
|
|
+# define VIR_CGROUP_BACKEND_CALL(group, controller, func, ret, ...) \
|
|
+ virCgroupBackendPtr backend = virCgroupBackendForController(group, controller); \
|
|
+ if (!backend) { \
|
|
+ virReportError(VIR_ERR_INTERNAL_ERROR, \
|
|
+ _("failed to get cgroup backend for '%s'"), #func); \
|
|
+ return ret; \
|
|
+ } \
|
|
+ if (!backend->func) { \
|
|
virReportError(VIR_ERR_OPERATION_UNSUPPORTED, \
|
|
_("operation '%s' not supported"), #func); \
|
|
return ret; \
|
|
} \
|
|
- return group->backend->func(group, ##__VA_ARGS__);
|
|
+ return backend->func(group, ##__VA_ARGS__);
|
|
|
|
#endif /* __VIR_CGROUP_BACKEND_H__ */
|
|
diff --git a/src/util/vircgrouppriv.h b/src/util/vircgrouppriv.h
|
|
index 4a0d75ddbc..8f24b0891e 100644
|
|
--- a/src/util/vircgrouppriv.h
|
|
+++ b/src/util/vircgrouppriv.h
|
|
@@ -56,7 +56,7 @@ typedef virCgroupV2Controller *virCgroupV2ControllerPtr;
|
|
struct _virCgroup {
|
|
char *path;
|
|
|
|
- virCgroupBackendPtr backend;
|
|
+ virCgroupBackendPtr backends[VIR_CGROUP_BACKEND_TYPE_LAST];
|
|
|
|
virCgroupV1Controller legacy[VIR_CGROUP_CONTROLLER_LAST];
|
|
virCgroupV2Controller unified;
|
|
--
|
|
2.22.0
|
|
|