This series of patches introduced the x86 Cache Monitoring Technology (CMT) to libvirt by interacting with kernel resource control (resctrl) interface. CMT is one of the Intel(R) x86 CPU feature which belongs to the Resource Director Technology (RDT). CMT reports the occupancy of the last level cache, which is shared by all CPU cores. We have serval discussion about the enabling of CMT, please refer to following links for the RFCs. RFCv3 https://www.redhat.com/archives/libvir-list/2018-August/msg01213.html RFCv2 https://www.redhat.com/archives/libvir-list/2018-July/msg00409.html https://www.redhat.com/archives/libvir-list/2018-July/msg01241.html RFCv1 https://www.redhat.com/archives/libvir-list/2018-June/msg00674.html 1. About reason why CMT is necessary in libvirt? The perf events of 'CMT, MBML, MBMT' have been phased out since Linux kernel commit c39a0e2c8850f08249383f2425dbd8dbe4baad69, in libvirt the perf based cmt,mbm will not work with the latest linux kernel. These patches add CMT feature to libvirt through kernel resctrlfs interface. 2. Interfaces for CMT from the high level. 2.1 Query the host capability of CMT. The element 'monitor' represents the host capabilities of CMT. The explanations of involved CMT attributes: - 'maxAllocs' denotes the maximum monitoring groups could be created, which is limited by the number of hardware 'RMID'. - 'threshold' denotes the upper bound of cache occupancy for current group, in bytes, to determine if an RMID can be reused. - element 'feature' denotes the monitoring feature supported. - 'llc_occupancy' is the feature for reporting the last level cache occupancy information. # virsh capabilities ... <cache> <bank id='0' level='3' type='both' size='15' unit='MiB' cpus='0-5'> <control granularity='768' unit='KiB' type='code' maxAllocs='8'/> <control granularity='768' unit='KiB' type='data' maxAllocs='8'/> + <monitor threshold='540672' unit='B' maxAllocs='176'/> + <feature name=llc_occupancy/> + </monitor> </bank> <bank id='1' level='3' type='both' size='15' unit='MiB' cpus='6-11'> <control granularity='768' unit='KiB' type='code' maxAllocs='8'/> <control granularity='768' unit='KiB' type='data' maxAllocs='8'/> + <monitor threshold='540672' unit='B' maxAllocs='176'/> + <feature name=llc_occupancy/> + </monitor> </bank> </cache> ... 2.2 Create cache monitoring group (cache monitor). The main interface for creating monitoring group is through XML file. The proposed configuration is like: <cputune> <cachetune vcpus='1'> <cache id='0' level='3' type='code' size='7680' unit='KiB'/> <cache id='1' level='3' type='data' size='3840' unit='KiB'/> + <monitor vcpus='1'/> </cachetune> <cachetune vcpus='4-7'> + <monitor vcpus='4-6'/> </cachetune> </cputune> In above XML, created 2 cache resctrl allocation groups and 2 resctrl monitoring groups. The changes of cache monitor will be effective in next booting of VM. 2.3 Show CMT result through command 'domstats' Adding the interface in qemu to report this information for resource monitor group through command 'virsh domstats --cpu-total'. Below is a typical output: # virsh domstats 1 --cpu-total Domain: 'ubuntu16.04-base' ... cpu.cache.monitor.count=2 cpu.cache.0.name=vcpus_1 cpu.cache.0.vcpus=1 cpu.cache.0.bank.count=2 cpu.cache.0.bank.0.id=0 cpu.cache.0.bank.0.bytes=4505600 cpu.cache.0.bank.1.id=1 cpu.cache.0.bank.1.bytes=5586944 cpu.cache.1.name=vcpus_4-6 cpu.cache.1.vcpus=4,5,6 cpu.cache.1.bank.count=2 cpu.cache.1.bank.0.id=0 cpu.cache.1.bank.0.bytes=17571840 cpu.cache.1.bank.1.id=1 cpu.cache.1.bank.1.bytes=29106176 **Changes Since RFCv3** In the output of 'domstats', added 'cpu.cache.<cmt_group_index>.bank.<bank_index>.id' to tell the OS assigned cache bank id of current cache. Changes is prefixed with a '+': # virsh domstats 1 --cpu-total Domain: 'ubuntu16.04-base' ... cpu.cache.monitor.count=2 cpu.cache.0.name=vcpus_1 cpu.cache.0.vcpus=1 cpu.cache.0.bank.count=2 + cpu.cache.0.bank.0.id=0 cpu.cache.0.bank.0.bytes=4505600 + cpu.cache.0.bank.1.id=1 cpu.cache.0.bank.1.bytes=5586944 cpu.cache.1.name=vcpus_4-6 cpu.cache.1.vcpus=4,5,6 cpu.cache.1.bank.count=2 + cpu.cache.1.bank.0.id=0 cpu.cache.1.bank.0.bytes=17571840 + cpu.cache.1.bank.1.id=1 cpu.cache.1.bank.1.bytes=29106176 Wang Huaqiang (10): conf: Renamed 'controlBuf' to 'childrenBuf' util: add interface retrieving CMT capability conf: Add CMT capability to host test: add test case for resctrl monitor util: resctrl: refactoring some functions util: Introduce resctrl monitor for CMT conf: refactor virDomainResctrlAppend conf: introduce resctrl monitor group in domain qemu: Introduce resctrl monitoring group qemu: Report cache occupancy (CMT) with domstats .gnulib | 1 - docs/formatdomain.html.in | 14 +- docs/schemas/capability.rng | 28 + docs/schemas/domaincommon.rng | 11 +- src/conf/capabilities.c | 51 +- src/conf/capabilities.h | 1 + src/conf/domain_conf.c | 159 +++++- src/conf/domain_conf.h | 20 + src/libvirt-domain.c | 9 + src/libvirt_private.syms | 6 + src/qemu/qemu_driver.c | 265 ++++++++- src/qemu/qemu_process.c | 40 +- src/util/virresctrl.c | 597 +++++++++++++++++++-- src/util/virresctrl.h | 48 +- tests/genericxml2xmlindata/cachetune-cdp.xml | 2 + .../cachetune-colliding-monitors.xml | 36 ++ tests/genericxml2xmlindata/cachetune-small.xml | 1 + tests/genericxml2xmlindata/cachetune.xml | 3 + tests/genericxml2xmltest.c | 4 + .../resctrl/info/L3_MON/max_threshold_occupancy | 1 + .../linux-resctrl/resctrl/info/L3_MON/mon_features | 3 + .../linux-resctrl/resctrl/info/L3_MON/num_rmids | 1 + tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml | 6 + 23 files changed, 1208 insertions(+), 99 deletions(-) delete mode 160000 .gnulib create mode 100644 tests/genericxml2xmlindata/cachetune-colliding-monitors.xml create mode 100644 tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/max_threshold_occupancy create mode 100644 tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/mon_features create mode 100644 tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/num_rmids -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list To add CMT/MBM feature and let code be consistent in later patches, renaming variable name from 'controlBuf' to 'childrenBuf', locates in functions 'virCapabilitiesFormatCaches' and 'virCapabilitiesFormatMemoryBandwidth'. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- .gnulib | 1 - src/conf/capabilities.c | 28 ++++++++++++++-------------- 2 files changed, 14 insertions(+), 15 deletions(-) delete mode 160000 .gnulib diff --git a/.gnulib b/.gnulib deleted file mode 160000 index XXXXXXX..XXXXXXX --- a/.gnulib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 68df637b5f1b5c10370f6981d2a43a5cf74368df diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/capabilities.c +++ b/src/conf/capabilities.c @@ -XXX,XX +XXX,XX @@ virCapabilitiesFormatCaches(virBufferPtr buf, { size_t i = 0; size_t j = 0; - virBuffer controlBuf = VIR_BUFFER_INITIALIZER; + virBuffer childrenBuf = VIR_BUFFER_INITIALIZER; if (!ncaches) return 0; @@ -XXX,XX +XXX,XX @@ virCapabilitiesFormatCaches(virBufferPtr buf, short_size, unit, cpus_str); VIR_FREE(cpus_str); - virBufferSetChildIndent(&controlBuf, buf); + virBufferSetChildIndent(&childrenBuf, buf); for (j = 0; j < bank->ncontrols; j++) { const char *min_unit; virResctrlInfoPerCachePtr controls = bank->controls[j]; @@ -XXX,XX +XXX,XX @@ virCapabilitiesFormatCaches(virBufferPtr buf, } } - virBufferAsprintf(&controlBuf, + virBufferAsprintf(&childrenBuf, "<control granularity='%llu'", gran_short_size); if (min_short_size) - virBufferAsprintf(&controlBuf, " min='%llu'", min_short_size); + virBufferAsprintf(&childrenBuf, " min='%llu'", min_short_size); - virBufferAsprintf(&controlBuf, + virBufferAsprintf(&childrenBuf, " unit='%s' type='%s' maxAllocs='%u'/>\n", unit, virCacheTypeToString(controls->scope), controls->max_allocation); } - if (virBufferCheckError(&controlBuf) < 0) + if (virBufferCheckError(&childrenBuf) < 0) return -1; - if (virBufferUse(&controlBuf)) { + if (virBufferUse(&childrenBuf)) { virBufferAddLit(buf, ">\n"); - virBufferAddBuffer(buf, &controlBuf); + virBufferAddBuffer(buf, &childrenBuf); virBufferAddLit(buf, "</bank>\n"); } else { virBufferAddLit(buf, "/>\n"); @@ -XXX,XX +XXX,XX @@ virCapabilitiesFormatMemoryBandwidth(virBufferPtr buf, virCapsHostMemBWNodePtr *nodes) { size_t i = 0; - virBuffer controlBuf = VIR_BUFFER_INITIALIZER; + virBuffer childrenBuf = VIR_BUFFER_INITIALIZER; if (!nnodes) return 0; @@ -XXX,XX +XXX,XX @@ virCapabilitiesFormatMemoryBandwidth(virBufferPtr buf, node->id, cpus_str); VIR_FREE(cpus_str); - virBufferSetChildIndent(&controlBuf, buf); - virBufferAsprintf(&controlBuf, + virBufferSetChildIndent(&childrenBuf, buf); + virBufferAsprintf(&childrenBuf, "<control granularity='%u' min ='%u' " "maxAllocs='%u'/>\n", control->granularity, control->min, control->max_allocation); - if (virBufferCheckError(&controlBuf) < 0) + if (virBufferCheckError(&childrenBuf) < 0) return -1; - if (virBufferUse(&controlBuf)) { + if (virBufferUse(&childrenBuf)) { virBufferAddLit(buf, ">\n"); - virBufferAddBuffer(buf, &controlBuf); + virBufferAddBuffer(buf, &childrenBuf); virBufferAddLit(buf, "</node>\n"); } else { virBufferAddLit(buf, "/>\n"); -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Introduce function for reporting CMT capability through going through files under /sys/fs/info/L3_MON. This patch is co-work with later patches and report these information to domain. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/conf/capabilities.c | 6 ++- src/conf/capabilities.h | 1 + src/util/virresctrl.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++-- src/util/virresctrl.h | 17 ++++++- 4 files changed, 137 insertions(+), 7 deletions(-) diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/capabilities.c +++ b/src/conf/capabilities.c @@ -XXX,XX +XXX,XX @@ virCapsHostCacheBankFree(virCapsHostCacheBankPtr ptr) virBitmapFree(ptr->cpus); for (i = 0; i < ptr->ncontrols; i++) VIR_FREE(ptr->controls[i]); + if (ptr->monitor && ptr->monitor->features) + virStringListFree(ptr->monitor->features); + VIR_FREE(ptr->monitor); VIR_FREE(ptr->controls); VIR_FREE(ptr); } @@ -XXX,XX +XXX,XX @@ virCapabilitiesInitCaches(virCapsPtr caps) bank->level, bank->size, &bank->ncontrols, - &bank->controls) < 0) + &bank->controls, + &bank->monitor) < 0) goto cleanup; if (VIR_APPEND_ELEMENT(caps->host.caches, diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/capabilities.h +++ b/src/conf/capabilities.h @@ -XXX,XX +XXX,XX @@ struct _virCapsHostCacheBank { virBitmapPtr cpus; /* All CPUs that share this bank */ size_t ncontrols; virResctrlInfoPerCachePtr *controls; + virResctrlInfoMonPtr monitor; }; typedef struct _virCapsHostMemBWNode virCapsHostMemBWNode; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ struct _virResctrlInfo { size_t nlevels; virResctrlInfoMemBWPtr membw_info; + + virResctrlInfoMonPtr monitor_info; }; @@ -XXX,XX +XXX,XX @@ virResctrlInfoDispose(void *obj) VIR_FREE(level); } + if (resctrl->monitor_info) + virStringListFree(resctrl->monitor_info->features); + VIR_FREE(resctrl->monitor_info); VIR_FREE(resctrl->membw_info); VIR_FREE(resctrl->levels); } @@ -XXX,XX +XXX,XX @@ virResctrlGetMemoryBandwidthInfo(virResctrlInfoPtr resctrl) static int +virResctrlGetMonitorInfo(virResctrlInfoPtr resctrl) +{ + int rv = -1; + char *featurestr = NULL; + char **lines = NULL; + size_t nlines = 0; + size_t i = 0; + int ret = -1; + virResctrlInfoMonPtr info = NULL; + + if (VIR_ALLOC(info) < 0) + return -1; + + rv = virFileReadValueUint(&info->max_allocation, + SYSFS_RESCTRL_PATH "/info/L3_MON/num_rmids"); + if (rv == -2) { + /* The file doesn't exist, so it's unusable for us, + * probably resource monitoring feature unsupported */ + VIR_WARN("The path '" SYSFS_RESCTRL_PATH "/info/L3_MON/num_rmids' " + "does not exist"); + + ret = 0; + goto cleanup; + } else if (rv < 0) { + /* Other failures are fatal, so just quit */ + goto cleanup; + } + + rv = virFileReadValueUint(&info->cache_threshold, + SYSFS_RESCTRL_PATH + "/info/L3_MON/max_threshold_occupancy"); + + if (rv == -2) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot get max_threshold_occupancy from resctrl" + " info")); + } + if (rv < 0) + goto cleanup; + + rv = virFileReadValueString(&featurestr, + SYSFS_RESCTRL_PATH + "/info/L3_MON/mon_features"); + if (rv == -2) + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot get mon_features from resctrl info")); + if (rv < 0) + goto cleanup; + + lines = virStringSplitCount(featurestr, "\n", 0, &nlines); + + for (i = 0; i < nlines; i++) { + if (STREQLEN(lines[i], "llc_", strlen("llc_")) || + STREQLEN(lines[i], "mbm_", strlen("mbm_"))) { + if (virStringListAdd(&info->features, lines[i]) < 0) + goto cleanup; + info->nfeatures++; + } + } + + VIR_FREE(featurestr); + virStringListFree(lines); + resctrl->monitor_info = info; + return 0; + + cleanup: + VIR_FREE(featurestr); + virStringListFree(lines); + virStringListFree(info->features); + VIR_FREE(info); + return ret; +} + + +static int virResctrlGetInfo(virResctrlInfoPtr resctrl) { DIR *dirp = NULL; @@ -XXX,XX +XXX,XX @@ virResctrlGetInfo(virResctrlInfoPtr resctrl) if (ret < 0) goto cleanup; + ret = virResctrlGetMonitorInfo(resctrl); + if (ret < 0) + goto cleanup; + ret = virResctrlGetCacheInfo(resctrl, dirp); if (ret < 0) goto cleanup; @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl, unsigned int level, unsigned long long size, size_t *ncontrols, - virResctrlInfoPerCachePtr **controls) + virResctrlInfoPerCachePtr **controls, + virResctrlInfoMonPtr *monitor) { virResctrlInfoPerLevelPtr i_level = NULL; virResctrlInfoPerTypePtr i_type = NULL; + virResctrlInfoMonPtr cachemon = NULL; size_t i = 0; int ret = -1; if (virResctrlInfoIsEmpty(resctrl)) return 0; + if (VIR_ALLOC(cachemon) < 0) + return -1; + /* Let's take the opportunity to update the number of last level * cache. This number of memory bandwidth controller is same with * last level cache */ @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl, memcpy((*controls)[*ncontrols - 1], &i_type->control, sizeof(i_type->control)); } - ret = 0; - cleanup: - return ret; + cachemon->max_allocation = 0; + + if (resctrl->monitor_info) { + virResctrlInfoMonPtr info = resctrl->monitor_info; + + cachemon->max_allocation = info->max_allocation; + cachemon->cache_threshold = info->cache_threshold; + for (i = 0; i < info->nfeatures; i++) { + /* Only cares about last level cache */ + if (STREQLEN(info->features[i], "llc_", strlen("llc_"))) { + if (virStringListAdd(&cachemon->features, + info->features[i]) < 0) + goto error; + cachemon->nfeatures++; + } + } + } + + if (cachemon->features) + *monitor = cachemon; + + return 0; error: while (*ncontrols) VIR_FREE((*controls)[--*ncontrols]); VIR_FREE(*controls); - goto cleanup; + virStringListFree(cachemon->features); + VIR_FREE(cachemon); + return ret; } diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ struct _virResctrlInfoMemBWPerNode { unsigned int max_allocation; }; +typedef struct _virResctrlInfoMon virResctrlInfoMon; +typedef virResctrlInfoMon *virResctrlInfoMonPtr; +/* Information about resource monitoring group */ +struct _virResctrlInfoMon { + /* null-terminal string list for hw supported monitor feature */ + char **features; + size_t nfeatures; + /* Maximum number of simultaneous allocations */ + unsigned int max_allocation; + /* determines the occupancy at which an RMID can be freed */ + unsigned int cache_threshold; +}; + typedef struct _virResctrlInfo virResctrlInfo; typedef virResctrlInfo *virResctrlInfoPtr; @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl, unsigned int level, unsigned long long size, size_t *ncontrols, - virResctrlInfoPerCachePtr **controls); + virResctrlInfoPerCachePtr **controls, + virResctrlInfoMonPtr *monitor); + int virResctrlInfoGetMemoryBandwidth(virResctrlInfoPtr resctrl, -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list CMT capability for each cache bank, includes -. Maximum CMT monitoring groups(sharing with MBM) could be created, which reflects the maximum hardware RMID count. -. 'cache threshold'. -. Statistical information of last level cache, the actual cache occupancy. cache is splitted into 'bank's, each bank MAY have different cache configuration, report cache monitoring capability in unit of cache bank. cache monitor capability is shown as below: <cache> <bank id='0' level='3' type='both' size='15' unit='MiB' cpus='0-5'> <control granularity='768' unit='KiB' type='code' maxAllocs='8'/> <control granularity='768' unit='KiB' type='data' maxAllocs='8'/> + <monitor threshold='540672' unit='B' maxAllocs='176'/> + <feature name=llc_occupancy/> + </monitor> </bank> <bank id='1' level='3' type='both' size='15' unit='MiB' cpus='6-11'> <control granularity='768' unit='KiB' type='code' maxAllocs='8'/> <control granularity='768' unit='KiB' type='data' maxAllocs='8'/> + <monitor threshold='540672' unit='B' maxAllocs='176'/> + <feature name=llc_occupancy/> + </monitor> </bank> </cache> Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- docs/schemas/capability.rng | 28 ++++++++++++++++++++++++++++ src/conf/capabilities.c | 17 +++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng index XXXXXXX..XXXXXXX 100644 --- a/docs/schemas/capability.rng +++ b/docs/schemas/capability.rng @@ -XXX,XX +XXX,XX @@ </attribute> </element> </zeroOrMore> + <zeroOrMore> + <element name='monitor'> + <attribute name='threshold'> + <ref name='unsignedInt'/> + </attribute> + <attribute name='unit'> + <ref name='unit'/> + </attribute> + <attribute name='maxAllocs'> + <ref name='unsignedInt'/> + </attribute> + <zeroOrMore> + <element name='feature'> + <ref name='monitorFeature'/> + </element> + </zeroOrMore> + </element> + </zeroOrMore> </element> </oneOrMore> </element> @@ -XXX,XX +XXX,XX @@ </attribute> </define> + <define name='monitorFeature'> + <attribute name='name'> + <choice> + <value>llc_occupancy</value> + <value>mbm_total_bytes</value> + <value>mbm_local_bytes</value> + </choice> + </attribute> + </define> + <define name='memory_bandwidth'> <element name='memory_bandwidth'> <oneOrMore> diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/capabilities.c +++ b/src/conf/capabilities.c @@ -XXX,XX +XXX,XX @@ virCapabilitiesFormatCaches(virBufferPtr buf, controls->max_allocation); } + if (bank->monitor && + bank->monitor->nfeatures) { + virBufferAsprintf(&childrenBuf, + "<monitor threshold='%u' unit='B' " + "maxAllocs='%u'>\n", + bank->monitor->cache_threshold, + bank->monitor->max_allocation); + for (j = 0; j < bank->monitor->nfeatures; j++) { + virBufferAdjustIndent(&childrenBuf, 2); + virBufferAsprintf(&childrenBuf, + "<feature name='%s'/>\n", + bank->monitor->features[j]); + virBufferAdjustIndent(&childrenBuf, -2); + } + virBufferAddLit(&childrenBuf, "</monitor>\n"); + } + if (virBufferCheckError(&childrenBuf) < 0) return -1; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- .../linux-resctrl/resctrl/info/L3_MON/max_threshold_occupancy | 1 + .../vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/mon_features | 3 +++ tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/num_rmids | 1 + tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml | 6 ++++++ 4 files changed, 11 insertions(+) create mode 100644 tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/max_threshold_occupancy create mode 100644 tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/mon_features create mode 100644 tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/num_rmids diff --git a/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/max_threshold_occupancy b/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/max_threshold_occupancy new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/max_threshold_occupancy @@ -0,0 +1 @@ +270336 diff --git a/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/mon_features b/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/mon_features new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/mon_features @@ -XXX,XX +XXX,XX @@ +llc_occupancy +mbm_total_bytes +mbm_local_bytes diff --git a/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/num_rmids b/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/num_rmids new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/vircaps2xmldata/linux-resctrl/resctrl/info/L3_MON/num_rmids @@ -0,0 +1 @@ +176 diff --git a/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml b/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml index XXXXXXX..XXXXXXX 100644 --- a/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml +++ b/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml @@ -XXX,XX +XXX,XX @@ <cache> <bank id='0' level='3' type='both' size='15' unit='MiB' cpus='0-5'> <control granularity='768' min='1536' unit='KiB' type='both' maxAllocs='4'/> + <monitor threshold='270336' unit='B' maxAllocs='176'> + <feature name='llc_occupancy'/> + </monitor> </bank> <bank id='1' level='3' type='both' size='15' unit='MiB' cpus='6-11'> <control granularity='768' min='1536' unit='KiB' type='both' maxAllocs='4'/> + <monitor threshold='270336' unit='B' maxAllocs='176'> + <feature name='llc_occupancy'/> + </monitor> </bank> </cache> <memory_bandwidth> -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Some code, in virresctrl.c, manupulating the file objects of resctrlfs could be reused for cache monitor interfaces. This patch refactor these functions for purpose of reusing code in later patch: virResctrlAllocDeterminePath virResctrlAllocCreate virResctrlAddPID Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/util/virresctrl.c | 126 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 93 insertions(+), 33 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlAllocAssign(virResctrlInfoPtr resctrl, } -int -virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, - const char *machinename) +static int +virResctrlDeterminePath(const char *id, + const char *root, + const char *parentpath, + const char *prefix, + char **path) { - if (!alloc->id) { + if (!id) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Resctrl Allocation ID must be set before creation")); + _("Resctrl resource ID must be set before creation")); return -1; } - if (!alloc->path && - virAsprintf(&alloc->path, "%s/%s-%s", - SYSFS_RESCTRL_PATH, machinename, alloc->id) < 0) + if (*path) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Resctrl group (%s) already created, path=%s."), + id, *path); return -1; + } + + if (!parentpath && !root) { + if (virAsprintf(path, "%s/%s-%s", + SYSFS_RESCTRL_PATH, prefix, id) < 0) + return -1; + } else if (!parentpath) { + if (virAsprintf(path, "%s/%s/%s-%s", + SYSFS_RESCTRL_PATH, parentpath, prefix, id) < 0) + return -1; + } else { + if (virAsprintf(path, "%s/%s/%s-%s", + root, parentpath, prefix, id) < 0) + return -1; + } return 0; } +int +virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, + const char *machinename) +{ + return virResctrlDeterminePath(alloc->id, NULL, NULL, + machinename, &alloc->path); +} + +static int +virResctrlCreateGroup(virResctrlInfoPtr resctrl, + char *path) +{ + int ret = -1; + int lockfd = -1; + + if (!path) + return -1; + + if (virResctrlInfoIsEmpty(resctrl)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Resource control is not supported on this host")); + return -1; + } + + if (STREQ(path, SYSFS_RESCTRL_PATH)) + return 0; + + if (virFileExists(path)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Path '%s' for resctrl resource group exists"), path); + goto cleanup; + } + + lockfd = virResctrlLockWrite(); + if (lockfd < 0) + goto cleanup; + + if (virFileMakePath(path) < 0) { + virReportSystemError(errno, + _("Cannot create resctrl directory '%s'"), path); + goto cleanup; + } + + ret = 0; + cleanup: + virResctrlUnlock(lockfd); + return ret; +} + + /* This checks if the directory for the alloc exists. If not it tries to create * it and apply appropriate alloc settings. */ int @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (!alloc) return 0; - if (virResctrlInfoIsEmpty(resctrl)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("Resource control is not supported on this host")); - return -1; - } - if (virResctrlAllocDeterminePath(alloc, machinename) < 0) return -1; - if (virFileExists(alloc->path)) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Path '%s' for resctrl allocation exists"), - alloc->path); - goto cleanup; - } + if (virResctrlCreateGroup(resctrl, alloc->path) < 0) + return -1; lockfd = virResctrlLockWrite(); if (lockfd < 0) @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virAsprintf(&schemata_path, "%s/schemata", alloc->path) < 0) goto cleanup; - if (virFileMakePath(alloc->path) < 0) { - virReportSystemError(errno, - _("Cannot create resctrl directory '%s'"), - alloc->path); - goto cleanup; - } - VIR_DEBUG("Writing resctrl schemata '%s' into '%s'", alloc_str, schemata_path); if (virFileWriteStr(schemata_path, alloc_str, 0) < 0) { rmdir(alloc->path); @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, } -int -virResctrlAllocAddPID(virResctrlAllocPtr alloc, - pid_t pid) +static int +virResctrlAddPID(char *path, + pid_t pid) { char *tasks = NULL; char *pidstr = NULL; int ret = 0; - if (!alloc->path) { + if (!path) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Cannot add pid to non-existing resctrl allocation")); + _("Cannot add pid to non-existing resctrl group")); return -1; } - if (virAsprintf(&tasks, "%s/tasks", alloc->path) < 0) + if (virAsprintf(&tasks, "%s/tasks", path) < 0) return -1; if (virAsprintf(&pidstr, "%lld", (long long int) pid) < 0) @@ -XXX,XX +XXX,XX @@ virResctrlAllocAddPID(virResctrlAllocPtr alloc, int +virResctrlAllocAddPID(virResctrlAllocPtr alloc, + pid_t pid) +{ + return virResctrlAddPID(alloc->path, pid); +} + + +int virResctrlAllocRemove(virResctrlAllocPtr alloc) { int ret = 0; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list 'virResctrlAllocMon' denotes a resctrl monitor reporting the resource consumption information. This patch introduced the interfaces for resctrl monitor. Relationship of 'resctrl allocation' and 'resctrl monitor': 1. resctrl monitor monitors resources (cache or memory bandwidth) of particular allocation. 2. resctrl allocation may refer to the 'default' allocation if no dedicated resource 'control' applied to it. The 'default' allocation enjoys remaining resource that not allocated. 3. resctrl monitor belongs to 'default' allocation if no 'cachetune' specified in XML file. 4. one resctrl allocation may have several monitors. It is also permitted that there is no resctrl monitor associated with an allocation. Key data structures: + struct _virResctrlAllocMon { + char *id; + char *path; + }; struct _virResctrlAlloc { virObject parent; @@ -276,6 +289,12 @@ struct _virResctrlAlloc { virResctrlAllocMemBWPtr mem_bw; + virResctrlAllocMonPtr *monitors; + size_t nmonitors; } Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt_private.syms | 6 + src/util/virresctrl.c | 361 ++++++++++++++++++++++++++++++++++++++++++++++- src/util/virresctrl.h | 31 ++++ 3 files changed, 394 insertions(+), 4 deletions(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virCacheKernelTypeFromString; virCacheKernelTypeToString; virCacheTypeFromString; virCacheTypeToString; +virResctrlAllocAddMonitorPID; virResctrlAllocAddPID; virResctrlAllocCreate; +virResctrlAllocCreateMonitor; +virResctrlAllocDeleteMonitor; +virResctrlAllocDetermineMonitorPath; virResctrlAllocDeterminePath; virResctrlAllocForeachCache; virResctrlAllocForeachMemory; virResctrlAllocFormat; +virResctrlAllocGetCacheOccupancy; virResctrlAllocGetID; virResctrlAllocGetUnused; virResctrlAllocNew; @@ -XXX,XX +XXX,XX @@ virResctrlAllocRemove; virResctrlAllocSetCacheSize; virResctrlAllocSetID; virResctrlAllocSetMemoryBandwidth; +virResctrlAllocSetMonitor; virResctrlInfoGetCache; virResctrlInfoNew; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ struct _virResctrlAllocMemBW { size_t nbandwidths; }; + +typedef struct _virResctrlAllocMon virResctrlAllocMon; +typedef virResctrlAllocMon *virResctrlAllocMonPtr; +/* virResctrlAllocMon denotes a resctrl monitoring group reporting the resource + * consumption information for resource of either cache or memory + * bandwidth. */ +struct _virResctrlAllocMon { + /* monitoring group identifier, should be unique in scope of allocation */ + char *id; + /* directory path under /sys/fs/resctrl*/ + char *path; +}; + struct _virResctrlAlloc { virObject parent; @@ -XXX,XX +XXX,XX @@ struct _virResctrlAlloc { virResctrlAllocMemBWPtr mem_bw; + /* monintoring groups associated with current resource allocation + * it might report resource consumption information at a finer + * granularity */ + virResctrlAllocMonPtr *monitors; + size_t nmonitors; + /* The identifier (any unique string for now) */ char *id; /* libvirt-generated path in /sys/fs/resctrl for this particular * allocation */ char *path; + /* is this a default resctrl group? + * true : default group, directory path equals '/sys/fs/resctrl' + * false: non-default group */ + bool default_group; }; @@ -XXX,XX +XXX,XX @@ virResctrlAllocDispose(void *obj) VIR_FREE(alloc->mem_bw); } + for (i = 0; i < alloc->nmonitors; i++) { + virResctrlAllocMonPtr monitor = alloc->monitors[i]; + VIR_FREE(monitor->id); + VIR_FREE(monitor->path); + VIR_FREE(monitor); + } + VIR_FREE(alloc->monitors); VIR_FREE(alloc->id); VIR_FREE(alloc->path); VIR_FREE(alloc->levels); @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl, memcpy((*controls)[*ncontrols - 1], &i_type->control, sizeof(i_type->control)); } + cachemon->nfeatures = 0; cachemon->max_allocation = 0; if (resctrl->monitor_info) { @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl, if (STREQLEN(info->features[i], "llc_", strlen("llc_"))) { if (virStringListAdd(&cachemon->features, info->features[i]) < 0) - goto error; + goto error; cachemon->nfeatures++; } } @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl, virResctrlAllocPtr virResctrlAllocNew(void) { + virResctrlAllocPtr ret = NULL; + if (virResctrlInitialize() < 0) return NULL; - return virObjectNew(virResctrlAllocClass); + ret = virObjectNew(virResctrlAllocClass); + if (!ret) + return NULL; + + /* By default, a resource group is a default group */ + ret->default_group = true; + + return ret; } @@ -XXX,XX +XXX,XX @@ virResctrlAllocIsEmpty(virResctrlAllocPtr alloc) if (alloc->mem_bw) return false; + if (alloc->nmonitors) + return false; + for (i = 0; i < alloc->nlevels; i++) { virResctrlAllocPerLevelPtr a_level = alloc->levels[i]; @@ -XXX,XX +XXX,XX @@ int virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, const char *machinename) { - return virResctrlDeterminePath(alloc->id, NULL, NULL, - machinename, &alloc->path); + if (alloc->default_group) { + if (VIR_STRDUP(alloc->path, SYSFS_RESCTRL_PATH) < 0) + return -1; + return 0; + } else { + + return virResctrlDeterminePath(alloc->id, NULL, NULL, + machinename, &alloc->path); + } } + static int virResctrlCreateGroup(virResctrlInfoPtr resctrl, char *path) @@ -XXX,XX +XXX,XX @@ virResctrlCreateGroup(virResctrlInfoPtr resctrl, return ret; } + /* In case of no explicit requirement for allocating cache and memory + * bandwidth, set 'alloc->default' to 'true', then the monitoring + * group will be created under '/sys/fs/resctrl/mon_groups' in later + * invocation of virResctrlAllocCreate. + * Otherwise, set 'alloc->default' to false, create a new directory + * under '/sys/fs/resctrl/'. This is will cost a hardware 'COSID'.*/ +static int +virResctrlAllocCheckDefault(virResctrlAllocPtr alloc) +{ + bool default_group = true; + if (!alloc) + return -1; + + if (alloc->nlevels) + default_group = false; + if (alloc->mem_bw && alloc->mem_bw->nbandwidths) + default_group = false; + + alloc->default_group = default_group; + return 0; +} /* This checks if the directory for the alloc exists. If not it tries to create * it and apply appropriate alloc settings. */ @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (!alloc) return 0; + virResctrlAllocCheckDefault(alloc); + if (virResctrlAllocDeterminePath(alloc, machinename) < 0) return -1; @@ -XXX,XX +XXX,XX @@ virResctrlAllocRemove(virResctrlAllocPtr alloc) return 0; VIR_DEBUG("Removing resctrl allocation %s", alloc->path); + + while (alloc->nmonitors > 0) { + ret = virResctrlAllocDeleteMonitor(alloc, alloc->monitors[0]->id); + if (ret < 0) + goto cleanup; + } + if (rmdir(alloc->path) != 0 && errno != ENOENT) { ret = -errno; VIR_ERROR(_("Unable to remove %s (%d)"), alloc->path, errno); } + ret = 0; + cleanup: + return ret; +} + + +static int +virResctrlAllocGetMonitor(virResctrlAllocPtr alloc, + const char *id, + virResctrlAllocMonPtr *monitor, + size_t *pos) +{ + size_t i = 0; + + if (!alloc || !id) + return -1; + + for (i = 0; i < alloc->nmonitors; i++) { + if (alloc->monitors[i]->id && + STREQ(id, (alloc->monitors[i])->id)) { + if (monitor) + *monitor = alloc->monitors[i]; + if (pos) + *pos = i; + return 0; + } + } + + return -1; +} + + +int +virResctrlAllocDetermineMonitorPath(virResctrlAllocPtr alloc, + const char *id, + const char *machinename) +{ + virResctrlAllocMonPtr monitor = NULL; + + if (virResctrlAllocGetMonitor(alloc, id, &monitor, NULL) < 0) + return -1; + + + return virResctrlDeterminePath(monitor->id, + alloc->path, + "mon_groups", + machinename, + &monitor->path); +} + + +int +virResctrlAllocAddMonitorPID(virResctrlAllocPtr alloc, + const char *id, + pid_t pid) +{ + virResctrlAllocMonPtr monitor = NULL; + + if (virResctrlAllocGetMonitor(alloc, id, &monitor, NULL) < 0) + return -1; + + return virResctrlAddPID(monitor->path, pid); +} + + +int +virResctrlAllocSetMonitor(virResctrlAllocPtr alloc, + const char *id) +{ + virResctrlAllocMonPtr monitor = NULL; + + if (!alloc || !id) + return - 1; + + if (virResctrlAllocGetMonitor(alloc, id, &monitor, NULL) < 0) { + if (VIR_ALLOC(monitor) < 0) + return -1; + } + + if (VIR_STRDUP(monitor->id, (char*)id) < 0) + return -1; + + if (VIR_APPEND_ELEMENT(alloc->monitors, alloc->nmonitors, monitor) < 0) + return -1; + + return 0; +} + +int +virResctrlAllocCreateMonitor(virResctrlInfoPtr resctrl, + virResctrlAllocPtr alloc, + const char *machinename, + const char *id) +{ + virResctrlAllocMonPtr monitor = NULL; + + if (virResctrlAllocGetMonitor(alloc, id, &monitor, NULL) < 0) + return - 1; + + if (virResctrlAllocDetermineMonitorPath(alloc, id, machinename) < 0) + return -1; + + VIR_DEBUG("Creating resctrl monitor %s", monitor->path); + if (virResctrlCreateGroup(resctrl, monitor->path) < 0) + return -1; + + return 0; +} + + +int +virResctrlAllocDeleteMonitor(virResctrlAllocPtr alloc, + const char *id) +{ + int ret = 0; + + virResctrlAllocMonPtr monitor = NULL; + size_t pos = 0; + + if (virResctrlAllocGetMonitor(alloc, id, &monitor, &pos) < 0) + return -1; + + VIR_DELETE_ELEMENT(alloc->monitors, pos, alloc->nmonitors); + + VIR_DEBUG("Deleting resctrl monitor %s ", monitor->path); + if (rmdir(monitor->path) != 0 && errno != ENOENT) { + ret = -errno; + VIR_ERROR(_("Unable to remove %s (%d)"), monitor->path, errno); + } + + VIR_FREE(monitor->id); + VIR_FREE(monitor->path); + VIR_FREE(monitor); + return ret; +} + + +static int +virResctrlAllocGetStatistic(virResctrlAllocPtr alloc, + const char *id, + const char *resfile, + unsigned int *nnodes, + unsigned int **nodeids, + unsigned int **nodevals) +{ + DIR *dirp = NULL; + int ret = -1; + int rv = -1; + struct dirent *ent = NULL; + virBuffer buf = VIR_BUFFER_INITIALIZER; + char *mondatapath = NULL; + size_t ntmpid = 0; + size_t ntmpval = 0; + virResctrlAllocMonPtr monitor = NULL; + + if (!nnodes || !nodeids || !nodevals) + return -1; + + if (virResctrlAllocGetMonitor(alloc, id, &monitor, NULL) < 0) + goto cleanup; + + if (!monitor || !monitor->path) + goto cleanup; + + rv = virDirOpenIfExists(&dirp, monitor->path); + if (rv <= 0) + goto cleanup; + + *nnodes = 0; + + virBufferAsprintf(&buf, "%s/mon_data", monitor->path); + + mondatapath = virBufferContentAndReset(&buf); + if (!mondatapath) + goto cleanup; + + if (virDirOpen(&dirp, mondatapath) < 0) + goto cleanup; + + while ((rv = virDirRead(dirp, &ent, mondatapath)) > 0) { + char *pstrid = NULL; + size_t i = 0; + unsigned int len = 0; + unsigned int counter = 0; + unsigned int cacheid = 0; + unsigned int cur_cacheid = 0; + unsigned int val = 0; + int tmpnodeid = 0; + int tmpnodeval = 0; + + if (ent->d_type != DT_DIR) + continue; + + /* mon_L3(|CODE|DATA)_xx, xx is cache id */ + if (STRNEQLEN(ent->d_name, "mon_L", 5)) + continue; + + len = strlen(ent->d_name); + pstrid = ent->d_name; + /* locating the cache id string: 'xx' */ + for (i = 0; i < len; i++) { + if (*(pstrid + i) == '_') + counter ++; + if (counter == 2) + break; + } + i++; + + if (i >= len) + goto cleanup; + + if (virStrToLong_uip(pstrid + i, NULL, 0, &cacheid) < 0) { + VIR_DEBUG("Cannot parse id from folder '%s'", ent->d_name); + goto cleanup; + } + + rv = virFileReadValueUint(&val, + "%s/%s/%s", + mondatapath, ent->d_name, resfile); + + if (rv == -2) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("file %s/%s/%s does not exist"), + mondatapath, ent->d_name, resfile); + goto cleanup; + } else { + if (rv < 0) + goto cleanup; + } + + /* The ultimate caller will be responiblefor free memory of + * 'nodeids' an 'nodevals' */ + if (VIR_APPEND_ELEMENT(*nodeids, ntmpid, cacheid) < 0) + goto cleanup; + if (VIR_APPEND_ELEMENT(*nodevals, ntmpval, val) < 0) + goto cleanup; + + cur_cacheid = ntmpval - 1; + /* sort the cache information in caach bank id's ascending order */ + for (i = 0; i < cur_cacheid; i++) { + if ((*nodeids)[cur_cacheid] < (*nodeids)[i]) { + tmpnodeid = (*nodeids)[cur_cacheid]; + tmpnodeval = (*nodevals)[cur_cacheid]; + (*nodeids)[cur_cacheid] = (*nodeids)[i]; + (*nodevals)[cur_cacheid] = (*nodevals)[i]; + (*nodeids)[i] = tmpnodeid; + (*nodevals)[i] = tmpnodeval; + } + } + } + + (*nnodes) = ntmpval; + ret = 0; + cleanup: + VIR_FREE(mondatapath); + VIR_DIR_CLOSE(dirp); + return ret; +} + + +int +virResctrlAllocGetCacheOccupancy(virResctrlAllocPtr alloc, + const char *id, + unsigned int *nbank, + unsigned int **bankids, + unsigned int **bankcaches) +{ + int ret = - 1; + + ret = virResctrlAllocGetStatistic(alloc, id, "llc_occupancy", + nbank, bankids, bankcaches); + return ret; } diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ virResctrlAllocAddPID(virResctrlAllocPtr alloc, int virResctrlAllocRemove(virResctrlAllocPtr alloc); +int +virResctrlAllocDetermineMonitorPath(virResctrlAllocPtr alloc, + const char *id, + const char *machinename); + +int +virResctrlAllocAddMonitorPID(virResctrlAllocPtr alloc, + const char *id, + pid_t pid); + +int +virResctrlAllocSetMonitor(virResctrlAllocPtr alloc, + const char *id); + +int +virResctrlAllocCreateMonitor(virResctrlInfoPtr resctrl, + virResctrlAllocPtr alloc, + const char *machinename, + const char *id); + +int +virResctrlAllocDeleteMonitor(virResctrlAllocPtr alloc, + const char *id); + +int +virResctrlAllocGetCacheOccupancy(virResctrlAllocPtr alloc, + const char *id, + unsigned int *nbank, + unsigned int **bankids, + unsigned int **bankcaches); + #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Changed the interface from virDomainResctrlAppend(virDomainDefPtr def, xmlNodePtr node, virResctrlAllocPtr alloc, virBitmapPtr vcpus, unsigned int flags); to virDomainResctrlAppend(virDomainDefPtr def, xmlNodePtr node, virDomainResctrlDefPtr resctrl, unsigned int flags); Changes will let virDomainRestrlAppend pass through more information with virDomainResctrlDefPtr, such as monitoring groups associated with the allocation. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/conf/domain_conf.c | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParseCache(xmlXPathContextPtr ctxt, static int virDomainResctrlAppend(virDomainDefPtr def, xmlNodePtr node, - virResctrlAllocPtr alloc, - virBitmapPtr vcpus, + virDomainResctrlDefPtr resctrl, unsigned int flags) { char *vcpus_str = NULL; char *alloc_id = NULL; - virDomainResctrlDefPtr tmp_resctrl = NULL; + virResctrlAllocPtr alloc = NULL; + virBitmapPtr vcpus = NULL; + int ret = -1; - if (VIR_ALLOC(tmp_resctrl) < 0) - goto cleanup; + if (!resctrl) + return -1; + + alloc = virObjectRef(resctrl->alloc); + vcpus = resctrl->vcpus; /* We need to format it back because we need to be consistent in the naming * even when users specify some "sub-optimal" string there. */ @@ -XXX,XX +XXX,XX @@ virDomainResctrlAppend(virDomainDefPtr def, if (virResctrlAllocSetID(alloc, alloc_id) < 0) goto cleanup; - tmp_resctrl->vcpus = vcpus; - tmp_resctrl->alloc = alloc; - - if (VIR_APPEND_ELEMENT(def->resctrls, def->nresctrls, tmp_resctrl) < 0) + if (VIR_APPEND_ELEMENT(def->resctrls, def->nresctrls, resctrl) < 0) goto cleanup; ret = 0; cleanup: - virDomainResctrlDefFree(tmp_resctrl); + virObjectUnref(alloc); VIR_FREE(alloc_id); VIR_FREE(vcpus_str); return ret; @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, xmlNodePtr *nodes = NULL; virBitmapPtr vcpus = NULL; virResctrlAllocPtr alloc = NULL; + virDomainResctrlDefPtr tmp_resctrl = NULL; + ssize_t i = 0; int n; int ret = -1; @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, goto cleanup; } - if (virDomainResctrlAppend(def, node, alloc, vcpus, flags) < 0) + if (VIR_ALLOC(tmp_resctrl) < 0) goto cleanup; - vcpus = NULL; + + tmp_resctrl->vcpus = vcpus; + tmp_resctrl->alloc = virObjectRef(alloc); + + if (virDomainResctrlAppend(def, node, tmp_resctrl, flags) < 0) + goto cleanup; + alloc = NULL; + vcpus = NULL; + tmp_resctrl = NULL; ret = 0; cleanup: ctxt->node = oldnode; virObjectUnref(alloc); + VIR_FREE(tmp_resctrl); virBitmapFree(vcpus); VIR_FREE(nodes); return ret; @@ -XXX,XX +XXX,XX @@ virDomainMemorytuneDefParse(virDomainDefPtr def, xmlNodePtr *nodes = NULL; virBitmapPtr vcpus = NULL; virResctrlAllocPtr alloc = NULL; + virDomainResctrlDefPtr tmp_resctrl = NULL; ssize_t i = 0; int n; int ret = -1; @@ -XXX,XX +XXX,XX @@ virDomainMemorytuneDefParse(virDomainDefPtr def, * just update the existing alloc information, which is done in above * virDomainMemorytuneDefParseMemory */ if (new_alloc) { - if (virDomainResctrlAppend(def, node, alloc, vcpus, flags) < 0) + if (VIR_ALLOC(tmp_resctrl) < 0) + goto cleanup; + + tmp_resctrl->alloc = virObjectRef(alloc); + tmp_resctrl->vcpus = vcpus; + if (virDomainResctrlAppend(def, node, tmp_resctrl, flags) < 0) goto cleanup; vcpus = NULL; alloc = NULL; + tmp_resctrl = NULL; } ret = 0; cleanup: ctxt->node = oldnode; - virObjectUnref(alloc); virBitmapFree(vcpus); + virObjectUnref(alloc); + VIR_FREE(tmp_resctrl); VIR_FREE(nodes); return ret; } -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Introduce resource monitoring group in domain configuration file to support CPU cache monitoring technology (CMT). Domain rng file changes, supporting following types of resource monitoring group regarding the allocation regin it belongs to: 1. monitoring group that working for partial working thread of current allocation: e.g. "<monitor vcpus='0'/>" creates monitoring group special for vcpu '0' while an allocation group is created for vcpus of '0' *and* '1'. 2. monitoring group for whole vcpu set of current allocation: e.g. "<monitor vcpus='0-1'/>" creates monitoring group for all vcpus belonging to current allocation. 3. monitoring group for vcpu(s) that does not have dedicated allocation group: e.g. "<monitor vcpus='3'/>" creates a monitoring group but no resource control applied to it. <cputune> <cachetune vcpus='0-1'> <cache id='0' level='3' type='code' size='7680' unit='KiB'/> <cache id='1' level='3' type='data' size='3840' unit='KiB'/> + <monitor vcpus='0-1'/> + <monitor vcpus='0'/> </cachetune> <cachetune vcpus='2'> <cache id='1' level='3' type='code' size='6' unit='MiB'/> + <monitor vcpus='2'/> </cachetune> <cachetune vcpus='3'> + <monitor vcpus='3'/> </cachetune> </cputune> Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- docs/formatdomain.html.in | 14 ++- docs/schemas/domaincommon.rng | 11 +- src/conf/domain_conf.c | 131 ++++++++++++++++++--- src/conf/domain_conf.h | 20 ++++ tests/genericxml2xmlindata/cachetune-cdp.xml | 2 + .../cachetune-colliding-monitors.xml | 36 ++++++ tests/genericxml2xmlindata/cachetune-small.xml | 1 + tests/genericxml2xmlindata/cachetune.xml | 3 + tests/genericxml2xmltest.c | 4 + 9 files changed, 204 insertions(+), 18 deletions(-) create mode 100644 tests/genericxml2xmlindata/cachetune-colliding-monitors.xml diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index XXXXXXX..XXXXXXX 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -XXX,XX +XXX,XX @@ &lt;cachetune vcpus='0-3'&gt; &lt;cache id='0' level='3' type='both' size='3' unit='MiB'/&gt; &lt;cache id='1' level='3' type='both' size='3' unit='MiB'/&gt; + &lt;monitor vcpus='0-1'/&gt; &lt;/cachetune&gt; &lt;memorytune vcpus='0-3'&gt; &lt;node id='0' bandwidth='60'/&gt; @@ -XXX,XX +XXX,XX @@ <dl> <dt><code>cache</code></dt> <dd> - This element controls the allocation of CPU cache and has the - following attributes: + This optional element controls the allocation of CPU cache and has + the following attributes: <dl> <dt><code>level</code></dt> <dd> @@ -XXX,XX +XXX,XX @@ </dd> </dl> </dd> + <dt><code>monitor</code></dt> + <dd> + The optional element <code>monitor</code> creates the cahce + monitoring group(s) for current cache allocation group. The required + attribute <code>vcpus</code> specifies to which vCPUs this + monitoring group applies. A vCPU can only be member of one + <code>cachetune</code> element allocation. And no overlap is + permitted. + </dd> </dl> </dd> diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index XXXXXXX..XXXXXXX 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -XXX,XX +XXX,XX @@ <attribute name="vcpus"> <ref name='cpuset'/> </attribute> - <oneOrMore> + <zeroOrMore> <element name="cache"> <attribute name="id"> <ref name='unsignedInt'/> @@ -XXX,XX +XXX,XX @@ </attribute> </optional> </element> - </oneOrMore> + </zeroOrMore> + <zeroOrMore> + <element name="monitor"> + <attribute name="vcpus"> + <ref name='cpuset'/> + </attribute> + </element> + </zeroOrMore> </element> </zeroOrMore> <zeroOrMore> diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainLoaderDefFree(virDomainLoaderDefPtr loader) static void +virDomainResctrlMonFree(virDomainResctrlMonitorPtr monitor) +{ + if (!monitor) + return; + + VIR_FREE(monitor->id); + virBitmapFree(monitor->vcpus); + VIR_FREE(monitor); +} + + +static void virDomainResctrlDefFree(virDomainResctrlDefPtr resctrl) { + size_t i = 0; + if (!resctrl) return; virObjectUnref(resctrl->alloc); virBitmapFree(resctrl->vcpus); + for (i = 0; i < resctrl->nmonitors; i++) + virDomainResctrlMonFree(resctrl->monitors[i]); + VIR_FREE(resctrl->monitors); VIR_FREE(resctrl); } @@ -XXX,XX +XXX,XX @@ virDomainResctrlAppend(virDomainDefPtr def, static int +virDomainResctrlParseMonitor(virDomainDefPtr def, + xmlXPathContextPtr ctxt, + xmlNodePtr node, + virDomainResctrlDefPtr resctrl) +{ + xmlNodePtr oldnode = ctxt->node; + virBitmapPtr vcpus = NULL; + char *id = NULL; + int vcpu = -1; + char *vcpus_str = NULL; + virDomainResctrlMonitorPtr tmp_domresmon = NULL; + int ret = -1; + + if (!resctrl || !resctrl->vcpus || !resctrl->alloc) + return -1; + + ctxt->node = node; + + if (VIR_ALLOC(tmp_domresmon) < 0) + goto cleanup; + + if (virDomainResctrlParseVcpus(def, node, &vcpus) < 0) + goto cleanup; + + /* empty monitoring group is not allowed */ + if (virBitmapIsAllClear(vcpus)) + goto cleanup; + + while ((vcpu = virBitmapNextSetBit(vcpus, vcpu)) >= 0) { + if (!virBitmapIsBitSet(resctrl->vcpus, vcpu)) + goto cleanup; + } + + vcpus_str = virBitmapFormat(vcpus); + if (!vcpus_str) + goto cleanup; + + if (virAsprintf(&id, "vcpus_%s", vcpus_str) < 0) + goto cleanup; + + if (VIR_STRDUP(tmp_domresmon->id, id) < 0) + goto cleanup; + + tmp_domresmon->vcpus = vcpus; + + if (VIR_APPEND_ELEMENT(resctrl->monitors, + resctrl->nmonitors, + tmp_domresmon) < 0) + goto cleanup; + + if (virResctrlAllocSetMonitor(resctrl->alloc, id) < 0) + goto cleanup; + + tmp_domresmon = NULL; + ret = 0; + cleanup: + ctxt->node = oldnode; + VIR_FREE(id); + VIR_FREE(vcpus_str); + virDomainResctrlMonFree(tmp_domresmon); + return ret; +} + + +static int virDomainCachetuneDefParse(virDomainDefPtr def, xmlXPathContextPtr ctxt, xmlNodePtr node, @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, int n; int ret = -1; + if (VIR_ALLOC(tmp_resctrl) < 0) + return -1; + ctxt->node = node; if (virDomainResctrlParseVcpus(def, node, &vcpus) < 0) @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, goto cleanup; } - if (virResctrlAllocIsEmpty(alloc)) { - ret = 0; + tmp_resctrl->vcpus = vcpus; + tmp_resctrl->alloc = virObjectRef(alloc); + + VIR_FREE(nodes); + ctxt->node = node; + + if ((n = virXPathNodeSet("./monitor", ctxt, &nodes)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot extract monitor nodes under cachetune")); goto cleanup; } - if (VIR_ALLOC(tmp_resctrl) < 0) - goto cleanup; + for (i = 0; i < n; i++) { + if (virDomainResctrlParseMonitor(def, ctxt, + nodes[i], tmp_resctrl) < 0) - tmp_resctrl->vcpus = vcpus; - tmp_resctrl->alloc = virObjectRef(alloc); + goto cleanup; + } + + if (virResctrlAllocIsEmpty(alloc)) { + VIR_WARN("cachetune: resctrl alloc is empty"); + ret = 0; + goto cleanup; + } if (virDomainResctrlAppend(def, node, tmp_resctrl, flags) < 0) goto cleanup; - alloc = NULL; - vcpus = NULL; tmp_resctrl = NULL; ret = 0; cleanup: ctxt->node = oldnode; - virObjectUnref(alloc); - VIR_FREE(tmp_resctrl); - virBitmapFree(vcpus); + virDomainResctrlDefFree(tmp_resctrl); VIR_FREE(nodes); return ret; } @@ -XXX,XX +XXX,XX @@ virDomainMemorytuneDefParse(virDomainDefPtr def, ret = 0; cleanup: ctxt->node = oldnode; - virBitmapFree(vcpus); - virObjectUnref(alloc); - VIR_FREE(tmp_resctrl); VIR_FREE(nodes); + virDomainResctrlDefFree(tmp_resctrl); return ret; } @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefFormat(virBufferPtr buf, { virBuffer childrenBuf = VIR_BUFFER_INITIALIZER; char *vcpus = NULL; + size_t i = 0; int ret = -1; virBufferSetChildIndent(&childrenBuf, buf); @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefFormat(virBufferPtr buf, if (virBufferCheckError(&childrenBuf) < 0) goto cleanup; + for (i = 0; i < resctrl->nmonitors; i++) { + vcpus = virBitmapFormat(resctrl->monitors[i]->vcpus); + if (!vcpus) + goto cleanup; + + virBufferAsprintf(&childrenBuf, "<monitor vcpus='%s'/>\n", vcpus); + VIR_FREE(vcpus); + } + if (!virBufferUse(&childrenBuf)) { ret = 0; goto cleanup; diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -XXX,XX +XXX,XX @@ struct _virDomainCputune { }; +typedef enum { + VIR_DOMAIN_RESCTRL_MONITOR_CACHE, + VIR_DOMAIN_RESCTRL_MONITOR_MEMBW, + VIR_DOMAIN_RESCTRL_MONITOR_CACHE_MEMBW, + + VIR_DOMAIN_RESCTRL_MONITOR_LAST +} virDomainResctrlMonType; + +typedef struct _virDomainResctrlMonitor virDomainResctrlMonitor; +typedef virDomainResctrlMonitor *virDomainResctrlMonitorPtr; +struct _virDomainResctrlMonitor { + int type; /* virDomainResctrlMonType*/ + char *id; + virBitmapPtr vcpus; +}; + + typedef struct _virDomainResctrlDef virDomainResctrlDef; typedef virDomainResctrlDef *virDomainResctrlDefPtr; struct _virDomainResctrlDef { virBitmapPtr vcpus; virResctrlAllocPtr alloc; + virDomainResctrlMonitorPtr *monitors; + size_t nmonitors; }; @@ -XXX,XX +XXX,XX @@ VIR_ENUM_DECL(virDomainIOMMUModel) VIR_ENUM_DECL(virDomainVsockModel) VIR_ENUM_DECL(virDomainShmemModel) VIR_ENUM_DECL(virDomainLaunchSecurity) +VIR_ENUM_DECL(virDomainResctrlMonType) /* from libvirt.h */ VIR_ENUM_DECL(virDomainState) VIR_ENUM_DECL(virDomainNostateReason) diff --git a/tests/genericxml2xmlindata/cachetune-cdp.xml b/tests/genericxml2xmlindata/cachetune-cdp.xml index XXXXXXX..XXXXXXX 100644 --- a/tests/genericxml2xmlindata/cachetune-cdp.xml +++ b/tests/genericxml2xmlindata/cachetune-cdp.xml @@ -XXX,XX +XXX,XX @@ <cachetune vcpus='0-1'> <cache id='0' level='3' type='code' size='7680' unit='KiB'/> <cache id='1' level='3' type='data' size='3840' unit='KiB'/> + <monitor vcpus='0-1'/> </cachetune> <cachetune vcpus='2'> <cache id='1' level='3' type='code' size='6' unit='MiB'/> + <monitor vcpus='2'/> </cachetune> <cachetune vcpus='3'> <cache id='1' level='3' type='data' size='6912' unit='KiB'/> diff --git a/tests/genericxml2xmlindata/cachetune-colliding-monitors.xml b/tests/genericxml2xmlindata/cachetune-colliding-monitors.xml new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/genericxml2xmlindata/cachetune-colliding-monitors.xml @@ -XXX,XX +XXX,XX @@ +<domain type='qemu'> + <name>QEMUGuest1</name> + <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>219136</memory> + <currentMemory unit='KiB'>219136</currentMemory> + <vcpu placement='static'>4</vcpu> + <cputune> + <cachetune vcpus='0-1'> + <cache id='0' level='3' type='both' size='3' unit='MiB'/> + <cache id='1' level='3' type='both' size='3' unit='MiB'/> + <monitor vcpus='0-2'/> + <monitor vcpus='0'/> + </cachetune> + <cachetune vcpus='3'> + <cache id='0' level='3' type='both' size='3' unit='MiB'/> + <monitor vcpus='3'/> + </cachetune> + </cputune> + <os> + <type arch='i686' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-i686</emulator> + <controller type='usb' index='0'/> + <controller type='ide' index='0'/> + <controller type='pci' index='0' model='pci-root'/> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <memballoon model='virtio'/> + </devices> +</domain> diff --git a/tests/genericxml2xmlindata/cachetune-small.xml b/tests/genericxml2xmlindata/cachetune-small.xml index XXXXXXX..XXXXXXX 100644 --- a/tests/genericxml2xmlindata/cachetune-small.xml +++ b/tests/genericxml2xmlindata/cachetune-small.xml @@ -XXX,XX +XXX,XX @@ <cputune> <cachetune vcpus='0-1'> <cache id='0' level='3' type='both' size='768' unit='KiB'/> + <monitor vcpus='0-1'/> </cachetune> </cputune> <os> diff --git a/tests/genericxml2xmlindata/cachetune.xml b/tests/genericxml2xmlindata/cachetune.xml index XXXXXXX..XXXXXXX 100644 --- a/tests/genericxml2xmlindata/cachetune.xml +++ b/tests/genericxml2xmlindata/cachetune.xml @@ -XXX,XX +XXX,XX @@ <cachetune vcpus='0-1'> <cache id='0' level='3' type='both' size='3' unit='MiB'/> <cache id='1' level='3' type='both' size='3' unit='MiB'/> + <monitor vcpus='0-1'/> + <monitor vcpus='0'/> </cachetune> <cachetune vcpus='3'> <cache id='0' level='3' type='both' size='3' unit='MiB'/> + <monitor vcpus='3'/> </cachetune> </cputune> <os> diff --git a/tests/genericxml2xmltest.c b/tests/genericxml2xmltest.c index XXXXXXX..XXXXXXX 100644 --- a/tests/genericxml2xmltest.c +++ b/tests/genericxml2xmltest.c @@ -XXX,XX +XXX,XX @@ mymain(void) TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST_FULL("cachetune-colliding-types", false, true, TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); + DO_TEST_FULL("cachetune-colliding-monitors", false, true, + TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST("memorytune"); DO_TEST_FULL("memorytune-colliding-allocs", false, true, TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST_FULL("memorytune-colliding-cachetune", false, true, TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); + DO_TEST_FULL("cachetune-colliding-monitors", false, true, + TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST("tseg"); -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Resource monitoring group monitors the resource consumption, cache and memory bandwidth, of particular resctrl allocation. Introduce the resctrl monitoring group. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/qemu/qemu_process.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver, { int ret = -1; size_t i = 0; + size_t j = 0; virCapsPtr caps = NULL; qemuDomainObjPrivatePtr priv = vm->privateData; @@ -XXX,XX +XXX,XX @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver, vm->def->resctrls[i]->alloc, priv->machineName) < 0) goto cleanup; + + /* Create resctrl monitoring groups associated with allocation */ + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonitorPtr monitor = NULL; + monitor = vm->def->resctrls[i]->monitors[j]; + + if (virResctrlAllocCreateMonitor(caps->host.resctrl, + vm->def->resctrls[i]->alloc, + priv->machineName, + monitor->id) < 0) + + goto cleanup; + + } } ret = 0; @@ -XXX,XX +XXX,XX @@ qemuProcessSetupVcpu(virDomainObjPtr vm, { pid_t vcpupid = qemuDomainGetVcpuPid(vm, vcpuid); virDomainVcpuDefPtr vcpu = virDomainDefGetVcpu(vm->def, vcpuid); + virDomainResctrlMonitorPtr mon = NULL; size_t i = 0; + size_t j = 0; if (qemuProcessSetupPid(vm, vcpupid, VIR_CGROUP_THREAD_VCPU, vcpuid, vcpu->cpumask, @@ -XXX,XX +XXX,XX @@ qemuProcessSetupVcpu(virDomainObjPtr vm, if (virBitmapIsBitSet(ct->vcpus, vcpuid)) { if (virResctrlAllocAddPID(ct->alloc, vcpupid) < 0) return -1; - break; + } + + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + mon = vm->def->resctrls[i]->monitors[j]; + if (virBitmapIsBitSet(mon->vcpus, vcpuid)) { + if (virResctrlAllocAddMonitorPID(ct->alloc, + mon->id, vcpupid) < 0) + return -1; + } } } @@ -XXX,XX +XXX,XX @@ qemuProcessReconnect(void *opaque) int reason; virQEMUDriverConfigPtr cfg; size_t i; + size_t j; unsigned int stopFlags = 0; bool jobStarted = false; virCapsPtr caps = NULL; bool retry = true; + virDomainResctrlDefPtr resctrl = NULL; VIR_FREE(data); @@ -XXX,XX +XXX,XX @@ qemuProcessReconnect(void *opaque) goto error; for (i = 0; i < obj->def->nresctrls; i++) { - if (virResctrlAllocDeterminePath(obj->def->resctrls[i]->alloc, + resctrl = obj->def->resctrls[i]; + + if (virResctrlAllocDeterminePath(resctrl->alloc, priv->machineName) < 0) goto error; + + for (j = 0; j < resctrl->nmonitors; j++) { + if (virResctrlAllocDetermineMonitorPath(resctrl->alloc, + resctrl->monitors[j]->id, + priv->machineName) < 0) + goto error; + } } /* update domain state XML with possibly updated state in virDomainObj */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Intel x86 RDT CMT is the technology to tell the last level cache occupancy information. Adding the interface in qemu to report this information for resource monitor group through command 'virsh domstats --cpu-total'. Below is a typical output: # virsh domstats 1 --cpu-total Domain: 'ubuntu16.04-base' ... cpu.cache.monitor.count=2 cpu.cache.0.name=vcpus_1 cpu.cache.0.vcpus=1 cpu.cache.0.bank.count=2 cpu.cache.0.bank.0.id=0 cpu.cache.0.bank.0.bytes=4505600 cpu.cache.0.bank.1.id=1 cpu.cache.0.bank.1.bytes=5586944 cpu.cache.1.name=vcpus_4-6 cpu.cache.1.vcpus=4,5,6 cpu.cache.1.bank.count=2 cpu.cache.1.bank.0.id=0 cpu.cache.1.bank.0.bytes=17571840 cpu.cache.1.bank.1.id=1 cpu.cache.1.bank.1.bytes=29106176 Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt-domain.c | 9 ++ src/qemu/qemu_driver.c | 265 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 253 insertions(+), 21 deletions(-) diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -XXX,XX +XXX,XX @@ virConnectGetDomainCapabilities(virConnectPtr conn, * "cpu.user" - user cpu time spent in nanoseconds as unsigned long long. * "cpu.system" - system cpu time spent in nanoseconds as unsigned long * long. + * "cpu.cache.monitor.count" - tocal cache monitoring groups + * "cpu.cache.M.name" - name for cache monitoring group 'M' + * "cpu.cache.M.vcpus" - vcpus for cache monitoring group 'M' + * "cpu.cache.M.bank.count" - total bank number for cache monitoring + * group 'M' + * "cpu.cache.M.bank.N.id" - OS assigned cache bank id for cache 'N' in + * cache monitoring group 'M' + * "cpu.cache.M.bank.N.bytes" - cache occupancy of cache bank 'N' in + * cache monitoring group 'M' * * VIR_DOMAIN_STATS_BALLOON: * Return memory balloon device information. diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ #include "virnuma.h" #include "dirname.h" #include "netdev_bandwidth_conf.h" +#include "c-ctype.h" #define VIR_FROM_THIS VIR_FROM_QEMU @@ -XXX,XX +XXX,XX @@ typedef enum { #define HAVE_JOB(flags) ((flags) & QEMU_DOMAIN_STATS_HAVE_JOB) +/* + * qemuDomainVcpuFormatHelper + * For vcpu string, both '1-3' and '1,3' are valid format and + * representing different vcpu set, but it is not easy to + * differentiate them at first galance, to avoid this case + * substituting all '-' with ',', e.g. substitute string '1-3' + * with '1,2,3'. + */ +static int +qemuDomainVcpuFormatHelper(char **vcpus) +{ + const char *cur = NULL; + size_t i = 0; + char *tmp = NULL; + int start, last; + virBuffer buf = VIR_BUFFER_INITIALIZER; + bool firstnum = 1; + + if (!*vcpus) + goto error; + + cur = *vcpus; + + virSkipSpaces(&cur); + + if (*cur == '\0') + goto error; + + while (*cur != 0) { + if (!c_isdigit(*cur)) + goto error; + + if (virStrToLong_i(cur, &tmp, 10, &start) < 0) + goto error; + if (start < 0) + goto error; + + cur = tmp; + + virSkipSpaces(&cur); + + if (*cur == ',' || *cur == 0) { + if (!firstnum) + virBufferAddChar(&buf, ','); + virBufferAsprintf(&buf, "%d", start); + firstnum = 0; + } else if (*cur == '-') { + cur++; + virSkipSpaces(&cur); + + if (virStrToLong_i(cur, &tmp, 10, &last) < 0) + + goto error; + if (last < start) + goto error; + cur = tmp; + + for (i = start; i <= last; i++) { + if (!firstnum) + + virBufferAddChar(&buf, ','); + virBufferAsprintf(&buf, "%ld", i); + firstnum = 0; + } + + virSkipSpaces(&cur); + } + + if (*cur == ',') { + cur++; + virSkipSpaces(&cur); + } else if (*cur == 0) { + break; + } else { + goto error; + } + } + VIR_FREE(*vcpus); + *vcpus = virBufferContentAndReset(&buf); + return 0; + error: + virBufferFreeAndReset(&buf); + return -1; +} + +static int +qemuDomainGetStatsCpuResource(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, + virDomainObjPtr dom, + virDomainStatsRecordPtr record, + int *maxparams, + unsigned int privflags ATTRIBUTE_UNUSED) +{ + size_t i = 0; + size_t j = 0; + char param_name[VIR_TYPED_PARAM_FIELD_LENGTH]; + virDomainResctrlDefPtr resctrl = NULL; + virDomainResctrlMonitorPtr monitor = NULL; + unsigned int nvals = 0; + unsigned int *ids = NULL; + unsigned int *vals = NULL; + unsigned int nmonitor = NULL; + char *vcpustr = NULL; + int ret = -1; + + for (i = 0; i < dom->def->nresctrls; i++) { + resctrl = dom->def->resctrls[i]; + + for (j = 0; j < resctrl->nmonitors; j++) { + monitor = resctrl->monitors[j]; + if (monitor->vcpus) + nmonitor++; + } + } + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.cache.monitor.count"); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + nmonitor) < 0) + goto cleanup; + + for (i = 0; i < dom->def->nresctrls; i++) { + resctrl = dom->def->resctrls[i]; + + for (j = 0; j < resctrl->nmonitors; j++) { + size_t l = 0; + + monitor = resctrl->monitors[j]; + + if (!(vcpustr = virBitmapFormat(monitor->vcpus))) + goto cleanup; + + if (qemuDomainVcpuFormatHelper(&vcpustr) < 0) + goto cleanup; + + switch ((virDomainResctrlMonType) monitor->type) { + case VIR_DOMAIN_RESCTRL_MONITOR_CACHE: + case VIR_DOMAIN_RESCTRL_MONITOR_CACHE_MEMBW: + if (!monitor->vcpus) + continue; + + if (virResctrlAllocGetCacheOccupancy(resctrl->alloc, + monitor->id, &nvals, + &ids, &vals) < 0) + goto cleanup; + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.cache.%ld.name", i); + if (virTypedParamsAddString(&record->params, + &record->nparams, + maxparams, + param_name, + monitor->id) < 0) + goto cleanup; + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.cache.%ld.vcpus", i); + + if (virTypedParamsAddString(&record->params, + &record->nparams, + maxparams, + param_name, + vcpustr) < 0) + goto cleanup; + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.cache.%ld.bank.count", i); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + nvals) < 0) + goto cleanup; + + for (l = 0; l < nvals; l++) { + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.cache.%ld.bank.%ld.id", i, l); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + ids[l]) < 0) + goto cleanup; + + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.cache.%ld.bank.%ld.bytes", i, l); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + vals[l]) < 0) + goto cleanup; + } + break; + + case VIR_DOMAIN_RESCTRL_MONITOR_MEMBW: + case VIR_DOMAIN_RESCTRL_MONITOR_LAST: + default: + break; + } + + VIR_FREE(ids); + VIR_FREE(vals); + VIR_FREE(vcpustr); + nvals = 0; + } + } + + ret = 0; + cleanup: + VIR_FREE(ids); + VIR_FREE(vals); + VIR_FREE(vcpustr); + return ret; +} + static int qemuDomainGetStatsCpu(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, virDomainObjPtr dom, @@ -XXX,XX +XXX,XX @@ qemuDomainGetStatsCpu(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, unsigned long long sys_time = 0; int err = 0; - if (!priv->cgroup) - return 0; + if (priv->cgroup) { + err = virCgroupGetCpuacctUsage(priv->cgroup, &cpu_time); + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.time", + cpu_time) < 0) + return -1; - err = virCgroupGetCpuacctUsage(priv->cgroup, &cpu_time); - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.time", - cpu_time) < 0) - return -1; + err = virCgroupGetCpuacctStat(priv->cgroup, &user_time, &sys_time); + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.user", + user_time) < 0) + return -1; + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.system", + sys_time) < 0) + return -1; + } - err = virCgroupGetCpuacctStat(priv->cgroup, &user_time, &sys_time); - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.user", - user_time) < 0) - return -1; - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.system", - sys_time) < 0) + if (qemuDomainGetStatsCpuResource(driver, dom, + record, maxparams, privflags) < 0) return -1; return 0; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list This series of patches and the series already been merged introduce the x86 Cache Monitoring Technology (CMT) to libvirt by interacting with kernel resource control (resctrl) interface. CMT is one of the Intel(R) x86 CPU feature which belongs to the Resource Director Technology (RDT). CMT reports the occupancy of the last level cache, which is shared by all CPU cores. In the v1 series, an original and complete feature for CMT was introduced The v2 and v3 patches address the feature for the host capability of CMT. v4 is addressing the feature for monitoring VM vcpu thread set cache occupancy and reporting it through a virsh command. We have serval discussion about the enabling of CMT, please refer to following links for the RFCs. RFCv3 https://www.redhat.com/archives/libvir-list/2018-August/msg01213.html RFCv2 https://www.redhat.com/archives/libvir-list/2018-July/msg00409.html https://www.redhat.com/archives/libvir-list/2018-July/msg01241.html RFCv1 https://www.redhat.com/archives/libvir-list/2018-June/msg00674.html And the merged commits are list as below, for host capability of CMT. 6af8417415508c31f8ce71234b573b4999f35980 8f6887998bf63594ae26e3db18d4d5896c5f2cb4 58fcee6f3a2b7e89c21c1fb4ec21429c31a0c5b8 12093f1feaf8f5023dcd9d65dff111022842183d a5d293c18831dcf69ec6195798387fbb70c9f461 1. About reason why CMT is necessary in libvirt? The perf events of 'CMT, MBML, MBMT' have been phased out since Linux kernel commit c39a0e2c8850f08249383f2425dbd8dbe4baad69, in libvirt the perf based cmt,mbm will not work with the latest linux kernel. These patches add CMT feature to libvirt through kernel resctrlfs interface. 2 Create cache monitoring group (cache monitor). The main interface for creating monitoring group is through XML file. The proposed configuration is like: <cputune> <cachetune vcpus='1'> <cache id='0' level='3' type='code' size='7680' unit='KiB'/> <cache id='1' level='3' type='data' size='3840' unit='KiB'/> + <monitor level='3' vcpus='1'/> </cachetune> <cachetune vcpus='4-7'> + <monitor level='3' vcpus='4-6'/> </cachetune> </cputune> In above XML, created 2 cache resctrl allocation groups and 2 resctrl monitoring groups. The changes of cache monitor will be effective in next booting of VM. 2 Show CMT result through command 'domstats' Adding the interface in qemu to report this information for resource monitor group through command 'virsh domstats --cpu-total'. Below is a typical output: # virsh domstats 1 --cpu-total Domain: 'ubuntu16.04-base' ... cpu.cache.monitor.count=2 cpu.cache.monitor.0.name=vcpus_1 cpu.cache.monitor.0.vcpus=1 cpu.cache.monitor.0.bank.count=2 cpu.cache.monitor.0.bank.0.id=0 cpu.cache.monitor.0.bank.0.bytes=4505600 cpu.cache.monitor.0.bank.1.id=1 cpu.cache.monitor.0.bank.1.bytes=5586944 cpu.cache.monitor.1.name=vcpus_4-6 cpu.cache.monitor.1.vcpus=4,5,6 cpu.cache.monitor.1.bank.count=2 cpu.cache.monitor.1.bank.0.id=0 cpu.cache.monitor.1.bank.0.bytes=17571840 cpu.cache.monitor.1.bank.1.id=1 cpu.cache.monitor.1.bank.1.bytes=29106176 Changes in v7: - Add several lines removed by mistake. Changes in v6: - Addressing John's review comments for v5. - Removed and cleaned the concepts of 'default allocation' and 'default monitor'. - qemu: virsh domstats --cpu-total output for CMT, add identifier 'monitor' for each itm. Changes in v5: - qemu: Setting up vcpu and adding pids to resctrl monitor groups during re-connection. - Add the document for domain configuration related to resctrl monitor. Changes in v4: v4 is addressing the feature for monitoring VM vcpu thread set cache occupancy and reporting it through a virsh command. - Introduced resctrl default allocation - Introduced resctrl monitor and default monitor Changes in v3: - Addressed John Ferlan's review. - Typo fixed. - Removed VIR_ENUM_DECL(virMonitor); Changes in v2: - Introduced MBM capability. - Capability layout changed * Moved <monitor> from cahe <bank> to <cache> * Renamed <Threshold> to <reuseThreshold> - Document for 'reuseThreshold' changed. - Introduced API virResctrlInfoGetMonitorPrefix - Added more tests, covering standalone CMT, fake new feature. - Creating CMT resource control group will be subsequent job. Wang Huaqiang (18): docs,util: Refactor schemas and virresctrl to support optional cache util: Introduce resctrl monitor for CMT util: Refactor code for determining allocation path util: Add interface to determine monitor path util: Refactor code for adding PID to the resource group util: Add interface for adding PID to the monitor util: Refactor code for creating resctrl group util: Add interface for creating monitor group util: Add more interfaces for resctrl monitor conf: Remove virDomainResctrlAppend and introduce virDomainResctrlNew conf: move virResctrlAllocIsEmpty to a place a litter lower conf: Introduce cache monitor element in cachetune qemu: enable resctrl monitor in qemu util: Add function for checking if monitor is running conf: Add 'id' to virDomainResctrlDef qemu: refactor qemuDomainGetStatsCpu qemu: Report cache occupancy (CMT) with domstats qemu: Setting up vcpu and adding pids to resctrl monitor groups during reconnection docs/formatdomain.html.in | 30 +- docs/schemas/domaincommon.rng | 14 +- src/conf/domain_conf.c | 310 ++++++++++-- src/conf/domain_conf.h | 12 + src/libvirt-domain.c | 9 + src/libvirt_private.syms | 11 + src/qemu/qemu_driver.c | 270 +++++++++- src/qemu/qemu_process.c | 69 ++- src/util/virresctrl.c | 560 +++++++++++++++++++-- src/util/virresctrl.h | 49 ++ tests/genericxml2xmlindata/cachetune-cdp.xml | 3 + .../cachetune-colliding-monitor.xml | 30 ++ tests/genericxml2xmlindata/cachetune-small.xml | 7 + tests/genericxml2xmltest.c | 2 + 14 files changed, 1283 insertions(+), 93 deletions(-) create mode 100644 tests/genericxml2xmlindata/cachetune-colliding-monitor.xml -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Refactor schemas and virresctrl to support optional <cache> element in <cachetune>. Later, the monitor entry will be introduced and to be placed under <cachetune>. Either cache entry or monitor entry is an optional element of <cachetune>. An cachetune has no <cache> element is taking the default resource allocating policy defined in '/sys/fs/resctrl/schemata'. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- docs/formatdomain.html.in | 4 ++-- docs/schemas/domaincommon.rng | 4 ++-- src/util/virresctrl.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index XXXXXXX..XXXXXXX 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -XXX,XX +XXX,XX @@ <dl> <dt><code>cache</code></dt> <dd> - This element controls the allocation of CPU cache and has the - following attributes: + This optional element controls the allocation of CPU cache and has + the following attributes: <dl> <dt><code>level</code></dt> <dd> diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index XXXXXXX..XXXXXXX 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -XXX,XX +XXX,XX @@ <attribute name="vcpus"> <ref name='cpuset'/> </attribute> - <oneOrMore> + <zeroOrMore> <element name="cache"> <attribute name="id"> <ref name='unsignedInt'/> @@ -XXX,XX +XXX,XX @@ </attribute> </optional> </element> - </oneOrMore> + </zeroOrMore> </element> </zeroOrMore> <zeroOrMore> diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlInfoMonFree(virResctrlInfoMonPtr mon) * in case there is no allocation for that particular cache allocation (level, * cache, ...) or memory allocation for particular node). * + * Allocation corresponding to root directory, /sys/fs/sysctrl/, defines the + * default resource allocating policy, which is created immediately after + * mounting, and owns all the tasks and cpus in the system. Cache or memory + * bandwidth resource will be shared for tasks in this allocation. + * * =====Cache allocation technology (CAT)===== * * Since one allocation can be made for caches on different levels, the first @@ -XXX,XX +XXX,XX @@ virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, return -1; } + /* If the allocation is empty, then the path will be SYSFS_RESCTRL_PATH */ + if (virResctrlAllocIsEmpty(alloc)) { + if (!alloc->path && + VIR_STRDUP(alloc->path, SYSFS_RESCTRL_PATH) < 0) + return -1; + + return 0; + } + if (!alloc->path && virAsprintf(&alloc->path, "%s/%s-%s", SYSFS_RESCTRL_PATH, machinename, alloc->id) < 0) @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virResctrlAllocDeterminePath(alloc, machinename) < 0) return -1; + /* If the allocation is empty, then do not create directory in underlying + * resctrl file system */ + if (virResctrlAllocIsEmpty(alloc)) + return 0; + if (virFileExists(alloc->path)) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Path '%s' for resctrl allocation exists"), @@ -XXX,XX +XXX,XX @@ virResctrlAllocAddPID(virResctrlAllocPtr alloc, char *pidstr = NULL; int ret = 0; + /* If the allocation is empty, then it is impossible to add a PID to + * allocation due to lacking of its 'tasks' file. Just return */ + if (virResctrlAllocIsEmpty(alloc)) + return 0; + if (!alloc->path) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Cannot add pid to non-existing resctrl allocation")); @@ -XXX,XX +XXX,XX @@ virResctrlAllocRemove(virResctrlAllocPtr alloc) { int ret = 0; + /* No directory have ever been created. Just return */ + if (virResctrlAllocIsEmpty(alloc)) + return 0; + if (!alloc->path) return 0; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Cache Monitoring Technology (aka CMT) provides the capability to report cache utilization information of system task. This patch introduces the concept of resctrl monitor through data structure virResctrlMonitor. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt_private.syms | 1 + src/util/virresctrl.c | 79 ++++++++++++++++++++++++++++++++++++++++++++---- src/util/virresctrl.h | 9 ++++++ 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache; virResctrlInfoGetMonitorPrefix; virResctrlInfoMonFree; virResctrlInfoNew; +virResctrlMonitorNew; # util/virrotatingfile.h diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ VIR_LOG_INIT("util.virresctrl") /* Resctrl is short for Resource Control. It might be implemented for various - * resources. Currently this supports cache allocation technology (aka CAT) and - * memory bandwidth allocation (aka MBA). More resources technologies may be - * added in the future. + * resources. Currently this supports cache allocation technology (aka CAT), + * memory bandwidth allocation (aka MBA) and cache monitoring technology (aka + * CMT). More resources technologies may be added in the future. */ @@ -XXX,XX +XXX,XX @@ typedef virResctrlAllocMemBW *virResctrlAllocMemBWPtr; /* Class definitions and initializations */ static virClassPtr virResctrlInfoClass; static virClassPtr virResctrlAllocClass; +static virClassPtr virResctrlMonitorClass; /* virResctrlInfo */ @@ -XXX,XX +XXX,XX @@ virResctrlInfoMonFree(virResctrlInfoMonPtr mon) } -/* virResctrlAlloc */ +/* virResctrlAlloc and virResctrlMonitor*/ /* - * virResctrlAlloc represents one allocation (in XML under cputune/cachetune and - * consequently a directory under /sys/fs/resctrl). Since it can have multiple + * virResctrlAlloc and virResctrlMonitor are representing a resource control + * group (in XML under cputune/cachetune and consequently a directory under + * /sys/fs/resctrl). virResctrlAlloc is the data structure for resource + * allocation, while the virResctrlMonitor represents the resource monitoring + * part. + * + * virResctrlAlloc represents one allocation. Since it can have multiple * parts of multiple caches allocated it is represented as bunch of nested * sparse arrays (by sparse I mean array of pointers so that each might be NULL * in case there is no allocation for that particular cache allocation (level, @@ -XXX,XX +XXX,XX @@ virResctrlInfoMonFree(virResctrlInfoMonPtr mon) * a sparse array to represent whether a memory bandwidth allocation happens * on corresponding node. The available memory controller number is collected * in 'virResctrlInfo'. + * + * =====Cache monitoring technology (CMT)===== + * + * Cache monitoring technology is used to perceive how many cache the process + * is using actually. virResctrlMonitor represents the resource control + * monitoring group, it is supported to monitor resource utilization + * information on granularity of vcpu. + * + * From hardware perspective, cache monitoring technology (CMT), memory + * bandwidth technology (MBM), as well as the CAT and MBA, are all orthogonal + * features. The monitor will be created under the scope of default resctl + * group if no specific CAT or MBA entries are provided for the guest." */ struct _virResctrlAllocPerType { /* There could be bool saying whether this is set or not, but since everything @@ -XXX,XX +XXX,XX @@ struct _virResctrlAlloc { char *path; }; +/* + * virResctrlMonitor is the data structure for resctrl monitor. Resctrl + * monitor represents a resctrl monitoring group, which can be used to + * monitor the resource utilization information for either cache or + * memory bandwidth. + */ +struct _virResctrlMonitor { + virObject parent; + + /* In resctrl, each monitor is associated with one specific allocation, + * either the allocation under / sys / fs / resctrl or allocation of the + * root directory itself. This pointer points to the allocation + * this monitor associated with. */ + virResctrlAllocPtr alloc; + /* The monitor identifier. For a monitor has the same @path name as its + * @alloc, the @id will be set to the same value as it is in @alloc->id. + */ + char *id; + /* libvirt-generated path in /sys/fs/resctrl for this particular + * monitor */ + char *path; +}; + static void virResctrlAllocDispose(void *obj) @@ -XXX,XX +XXX,XX @@ virResctrlAllocDispose(void *obj) } +static void +virResctrlMonitorDispose(void *obj) +{ + virResctrlMonitorPtr monitor = obj; + + virObjectUnref(monitor->alloc); + VIR_FREE(monitor->id); + VIR_FREE(monitor->path); +} + + /* Global initialization for classes */ static int virResctrlOnceInit(void) @@ -XXX,XX +XXX,XX @@ virResctrlOnceInit(void) if (!VIR_CLASS_NEW(virResctrlAlloc, virClassForObject())) return -1; + if (!VIR_CLASS_NEW(virResctrlMonitor, virClassForObject())) + return -1; + return 0; } @@ -XXX,XX +XXX,XX @@ virResctrlAllocRemove(virResctrlAllocPtr alloc) return ret; } + + +/* virResctrlMonitor-related definitions */ + +virResctrlMonitorPtr +virResctrlMonitorNew(void) +{ + if (virResctrlInitialize() < 0) + return NULL; + + return virObjectNew(virResctrlMonitorClass); +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ int virResctrlInfoGetMonitorPrefix(virResctrlInfoPtr resctrl, const char *prefix, virResctrlInfoMonPtr *monitor); + +/* Monitor-related things */ + +typedef struct _virResctrlMonitor virResctrlMonitor; +typedef virResctrlMonitor *virResctrlMonitorPtr; + + +virResctrlMonitorPtr +virResctrlMonitorNew(void); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list The code for determining resctrl allocation path could be reused for monitor. Refactor it for reuse. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/util/virresctrl.c | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlAllocAssign(virResctrlInfoPtr resctrl, } +static char * +virResctrlDeterminePath(const char *parentpath, + const char *prefix, + const char *id) +{ + char *path = NULL; + + if (!id) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Resctrl ID must be set before determining resctrl " + "parentpath='%s'"), parentpath); + return NULL; + } + + if (virAsprintf(&path, "%s/%s-%s", parentpath, prefix, id) < 0) + return NULL; + + return path; +} + + int virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, const char *machinename) { - if (!alloc->id) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Resctrl Allocation ID must be set before creation")); + if (alloc->path) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Resctrl allocation path is already set to '%s'"), + alloc->path); return -1; } /* If the allocation is empty, then the path will be SYSFS_RESCTRL_PATH */ if (virResctrlAllocIsEmpty(alloc)) { - if (!alloc->path && - VIR_STRDUP(alloc->path, SYSFS_RESCTRL_PATH) < 0) + if (VIR_STRDUP(alloc->path, SYSFS_RESCTRL_PATH) < 0) return -1; return 0; } - if (!alloc->path && - virAsprintf(&alloc->path, "%s/%s-%s", - SYSFS_RESCTRL_PATH, machinename, alloc->id) < 0) + alloc->path = virResctrlDeterminePath(SYSFS_RESCTRL_PATH, + machinename, alloc->id); + + if (!alloc->path) return -1; return 0; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Add interface for resctrl monitor to determine the path. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt_private.syms | 1 + src/util/virresctrl.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ src/util/virresctrl.h | 5 ++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache; virResctrlInfoGetMonitorPrefix; virResctrlInfoMonFree; virResctrlInfoNew; +virResctrlMonitorDeterminePath; virResctrlMonitorNew; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlMonitorNew(void) return virObjectNew(virResctrlMonitorClass); } + + +/* + * virResctrlMonitorDeterminePath + * + * @monitor: Pointer to a resctrl monitor + * @machinename: Name string of the VM + * + * Determines the directory path that the underlying resctrl group will be + * created with. + * + * A monitor represents a directory under resource control file system, + * its directory path could be the same path as @monitor->alloc, could be a + * path of directory under 'mon_groups' of @monitor->alloc, or a path of + * directory under '/sys/fs/resctrl/mon_groups' if @monitor->alloc is NULL. + * + * Returns 0 on success, -1 on error. + */ +int +virResctrlMonitorDeterminePath(virResctrlMonitorPtr monitor, + const char *machinename) +{ + VIR_AUTOFREE(char *) parentpath = NULL; + + if (!monitor) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Invalid resctrl monitor")); + return -1; + } + + if (monitor->path) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Resctrl monitor path is already set to '%s'"), + monitor->path); + return -1; + } + + if (monitor->id && monitor->alloc && monitor->alloc->id) { + if (STREQ(monitor->id, monitor->alloc->id)) { + if (VIR_STRDUP(monitor->path, monitor->alloc->path) < 0) + return -1; + return 0; + } + } + + if (virAsprintf(&parentpath, "%s/mon_groups", monitor->alloc->path) < 0) + return -1; + + monitor->path = virResctrlDeterminePath(parentpath, machinename, + monitor->id); + if (!monitor->path) + return -1; + + return 0; +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetMonitorPrefix(virResctrlInfoPtr resctrl, typedef struct _virResctrlMonitor virResctrlMonitor; typedef virResctrlMonitor *virResctrlMonitorPtr; - virResctrlMonitorPtr virResctrlMonitorNew(void); + +int +virResctrlMonitorDeterminePath(virResctrlMonitorPtr monitor, + const char *machinename); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list The code of adding PID to the allocation could be reused, refactor it for later reuse. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/util/virresctrl.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, } -int -virResctrlAllocAddPID(virResctrlAllocPtr alloc, - pid_t pid) +static int +virResctrlAddPID(const char *path, + pid_t pid) { char *tasks = NULL; char *pidstr = NULL; int ret = 0; - /* If the allocation is empty, then it is impossible to add a PID to - * allocation due to lacking of its 'tasks' file. Just return */ - if (virResctrlAllocIsEmpty(alloc)) - return 0; - - if (!alloc->path) { + if (!path) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Cannot add pid to non-existing resctrl allocation")); + _("Cannot add pid to non-existing resctrl group")); return -1; } - if (virAsprintf(&tasks, "%s/tasks", alloc->path) < 0) + if (virAsprintf(&tasks, "%s/tasks", path) < 0) return -1; if (virAsprintf(&pidstr, "%lld", (long long int) pid) < 0) @@ -XXX,XX +XXX,XX @@ virResctrlAllocAddPID(virResctrlAllocPtr alloc, int +virResctrlAllocAddPID(virResctrlAllocPtr alloc, + pid_t pid) +{ + /* If allocation is empty, then no resctrl directory and the 'tasks' file + * exists, just return */ + if (virResctrlAllocIsEmpty(alloc)) + return 0; + + return virResctrlAddPID(alloc->path, pid); +} + + +int virResctrlAllocRemove(virResctrlAllocPtr alloc) { int ret = 0; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Add interface for adding task PID to the monitor. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt_private.syms | 1 + src/util/virresctrl.c | 8 ++++++++ src/util/virresctrl.h | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetCache; virResctrlInfoGetMonitorPrefix; virResctrlInfoMonFree; virResctrlInfoNew; +virResctrlMonitorAddPID; virResctrlMonitorDeterminePath; virResctrlMonitorNew; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlMonitorDeterminePath(virResctrlMonitorPtr monitor, return 0; } + + +int +virResctrlMonitorAddPID(virResctrlMonitorPtr monitor, + pid_t pid) +{ + return virResctrlAddPID(monitor->path, pid); +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ virResctrlMonitorNew(void); int virResctrlMonitorDeterminePath(virResctrlMonitorPtr monitor, const char *machinename); + +int +virResctrlMonitorAddPID(virResctrlMonitorPtr monitor, + pid_t pid); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list The code for creating resctrl allocation group could be reused for monitoring group, refactor it for reuse in the later patch. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/util/virresctrl.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, } +/* This function creates a resctrl directory in resource control file system, + * and the directory path is specified by @path. */ +static int +virResctrlCreateGroupPath(const char *path) +{ + /* Directory exists, return */ + if (virFileExists(path)) + return 0; + + if (virFileMakePath(path) < 0) { + virReportSystemError(errno, + _("Cannot create resctrl directory '%s'"), + path); + return -1; + } + + return 0; +} + + /* This checks if the directory for the alloc exists. If not it tries to create * it and apply appropriate alloc settings. */ int @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virResctrlAllocIsEmpty(alloc)) return 0; - if (virFileExists(alloc->path)) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Path '%s' for resctrl allocation exists"), - alloc->path); - goto cleanup; - } - lockfd = virResctrlLockWrite(); if (lockfd < 0) goto cleanup; @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virResctrlAllocAssign(resctrl, alloc) < 0) goto cleanup; + if (virResctrlCreateGroupPath(alloc->path) < 0) + goto cleanup; + alloc_str = virResctrlAllocFormat(alloc); if (!alloc_str) goto cleanup; @@ -XXX,XX +XXX,XX @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virAsprintf(&schemata_path, "%s/schemata", alloc->path) < 0) goto cleanup; - if (virFileMakePath(alloc->path) < 0) { - virReportSystemError(errno, - _("Cannot create resctrl directory '%s'"), - alloc->path); - goto cleanup; - } - VIR_DEBUG("Writing resctrl schemata '%s' into '%s'", alloc_str, schemata_path); if (virFileWriteStr(schemata_path, alloc_str, 0) < 0) { rmdir(alloc->path); -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Add interface for creating the resource monitoring group according to '@virResctrlMonitor->path'. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt_private.syms | 1 + src/util/virresctrl.c | 24 ++++++++++++++++++++++++ src/util/virresctrl.h | 4 ++++ 3 files changed, 29 insertions(+) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetMonitorPrefix; virResctrlInfoMonFree; virResctrlInfoNew; virResctrlMonitorAddPID; +virResctrlMonitorCreate; virResctrlMonitorDeterminePath; virResctrlMonitorNew; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlMonitorAddPID(virResctrlMonitorPtr monitor, { return virResctrlAddPID(monitor->path, pid); } + + +int +virResctrlMonitorCreate(virResctrlMonitorPtr monitor, + const char *machinename) +{ + int lockfd = -1; + int ret = -1; + + if (!monitor) + return 0; + + if (virResctrlMonitorDeterminePath(monitor, machinename) < 0) + return -1; + + lockfd = virResctrlLockWrite(); + if (lockfd < 0) + return -1; + + ret = virResctrlCreateGroupPath(monitor->path); + + virResctrlUnlock(lockfd); + return ret; +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ virResctrlMonitorDeterminePath(virResctrlMonitorPtr monitor, int virResctrlMonitorAddPID(virResctrlMonitorPtr monitor, pid_t pid); + +int +virResctrlMonitorCreate(virResctrlMonitorPtr monitor, + const char *machinename); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Add interfaces monitor group to support operations such as add PID, set ID, remove group ... etc. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt_private.syms | 5 +++++ src/util/virresctrl.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virresctrl.h | 14 ++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virResctrlInfoNew; virResctrlMonitorAddPID; virResctrlMonitorCreate; virResctrlMonitorDeterminePath; +virResctrlMonitorGetID; virResctrlMonitorNew; +virResctrlMonitorRemove; +virResctrlMonitorSetAlloc; +virResctrlMonitorSetID; + # util/virrotatingfile.h diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlMonitorCreate(virResctrlMonitorPtr monitor, virResctrlUnlock(lockfd); return ret; } + + +int +virResctrlMonitorSetID(virResctrlMonitorPtr monitor, + const char *id) +{ + if (!id) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Resctrl monitor 'id' cannot be NULL")); + return -1; + } + + return VIR_STRDUP(monitor->id, id); +} + + +const char * +virResctrlMonitorGetID(virResctrlMonitorPtr monitor) +{ + return monitor->id; +} + + +void +virResctrlMonitorSetAlloc(virResctrlMonitorPtr monitor, + virResctrlAllocPtr alloc) +{ + monitor->alloc = virObjectRef(alloc); +} + + +int +virResctrlMonitorRemove(virResctrlMonitorPtr monitor) +{ + int ret = 0; + + if (!monitor->path) + return 0; + + VIR_DEBUG("Removing resctrl monitor%s", monitor->path); + if (rmdir(monitor->path) != 0 && errno != ENOENT) { + ret = -errno; + VIR_ERROR(_("Unable to remove %s (%d)"), monitor->path, errno); + } + + return ret; +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ virResctrlMonitorAddPID(virResctrlMonitorPtr monitor, int virResctrlMonitorCreate(virResctrlMonitorPtr monitor, const char *machinename); + +int +virResctrlMonitorSetID(virResctrlMonitorPtr monitor, + const char *id); + +const char * +virResctrlMonitorGetID(virResctrlMonitorPtr monitor); + +void +virResctrlMonitorSetAlloc(virResctrlMonitorPtr monitor, + virResctrlAllocPtr alloc); + +int +virResctrlMonitorRemove(virResctrlMonitorPtr monitor); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Introduced virDomainResctrlNew to do the most part of virDomainResctrlAppend and move the operation of appending resctrl to @def->resctrls out of function. Rather than rely on virDomainResctrlAppend to perform the allocation, move the onus to the caller and make use of virBitmapNewCopy for @vcpus and virObjectRef for @alloc, thus removing the need to set each to NULL after the call. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/conf/domain_conf.c | 60 +++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParseCache(xmlXPathContextPtr ctxt, } -static int -virDomainResctrlAppend(virDomainDefPtr def, - xmlNodePtr node, - virResctrlAllocPtr alloc, - virBitmapPtr vcpus, - unsigned int flags) +static virDomainResctrlDefPtr +virDomainResctrlNew(xmlNodePtr node, + virResctrlAllocPtr *alloc, + virBitmapPtr *vcpus, + unsigned int flags) { char *vcpus_str = NULL; char *alloc_id = NULL; - virDomainResctrlDefPtr tmp_resctrl = NULL; - int ret = -1; - - if (VIR_ALLOC(tmp_resctrl) < 0) - goto cleanup; + virDomainResctrlDefPtr resctrl = NULL; + virDomainResctrlDefPtr ret = NULL; /* We need to format it back because we need to be consistent in the naming * even when users specify some "sub-optimal" string there. */ - vcpus_str = virBitmapFormat(vcpus); + vcpus_str = virBitmapFormat(*vcpus); if (!vcpus_str) - goto cleanup; + return NULL; if (!(flags & VIR_DOMAIN_DEF_PARSE_INACTIVE)) alloc_id = virXMLPropString(node, "id"); @@ -XXX,XX +XXX,XX @@ virDomainResctrlAppend(virDomainDefPtr def, goto cleanup; } - if (virResctrlAllocSetID(alloc, alloc_id) < 0) + if (virResctrlAllocSetID(*alloc, alloc_id) < 0) goto cleanup; - tmp_resctrl->vcpus = vcpus; - tmp_resctrl->alloc = alloc; + if (VIR_ALLOC(resctrl) < 0) + goto cleanup; - if (VIR_APPEND_ELEMENT(def->resctrls, def->nresctrls, tmp_resctrl) < 0) + if (!(resctrl->vcpus = virBitmapNewCopy(*vcpus))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to copy 'vcpus'")); goto cleanup; + } - ret = 0; + resctrl->alloc = virObjectRef(*alloc); + + VIR_STEAL_PTR(ret, resctrl); cleanup: - virDomainResctrlDefFree(tmp_resctrl); + virDomainResctrlDefFree(resctrl); VIR_FREE(alloc_id); VIR_FREE(vcpus_str); return ret; @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, xmlNodePtr *nodes = NULL; virBitmapPtr vcpus = NULL; virResctrlAllocPtr alloc = NULL; + virDomainResctrlDefPtr resctrl = NULL; ssize_t i = 0; int n; int ret = -1; @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, goto cleanup; } - if (virDomainResctrlAppend(def, node, alloc, vcpus, flags) < 0) + resctrl = virDomainResctrlNew(node, &alloc, &vcpus, flags); + if (!resctrl) + goto cleanup; + + if (VIR_APPEND_ELEMENT(def->resctrls, def->nresctrls, resctrl) < 0) goto cleanup; - vcpus = NULL; - alloc = NULL; ret = 0; cleanup: ctxt->node = oldnode; + virDomainResctrlDefFree(resctrl); virObjectUnref(alloc); virBitmapFree(vcpus); VIR_FREE(nodes); @@ -XXX,XX +XXX,XX @@ virDomainMemorytuneDefParse(virDomainDefPtr def, xmlNodePtr *nodes = NULL; virBitmapPtr vcpus = NULL; virResctrlAllocPtr alloc = NULL; + virDomainResctrlDefPtr resctrl = NULL; + ssize_t i = 0; int n; int ret = -1; @@ -XXX,XX +XXX,XX @@ virDomainMemorytuneDefParse(virDomainDefPtr def, * just update the existing alloc information, which is done in above * virDomainMemorytuneDefParseMemory */ if (new_alloc) { - if (virDomainResctrlAppend(def, node, alloc, vcpus, flags) < 0) + resctrl = virDomainResctrlNew(node, &alloc, &vcpus, flags); + if (!resctrl) + goto cleanup; + + if (VIR_APPEND_ELEMENT(def->resctrls, def->nresctrls, resctrl) < 0) goto cleanup; - vcpus = NULL; - alloc = NULL; } ret = 0; cleanup: ctxt->node = oldnode; + virDomainResctrlDefFree(resctrl); virObjectUnref(alloc); virBitmapFree(vcpus); VIR_FREE(nodes); -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list This refactor allows to add some code between virDomainResctrlNew and virResctrlAllocIsEmpty to extend the scope of resctrl. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/conf/domain_conf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, goto cleanup; } + resctrl = virDomainResctrlNew(node, &alloc, &vcpus, flags); + if (!resctrl) + goto cleanup; + if (virResctrlAllocIsEmpty(alloc)) { ret = 0; goto cleanup; } - resctrl = virDomainResctrlNew(node, &alloc, &vcpus, flags); - if (!resctrl) - goto cleanup; - if (VIR_APPEND_ELEMENT(def->resctrls, def->nresctrls, resctrl) < 0) goto cleanup; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Introducing <monitor> element under <cachetune> to represent a cache monitor. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- docs/formatdomain.html.in | 26 +++ docs/schemas/domaincommon.rng | 10 + src/conf/domain_conf.c | 234 ++++++++++++++++++++- src/conf/domain_conf.h | 11 + tests/genericxml2xmlindata/cachetune-cdp.xml | 3 + .../cachetune-colliding-monitor.xml | 30 +++ tests/genericxml2xmlindata/cachetune-small.xml | 7 + tests/genericxml2xmltest.c | 2 + 8 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 tests/genericxml2xmlindata/cachetune-colliding-monitor.xml diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index XXXXXXX..XXXXXXX 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -XXX,XX +XXX,XX @@ &lt;cachetune vcpus='0-3'&gt; &lt;cache id='0' level='3' type='both' size='3' unit='MiB'/&gt; &lt;cache id='1' level='3' type='both' size='3' unit='MiB'/&gt; + &lt;monitor level='3' vcpus='1'/&gt; + &lt;monitor level='3' vcpus='0-3'/&gt; + &lt;/cachetune&gt; + &lt;cachetune vcpus='4-5'&gt; + &lt;monitor level='3' vcpus='4'/&gt; + &lt;monitor level='3' vcpus='5'/&gt; &lt;/cachetune&gt; &lt;memorytune vcpus='0-3'&gt; &lt;node id='0' bandwidth='60'/&gt; @@ -XXX,XX +XXX,XX @@ </dd> </dl> </dd> + <dt><code>monitor</code></dt> + <dd> + The optional element <code>monitor</code> creates the cache + monitor(s) for current cache allocation and has the following + required attributes: + <dl> + <dt><code>level</code></dt> + <dd> + Host cache level the monitor belongs to. + </dd> + <dt><code>vcpus</code></dt> + <dd> + vCPU list the monitor applies to. A monitor's vCPU list + can only be the member(s) of the vCPU list of associating + allocation. The default monitor has the same vCPU list as the + associating allocation. For non-default monitors, there + are no vCPU overlap permitted. + </dd> + </dl> + </dd> </dl> </dd> diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index XXXXXXX..XXXXXXX 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -XXX,XX +XXX,XX @@ </optional> </element> </zeroOrMore> + <zeroOrMore> + <element name="monitor"> + <attribute name="level"> + <ref name='unsignedInt'/> + </attribute> + <attribute name="vcpus"> + <ref name='cpuset'/> + </attribute> + </element> + </zeroOrMore> </element> </zeroOrMore> <zeroOrMore> diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainLoaderDefFree(virDomainLoaderDefPtr loader) static void +virDomainResctrlMonDefFree(virDomainResctrlMonDefPtr domresmon) +{ + if (!domresmon) + return; + + virBitmapFree(domresmon->vcpus); + virObjectUnref(domresmon->instance); + VIR_FREE(domresmon); +} + + +static void virDomainResctrlDefFree(virDomainResctrlDefPtr resctrl) { + size_t i = 0; + if (!resctrl) return; + for (i = 0; i < resctrl->nmonitors; i++) + virDomainResctrlMonDefFree(resctrl->monitors[i]); + virObjectUnref(resctrl->alloc); virBitmapFree(resctrl->vcpus); + VIR_FREE(resctrl->monitors); VIR_FREE(resctrl); } @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParseCache(xmlXPathContextPtr ctxt, } +/* Checking if the monitor's vcpus is conflicted with existing allocation + * and monitors. + * + * Returns 1 if @vcpus equals to @resctrl->vcpus, then the monitor will + * share the underlying resctrl group with @resctrl->alloc. Returns - 1 + * if any conflict found. Returns 0 if no conflict and @vcpus is not equal + * to @resctrl->vcpus. + */ +static int +virDomainResctrlMonValidateVcpus(virDomainResctrlDefPtr resctrl, + virBitmapPtr vcpus) +{ + size_t i = 0; + int vcpu = -1; + size_t mons_same_alloc_vcpus = 0; + + if (virBitmapIsAllClear(vcpus)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("vcpus is empty")); + return -1; + } + + while ((vcpu = virBitmapNextSetBit(vcpus, vcpu)) >= 0) { + if (!virBitmapIsBitSet(resctrl->vcpus, vcpu)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Monitor vcpus conflicts with allocation")); + return -1; + } + } + + if (virBitmapEqual(vcpus, resctrl->vcpus)) + return 1; + + for (i = 0; i < resctrl->nmonitors; i++) { + if (virBitmapEqual(resctrl->vcpus, resctrl->monitors[i]->vcpus)) { + mons_same_alloc_vcpus++; + continue; + } + + if (virBitmapOverlaps(vcpus, resctrl->monitors[i]->vcpus)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Monitor vcpus conflicts with monitors")); + + return -1; + } + } + + if (mons_same_alloc_vcpus > 1) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Too many monitors have the same vcpu as allocation")); + return -1; + } + + return 0; +} + + +#define VIR_DOMAIN_RESCTRL_MONITOR_CACHELEVEL 3 + +static int +virDomainResctrlMonDefParse(virDomainDefPtr def, + xmlXPathContextPtr ctxt, + xmlNodePtr node, + virResctrlMonitorType tag, + virDomainResctrlDefPtr resctrl) +{ + virDomainResctrlMonDefPtr domresmon = NULL; + xmlNodePtr oldnode = ctxt->node; + xmlNodePtr *nodes = NULL; + unsigned int level = 0; + char *tmp = NULL; + char *id = NULL; + size_t i = 0; + int n = 0; + int rv = -1; + int ret = -1; + + ctxt->node = node; + + if ((n = virXPathNodeSet("./monitor", ctxt, &nodes)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot extract monitor nodes")); + goto cleanup; + } + + for (i = 0; i < n; i++) { + if (VIR_ALLOC(domresmon) < 0) + goto cleanup; + + domresmon->tag = tag; + + domresmon->instance = virResctrlMonitorNew(); + if (!domresmon->instance) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Could not create monitor")); + goto cleanup; + } + + if (tag == VIR_RESCTRL_MONITOR_TYPE_CACHE) { + tmp = virXMLPropString(nodes[i], "level"); + if (!tmp) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("Missing monitor attribute 'level'")); + goto cleanup; + } + + if (virStrToLong_uip(tmp, NULL, 10, &level) < 0) { + virReportError(VIR_ERR_XML_ERROR, + _("Invalid monitor attribute 'level' value '%s'"), + tmp); + goto cleanup; + } + + if (level != VIR_DOMAIN_RESCTRL_MONITOR_CACHELEVEL) { + virReportError(VIR_ERR_XML_ERROR, + _("Invalid monitor cache level '%d'"), + level); + goto cleanup; + } + + VIR_FREE(tmp); + } + + if (virDomainResctrlParseVcpus(def, nodes[i], &domresmon->vcpus) < 0) + goto cleanup; + + rv = virDomainResctrlMonValidateVcpus(resctrl, domresmon->vcpus); + if (rv < 0) + goto cleanup; + + /* If monitor's vcpu list is identical to the vcpu list of the + * associated allocation, set monitor's id to the same value + * as the allocation. */ + if (rv == 1) { + const char *alloc_id = virResctrlAllocGetID(resctrl->alloc); + + if (VIR_STRDUP(id, alloc_id) < 0) + goto cleanup; + } else { + if (!(tmp = virBitmapFormat(domresmon->vcpus))) + goto cleanup; + + if (virAsprintf(&id, "vcpus_%s", tmp) < 0) + goto cleanup; + } + + virResctrlMonitorSetAlloc(domresmon->instance, resctrl->alloc); + + if (virResctrlMonitorSetID(domresmon->instance, id) < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(resctrl->monitors, + resctrl->nmonitors, + domresmon) < 0) + goto cleanup; + + VIR_FREE(id); + VIR_FREE(tmp); + } + + ret = 0; + cleanup: + ctxt->node = oldnode; + VIR_FREE(id); + VIR_FREE(tmp); + VIR_FREE(nodes); + virDomainResctrlMonDefFree(domresmon); + return ret; +} + + static virDomainResctrlDefPtr virDomainResctrlNew(xmlNodePtr node, virResctrlAllocPtr *alloc, @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefParse(virDomainDefPtr def, if (!resctrl) goto cleanup; - if (virResctrlAllocIsEmpty(alloc)) { + if (virDomainResctrlMonDefParse(def, ctxt, node, + VIR_RESCTRL_MONITOR_TYPE_CACHE, + resctrl) < 0) + goto cleanup; + + /* If no <cache> element or <monitor> element in <cachetune>, do not + * append any resctrl element */ + if (virResctrlAllocIsEmpty(alloc) && !resctrl->nmonitors) { ret = 0; goto cleanup; } @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefFormatHelper(unsigned int level, static int +virDomainResctrlMonDefFormatHelper(virDomainResctrlMonDefPtr domresmon, + virResctrlMonitorType tag, + virBufferPtr buf) +{ + char *vcpus = NULL; + + if (domresmon->tag != tag) + return 0; + + virBufferAddLit(buf, "<monitor "); + + if (tag == VIR_RESCTRL_MONITOR_TYPE_CACHE) { + virBufferAsprintf(buf, "level='%u' ", + VIR_DOMAIN_RESCTRL_MONITOR_CACHELEVEL); + } + + vcpus = virBitmapFormat(domresmon->vcpus); + if (!vcpus) + return -1; + + virBufferAsprintf(buf, "vcpus='%s'/>\n", vcpus); + + VIR_FREE(vcpus); + return 0; +} + + +static int virDomainCachetuneDefFormat(virBufferPtr buf, virDomainResctrlDefPtr resctrl, unsigned int flags) { virBuffer childrenBuf = VIR_BUFFER_INITIALIZER; char *vcpus = NULL; + size_t i = 0; int ret = -1; virBufferSetChildIndent(&childrenBuf, buf); @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefFormat(virBufferPtr buf, &childrenBuf) < 0) goto cleanup; + for (i = 0; i < resctrl->nmonitors; i ++) { + if (virDomainResctrlMonDefFormatHelper(resctrl->monitors[i], + VIR_RESCTRL_MONITOR_TYPE_CACHE, + &childrenBuf) < 0) + goto cleanup; + } + if (virBufferCheckError(&childrenBuf) < 0) goto cleanup; diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -XXX,XX +XXX,XX @@ struct _virDomainCputune { }; +typedef struct _virDomainResctrlMonDef virDomainResctrlMonDef; +typedef virDomainResctrlMonDef *virDomainResctrlMonDefPtr; +struct _virDomainResctrlMonDef { + virBitmapPtr vcpus; + virResctrlMonitorType tag; + virResctrlMonitorPtr instance; +}; + typedef struct _virDomainResctrlDef virDomainResctrlDef; typedef virDomainResctrlDef *virDomainResctrlDefPtr; struct _virDomainResctrlDef { virBitmapPtr vcpus; virResctrlAllocPtr alloc; + + virDomainResctrlMonDefPtr *monitors; + size_t nmonitors; }; diff --git a/tests/genericxml2xmlindata/cachetune-cdp.xml b/tests/genericxml2xmlindata/cachetune-cdp.xml index XXXXXXX..XXXXXXX 100644 --- a/tests/genericxml2xmlindata/cachetune-cdp.xml +++ b/tests/genericxml2xmlindata/cachetune-cdp.xml @@ -XXX,XX +XXX,XX @@ <cachetune vcpus='0-1'> <cache id='0' level='3' type='code' size='7680' unit='KiB'/> <cache id='1' level='3' type='data' size='3840' unit='KiB'/> + <monitor level='3' vcpus='0'/> + <monitor level='3' vcpus='1'/> </cachetune> <cachetune vcpus='2'> <cache id='1' level='3' type='code' size='6' unit='MiB'/> + <monitor level='3' vcpus='2'/> </cachetune> <cachetune vcpus='3'> <cache id='1' level='3' type='data' size='6912' unit='KiB'/> diff --git a/tests/genericxml2xmlindata/cachetune-colliding-monitor.xml b/tests/genericxml2xmlindata/cachetune-colliding-monitor.xml new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/genericxml2xmlindata/cachetune-colliding-monitor.xml @@ -XXX,XX +XXX,XX @@ +<domain type='qemu'> + <name>QEMUGuest1</name> + <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>219136</memory> + <currentMemory unit='KiB'>219136</currentMemory> + <vcpu placement='static'>4</vcpu> + <cputune> + <cachetune vcpus='0-1'> + <cache id='0' level='3' type='both' size='768' unit='KiB'/> + <monitor level='3' vcpus='2'/> + </cachetune> + </cputune> + <os> + <type arch='i686' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-i686</emulator> + <controller type='usb' index='0'/> + <controller type='ide' index='0'/> + <controller type='pci' index='0' model='pci-root'/> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <memballoon model='virtio'/> + </devices> +</domain> diff --git a/tests/genericxml2xmlindata/cachetune-small.xml b/tests/genericxml2xmlindata/cachetune-small.xml index XXXXXXX..XXXXXXX 100644 --- a/tests/genericxml2xmlindata/cachetune-small.xml +++ b/tests/genericxml2xmlindata/cachetune-small.xml @@ -XXX,XX +XXX,XX @@ <cputune> <cachetune vcpus='0-1'> <cache id='0' level='3' type='both' size='768' unit='KiB'/> + <monitor level='3' vcpus='0'/> + <monitor level='3' vcpus='1'/> + <monitor level='3' vcpus='0-1'/> + </cachetune> + <cachetune vcpus='2-3'> + <monitor level='3' vcpus='2'/> + <monitor level='3' vcpus='3'/> </cachetune> </cputune> <os> diff --git a/tests/genericxml2xmltest.c b/tests/genericxml2xmltest.c index XXXXXXX..XXXXXXX 100644 --- a/tests/genericxml2xmltest.c +++ b/tests/genericxml2xmltest.c @@ -XXX,XX +XXX,XX @@ mymain(void) TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST_FULL("cachetune-colliding-types", false, true, TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); + DO_TEST_FULL("cachetune-colliding-monitor", false, true, + TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST("memorytune"); DO_TEST_FULL("memorytune-colliding-allocs", false, true, TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Add functions for creating, destroying, reconnecting resctrl monitor in qemu according to the configuration in domain XML. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/qemu/qemu_process.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver, return -1; for (i = 0; i < vm->def->nresctrls; i++) { + size_t j = 0; if (virResctrlAllocCreate(caps->host.resctrl, vm->def->resctrls[i]->alloc, priv->machineName) < 0) goto cleanup; + + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonDefPtr mon = NULL; + + mon = vm->def->resctrls[i]->monitors[j]; + if (virResctrlMonitorCreate(mon->instance, + priv->machineName) < 0) + goto cleanup; + + } } ret = 0; @@ -XXX,XX +XXX,XX @@ qemuProcessSetupVcpu(virDomainObjPtr vm, { pid_t vcpupid = qemuDomainGetVcpuPid(vm, vcpuid); virDomainVcpuDefPtr vcpu = virDomainDefGetVcpu(vm->def, vcpuid); + virDomainResctrlMonDefPtr mon = NULL; size_t i = 0; if (qemuProcessSetupPid(vm, vcpupid, VIR_CGROUP_THREAD_VCPU, @@ -XXX,XX +XXX,XX @@ qemuProcessSetupVcpu(virDomainObjPtr vm, return -1; for (i = 0; i < vm->def->nresctrls; i++) { + size_t j = 0; virDomainResctrlDefPtr ct = vm->def->resctrls[i]; if (virBitmapIsBitSet(ct->vcpus, vcpuid)) { if (virResctrlAllocAddPID(ct->alloc, vcpupid) < 0) return -1; + + /* The order of invoking virResctrlMonitorAddPID matters, it is + * required to invoke this function first for monitor that has + * the same vcpus setting as the allocation in same def->resctrl. + * Otherwise, some other monitor's pid may be removed from its + * resource group's 'tasks' file.*/ + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + mon = vm->def->resctrls[i]->monitors[j]; + + if (!virBitmapEqual(ct->vcpus, mon->vcpus)) + continue; + + if (virBitmapIsBitSet(mon->vcpus, vcpuid)) { + if (virResctrlMonitorAddPID(mon->instance, vcpupid) < 0) + return -1; + } + break; + } + + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + mon = vm->def->resctrls[i]->monitors[j]; + + if (virBitmapEqual(ct->vcpus, mon->vcpus)) + continue; + + if (virBitmapIsBitSet(mon->vcpus, vcpuid)) { + if (virResctrlMonitorAddPID(mon->instance, vcpupid) < 0) + return -1; + } + } break; } } @@ -XXX,XX +XXX,XX @@ void qemuProcessStop(virQEMUDriverPtr driver, /* Remove resctrl allocation after cgroups are cleaned up which makes it * kind of safer (although removing the allocation should work even with * pids in tasks file */ - for (i = 0; i < vm->def->nresctrls; i++) + for (i = 0; i < vm->def->nresctrls; i++) { + size_t j = 0; + + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonDefPtr mon = NULL; + + mon = vm->def->resctrls[i]->monitors[j]; + virResctrlMonitorRemove(mon->instance); + } + virResctrlAllocRemove(vm->def->resctrls[i]->alloc); + } qemuProcessRemoveDomainStatus(driver, vm); @@ -XXX,XX +XXX,XX @@ qemuProcessReconnect(void *opaque) goto error; for (i = 0; i < obj->def->nresctrls; i++) { + size_t j = 0; + if (virResctrlAllocDeterminePath(obj->def->resctrls[i]->alloc, priv->machineName) < 0) goto error; + + for (j = 0; j < obj->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonDefPtr mon = NULL; + + mon = obj->def->resctrls[i]->monitors[j]; + if (virResctrlMonitorDeterminePath(mon->instance, + priv->machineName) < 0) + goto error; + } } /* update domain state XML with possibly updated state in virDomainObj */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Check whether monitor is running by checking the monitor's PIDs status. Monitor is looked as running normally if the vcpu PID list matches with the content of monitor's 'tasks' file. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt_private.syms | 1 + src/util/virresctrl.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++- src/util/virresctrl.h | 3 ++ 3 files changed, 105 insertions(+), 1 deletion(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virResctrlMonitorAddPID; virResctrlMonitorCreate; virResctrlMonitorDeterminePath; virResctrlMonitorGetID; +virResctrlMonitorIsRunning; virResctrlMonitorNew; virResctrlMonitorRemove; virResctrlMonitorSetAlloc; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ struct _virResctrlMonitor { /* libvirt-generated path in /sys/fs/resctrl for this particular * monitor */ char *path; + /* Tracking the tasks' PID associated with this monitor */ + pid_t *pids; + size_t npids; }; @@ -XXX,XX +XXX,XX @@ virResctrlMonitorDispose(void *obj) virObjectUnref(monitor->alloc); VIR_FREE(monitor->id); VIR_FREE(monitor->path); + VIR_FREE(monitor->pids); } @@ -XXX,XX +XXX,XX @@ int virResctrlMonitorAddPID(virResctrlMonitorPtr monitor, pid_t pid) { - return virResctrlAddPID(monitor->path, pid); + size_t i = 0; + + if (virResctrlAddPID(monitor->path, pid) < 0) + return -1; + + for (i = 0; i < monitor->npids; i++) { + if (pid == monitor->pids[i]) + return 0; + } + + if (VIR_APPEND_ELEMENT(monitor->pids, monitor->npids, pid) < 0) + return -1; + + return 0; } @@ -XXX,XX +XXX,XX @@ virResctrlMonitorRemove(virResctrlMonitorPtr monitor) return ret; } + + +static int +virResctrlPIDSorter(const void *pida, const void *pidb) +{ + return *(pid_t*)pida - *(pid_t*)pidb; +} + + +bool +virResctrlMonitorIsRunning(virResctrlMonitorPtr monitor) +{ + char *pidstr = NULL; + char **spids = NULL; + size_t nspids = 0; + pid_t *pids = NULL; + size_t npids = 0; + size_t i = 0; + int rv = -1; + bool ret = false; + + /* path is not determined yet, monitor is not running*/ + if (!monitor->path) + return false; + + /* No vcpu PID filled, regard monitor as not running */ + if (monitor->npids == 0) + return false; + + /* If no 'tasks' file found, underlying resctrl directory is not + * ready, regard monitor as not running */ + rv = virFileReadValueString(&pidstr, "%s/tasks", monitor->path); + if (rv < 0) + goto cleanup; + + /* no PID in task file, monitor is not running */ + if (!*pidstr) + goto cleanup; + + /* The tracking monitor PID list is not identical to the + * list in current resctrl directory. monitor is corrupted, + * report error and un-running state */ + spids = virStringSplitCount(pidstr, "\n", 0, &nspids); + if (nspids != monitor->npids) { + VIR_ERROR(_("Monitor %s PID list mismatch in length"), monitor->path); + goto cleanup; + } + + for (i = 0; i < nspids; i++) { + unsigned int val = 0; + pid_t pid = 0; + + if (virStrToLong_uip(spids[i], NULL, 0, &val) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Monitor %s failed in parse PID list"), + monitor->path); + goto cleanup; + } + + pid = (pid_t)val; + + if (VIR_APPEND_ELEMENT(pids, npids, pid) < 0) + goto cleanup; + } + + qsort(pids, npids, sizeof(pid_t), virResctrlPIDSorter); + qsort(monitor->pids, monitor->npids, sizeof(pid_t), virResctrlPIDSorter); + + for (i = 0; i < monitor->npids; i++) { + if (monitor->pids[i] != pids[i]) { + VIR_ERROR(_("Monitor %s PID list corrupted"), monitor->path); + goto cleanup; + } + } + + ret = true; + cleanup: + virStringListFree(spids); + VIR_FREE(pids); + VIR_FREE(pidstr); + + return ret; +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ virResctrlMonitorSetAlloc(virResctrlMonitorPtr monitor, int virResctrlMonitorRemove(virResctrlMonitorPtr monitor); + +bool +virResctrlMonitorIsRunning(virResctrlMonitorPtr monitor); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Adding element 'id' to virDomainResctrlDef tracking resource group id, it reflects the attribute 'id' of of element <cachetune> in XML. virResctrlAlloc.id is a copy from virDomanResctrlDef.id. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/conf/domain_conf.c | 20 ++++++++------------ src/conf/domain_conf.h | 1 + 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainResctrlDefFree(virDomainResctrlDefPtr resctrl) virObjectUnref(resctrl->alloc); virBitmapFree(resctrl->vcpus); VIR_FREE(resctrl->monitors); + VIR_FREE(resctrl->id); VIR_FREE(resctrl); } @@ -XXX,XX +XXX,XX @@ virDomainResctrlNew(xmlNodePtr node, if (VIR_ALLOC(resctrl) < 0) goto cleanup; + if (VIR_STRDUP(resctrl->id, alloc_id) < 0) + goto cleanup; + if (!(resctrl->vcpus = virBitmapNewCopy(*vcpus))) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("failed to copy 'vcpus'")); @@ -XXX,XX +XXX,XX @@ virDomainCachetuneDefFormat(virBufferPtr buf, virBufferAsprintf(buf, "<cachetune vcpus='%s'", vcpus); - if (!(flags & VIR_DOMAIN_DEF_FORMAT_INACTIVE)) { - const char *alloc_id = virResctrlAllocGetID(resctrl->alloc); - if (!alloc_id) - goto cleanup; + if (!(flags & VIR_DOMAIN_DEF_FORMAT_INACTIVE)) + virBufferAsprintf(buf, " id='%s'", resctrl->id); - virBufferAsprintf(buf, " id='%s'", alloc_id); - } virBufferAddLit(buf, ">\n"); virBufferAddBuffer(buf, &childrenBuf); @@ -XXX,XX +XXX,XX @@ virDomainMemorytuneDefFormat(virBufferPtr buf, virBufferAsprintf(buf, "<memorytune vcpus='%s'", vcpus); - if (!(flags & VIR_DOMAIN_DEF_FORMAT_INACTIVE)) { - const char *alloc_id = virResctrlAllocGetID(resctrl->alloc); - if (!alloc_id) - goto cleanup; + if (!(flags & VIR_DOMAIN_DEF_FORMAT_INACTIVE)) + virBufferAsprintf(buf, " id='%s'", resctrl->id); - virBufferAsprintf(buf, " id='%s'", alloc_id); - } virBufferAddLit(buf, ">\n"); virBufferAddBuffer(buf, &childrenBuf); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -XXX,XX +XXX,XX @@ typedef struct _virDomainResctrlDef virDomainResctrlDef; typedef virDomainResctrlDef *virDomainResctrlDefPtr; struct _virDomainResctrlDef { + char *id; virBitmapPtr vcpus; virResctrlAllocPtr alloc; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Refactoring qemuDomainGetStatsCpu, make it possible to add more CPU statistics. Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/qemu/qemu_driver.c | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ qemuDomainGetStatsCpu(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, unsigned long long sys_time = 0; int err = 0; - if (!priv->cgroup) - return 0; - - err = virCgroupGetCpuacctUsage(priv->cgroup, &cpu_time); - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.time", - cpu_time) < 0) - return -1; + if (priv->cgroup) { + err = virCgroupGetCpuacctUsage(priv->cgroup, &cpu_time); + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.time", + cpu_time) < 0) + return -1; - err = virCgroupGetCpuacctStat(priv->cgroup, &user_time, &sys_time); - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.user", - user_time) < 0) - return -1; - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.system", - sys_time) < 0) - return -1; + err = virCgroupGetCpuacctStat(priv->cgroup, &user_time, &sys_time); + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.user", + user_time) < 0) + return -1; + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.system", + sys_time) < 0) + return -1; + } return 0; } -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Adding the interface in qemu to report CMT statistic information through command 'virsh domstats --cpu-total'. Below is a typical output: # virsh domstats 1 --cpu-total Domain: 'ubuntu16.04-base' ... cpu.cache.monitor.count=2 cpu.cache.monitor.0.name=vcpus_1 cpu.cache.monitor.0.vcpus=1 cpu.cache.monitor.0.bank.count=2 cpu.cache.monitor.0.bank.0.id=0 cpu.cache.monitor.0.bank.0.bytes=4505600 cpu.cache.monitor.0.bank.1.id=1 cpu.cache.monitor.0.bank.1.bytes=5586944 cpu.cache.monitor.1.name=vcpus_4-6 cpu.cache.monitor.1.vcpus=4,5,6 cpu.cache.monitor.1.bank.count=2 cpu.cache.monitor.1.bank.0.id=0 cpu.cache.monitor.1.bank.0.bytes=17571840 cpu.cache.monitor.1.bank.1.id=1 cpu.cache.monitor.1.bank.1.bytes=29106176 Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/libvirt-domain.c | 9 ++ src/libvirt_private.syms | 1 + src/qemu/qemu_driver.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virresctrl.c | 130 +++++++++++++++++++++++++++ src/util/virresctrl.h | 12 +++ 5 files changed, 381 insertions(+) diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -XXX,XX +XXX,XX @@ virConnectGetDomainCapabilities(virConnectPtr conn, * "cpu.user" - user cpu time spent in nanoseconds as unsigned long long. * "cpu.system" - system cpu time spent in nanoseconds as unsigned long * long. + * "cpu.cache.monitor.count" - tocal cache monitoring groups + * "cpu.cache.monitor.M.name" - name of cache monitoring group 'M' + * "cpu.cache.monitor.M.vcpus" - vcpus for cache monitoring group 'M' + * "cpu.cache.monitor.M.bank.count" - total bank number of cache monitoring + * group 'M' + * "cpu.cache.monitor.M.bank.N.id" - OS assigned cache bank id for cache + * 'N' in cache monitoring group 'M' + * "cpu.cache.monitor.M.bank.N.bytes" - monitor's cache occupancy of cache + * bank 'N' in cache monitoring group 'M' * * VIR_DOMAIN_STATS_BALLOON: * Return memory balloon device information. diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virResctrlInfoNew; virResctrlMonitorAddPID; virResctrlMonitorCreate; virResctrlMonitorDeterminePath; +virResctrlMonitorGetCacheOccupancy; virResctrlMonitorGetID; virResctrlMonitorIsRunning; virResctrlMonitorNew; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ #include "virnuma.h" #include "dirname.h" #include "netdev_bandwidth_conf.h" +#include "c-ctype.h" #define VIR_FROM_THIS VIR_FROM_QEMU @@ -XXX,XX +XXX,XX @@ typedef enum { #define HAVE_JOB(flags) ((flags) & QEMU_DOMAIN_STATS_HAVE_JOB) +/* In terms of the output of virBitmapFormat, both '1-3' and '1,3' are valid + * outputs and represent different vcpu set. But it is not easy to + * differentiate these two vcpu set formats at first glance. + * + * This function could be used to clear this ambiguity, it substitutes all '-' + * with ',' while generates semantically correct vcpu set. + * e.g. vcpu set string '1-3' will be replaced by string '1,2,3'. */ +static char * +qemuDomainVcpuFormatHelper(const char *vcpus) +{ + size_t i = 0; + int last = 0; + int start = 0; + char * tmp = NULL; + bool firstnum = true; + const char *cur = vcpus; + virBuffer buf = VIR_BUFFER_INITIALIZER; + char *ret = NULL; + + if (virStringIsEmpty(cur)) + return NULL; + + while (*cur != '\0') { + if (!c_isdigit(*cur)) + goto cleanup; + + if (virStrToLong_i(cur, &tmp, 10, &start) < 0) + goto cleanup; + if (start < 0) + goto cleanup; + + cur = tmp; + + virSkipSpaces(&cur); + + if (*cur == ',' || *cur == 0) { + if (!firstnum) + virBufferAddChar(&buf, ','); + virBufferAsprintf(&buf, "%d", start); + firstnum = false; + } else if (*cur == '-') { + cur++; + virSkipSpaces(&cur); + + if (virStrToLong_i(cur, &tmp, 10, &last) < 0) + goto cleanup; + + if (last < start) + goto cleanup; + cur = tmp; + + for (i = start; i <= last; i++) { + if (!firstnum) + + virBufferAddChar(&buf, ','); + virBufferAsprintf(&buf, "%ld", i); + firstnum = 0; + } + + virSkipSpaces(&cur); + } + + if (*cur == ',') { + cur++; + virSkipSpaces(&cur); + } else if (*cur == 0) { + break; + } else { + goto cleanup; + } + } + + ret = virBufferContentAndReset(&buf); + cleanup: + virBufferFreeAndReset(&buf); + return ret; +} + + +static int +qemuDomainGetStatsCpuResource(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, + virDomainObjPtr dom, + virDomainStatsRecordPtr record, + int *maxparams, + unsigned int privflags ATTRIBUTE_UNUSED, + virResctrlMonitorType restag) +{ + char param_name[VIR_TYPED_PARAM_FIELD_LENGTH]; + virDomainResctrlMonDefPtr domresmon = NULL; + virResctrlMonitorStatsPtr stats = NULL; + size_t nstats = 0; + virDomainResctrlDefPtr resctrl = NULL; + unsigned int nmonitors = 0; + unsigned int imonitor = 0; + const char *restype = NULL; + char *rawvcpus = NULL; + char *vcpus = NULL; + size_t i = 0; + size_t j = 0; + int ret = -1; + + if (!virDomainObjIsActive(dom)) + return 0; + + if (restag == VIR_RESCTRL_MONITOR_TYPE_CACHE) { + restype = "cache"; + } else { + VIR_DEBUG("Invalid CPU resource type"); + return -1; + } + + for (i = 0; i < dom->def->nresctrls; i++) { + resctrl = dom->def->resctrls[i]; + + for (j = 0; j < resctrl->nmonitors; j++) { + domresmon = resctrl->monitors[j]; + if (virResctrlMonitorIsRunning(domresmon->instance) && + domresmon->tag == restag) + nmonitors++; + } + } + + if (nmonitors) { + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.%s.monitor.count", restype); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + nmonitors) < 0) + goto cleanup; + } + + for (i = 0; i < dom->def->nresctrls; i++) { + resctrl = dom->def->resctrls[i]; + + for (j = 0; j < resctrl->nmonitors; j++) { + size_t l = 0; + virResctrlMonitorPtr monitor = resctrl->monitors[j]->instance; + const char *id = virResctrlMonitorGetID(monitor); + + if (!id) + goto cleanup; + + domresmon = resctrl->monitors[j]; + + if (!virResctrlMonitorIsRunning(domresmon->instance)) + continue; + + if (!(rawvcpus = virBitmapFormat(domresmon->vcpus))) + goto cleanup; + + vcpus = qemuDomainVcpuFormatHelper(rawvcpus); + if (!vcpus) + goto cleanup; + + if (virResctrlMonitorGetCacheOccupancy(monitor, &stats, + &nstats) < 0) + goto cleanup; + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.%s.monitor.%d.name", restype, imonitor); + if (virTypedParamsAddString(&record->params, + &record->nparams, + maxparams, + param_name, + id) < 0) + goto cleanup; + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.%s.monitor.%d.vcpus", restype, imonitor); + + if (virTypedParamsAddString(&record->params, + &record->nparams, + maxparams, + param_name, + vcpus) < 0) + goto cleanup; + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.%s.monitor.%d.bank.count", restype, imonitor); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + nstats) < 0) + goto cleanup; + + for (l = 0; l < nstats; l++) { + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.%s.monitor.%d.bank.%ld.id", + restype, imonitor, l); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + stats[l].id) < 0) + goto cleanup; + + + snprintf(param_name, VIR_TYPED_PARAM_FIELD_LENGTH, + "cpu.%s.monitor.%d.bank.%ld.bytes", + restype, imonitor, l); + if (virTypedParamsAddUInt(&record->params, + &record->nparams, + maxparams, + param_name, + stats[l].val) < 0) + goto cleanup; + } + + VIR_FREE(stats); + VIR_FREE(vcpus); + imonitor++; + } + } + + ret = 0; + cleanup: + VIR_FREE(vcpus); + return ret; +} + + static int qemuDomainGetStatsCpu(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, virDomainObjPtr dom, @@ -XXX,XX +XXX,XX @@ qemuDomainGetStatsCpu(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, return -1; } + if (qemuDomainGetStatsCpuResource(driver, dom, record, maxparams, privflags, + VIR_RESCTRL_MONITOR_TYPE_CACHE) < 0) + return -1; + return 0; } diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -XXX,XX +XXX,XX @@ virResctrlMonitorIsRunning(virResctrlMonitorPtr monitor) return ret; } + + +static int +virResctrlMonitorStatsSorter(const void *a, + const void *b) +{ + return ((virResctrlMonitorStatsPtr)a)->id + - ((virResctrlMonitorStatsPtr)b)->id; +} + + +/* + * virResctrlMonitorGetStats + * + * @monitor: The monitor that the statistic data will be retrieved from. + * @resource: The name for resource name. 'llc_occupancy' for cache resource. + * "mbm_total_bytes" and "mbm_local_bytes" for memory bandwidth resource. + * @stats: Array of virResctrlMonitorStatsPtr for holding cache or memory + * bandwidth usage data. + * @nstats: A size_t pointer to hold the returned array length of @stats + * + * Get cache or memory bandwidth utilization information. + * + * Returns 0 on success, -1 on error. + */ +static int +virResctrlMonitorGetStats(virResctrlMonitorPtr monitor, + const char *resource, + virResctrlMonitorStatsPtr *stats, + size_t *nstats) +{ + int rv = -1; + int ret = -1; + DIR *dirp = NULL; + char *datapath = NULL; + struct dirent *ent = NULL; + virResctrlMonitorStatsPtr stat = NULL; + + if (!monitor) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Invalid resctrl monitor")); + return -1; + } + + if (virAsprintf(&datapath, "%s/mon_data", monitor->path) < 0) + return -1; + + if (virDirOpen(&dirp, datapath) < 0) + goto cleanup; + + *nstats = 0; + while (virDirRead(dirp, &ent, datapath) > 0) { + char *node_id = NULL; + + if (VIR_ALLOC(stat) < 0) + goto cleanup; + + /* Looking for directory that contains resource utilization + * information file. The directory name is arranged in format + * "mon_<node_name>_<node_id>". For example, "mon_L3_00" and + * "mon_L3_01" are two target directories for a two nodes system + * with resource utilization data file for each node respectively. + */ + if (ent->d_type != DT_DIR) + continue; + + /* Looking for directory has a prefix 'mon_L' */ + if (!(node_id = STRSKIP(ent->d_name, "mon_L"))) + continue; + + /* Looking for directory has another '_' */ + node_id = strchr(node_id, '_'); + if (!node_id) + continue; + + /* Skip the character '_' */ + if (!(node_id = STRSKIP(node_id, "_"))) + continue; + + /* The node ID number should be here, parsing it. */ + if (virStrToLong_uip(node_id, NULL, 0, &stat->id) < 0) + goto cleanup; + + rv = virFileReadValueUint(&stat->val, "%s/%s/%s", datapath, + ent->d_name, resource); + if (rv == -2) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("File '%s/%s/%s' does not exist."), + datapath, ent->d_name, resource); + } + if (rv < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(*stats, *nstats, *stat) < 0) + goto cleanup; + } + + /* Sort in id's ascending order */ + qsort(*stats, *nstats, sizeof(*stat), virResctrlMonitorStatsSorter); + + ret = 0; + cleanup: + VIR_FREE(datapath); + VIR_FREE(stat); + VIR_DIR_CLOSE(dirp); + return ret; +} + + +/* + * virResctrlMonitorGetCacheOccupancy + * + * @monitor: The monitor that the statistic data will be retrieved from. + * @caches: Array of virResctrlMonitorStatsPtr for receiving cache occupancy + * data. Caller is responsible to free this array. + * @ncaches: A size_t pointer to hold the returned array length of @caches + * + * Get cache or memory bandwidth utilization information. + * + * Returns 0 on success, -1 on error. + */ + +int +virResctrlMonitorGetCacheOccupancy(virResctrlMonitorPtr monitor, + virResctrlMonitorStatsPtr *caches, + size_t *ncaches) +{ + return virResctrlMonitorGetStats(monitor, "llc_occupancy", + caches, ncaches); +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -XXX,XX +XXX,XX @@ virResctrlInfoGetMonitorPrefix(virResctrlInfoPtr resctrl, typedef struct _virResctrlMonitor virResctrlMonitor; typedef virResctrlMonitor *virResctrlMonitorPtr; +typedef struct _virResctrlMonitorStats virResctrlMonitorStats; +typedef virResctrlMonitorStats *virResctrlMonitorStatsPtr; +struct _virResctrlMonitorStats { + unsigned int id; + unsigned int val; +}; + virResctrlMonitorPtr virResctrlMonitorNew(void); @@ -XXX,XX +XXX,XX @@ virResctrlMonitorRemove(virResctrlMonitorPtr monitor); bool virResctrlMonitorIsRunning(virResctrlMonitorPtr monitor); + +int +virResctrlMonitorGetCacheOccupancy(virResctrlMonitorPtr monitor, + virResctrlMonitorStatsPtr *caches, + size_t *ncaches); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list Invoking qemuProcessSetupVcpus in process of VM reconnection. The vcpu pid information need to be refilled to resctrl monitor after a VM reconnection./ Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com> --- src/qemu/qemu_process.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessReconnect(void *opaque) } } + if (qemuProcessSetupVcpus(obj) < 0) + goto error; + /* update domain state XML with possibly updated state in virDomainObj */ if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, obj, driver->caps) < 0) goto error; -- 2.7.4 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list