From nobody Sat Jul 12 04:31:00 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.39 as permitted sender) client-ip=209.132.183.39; envelope-from=libvir-list-bounces@redhat.com; helo=mx6-phx2.redhat.com; Authentication-Results: mx.zoho.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.39 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; Return-Path: Received: from mx6-phx2.redhat.com (mx6-phx2.redhat.com [209.132.183.39]) by mx.zohomail.com with SMTPS id 1488794895963407.85237162625936; Mon, 6 Mar 2017 02:08:15 -0800 (PST) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by mx6-phx2.redhat.com (8.14.4/8.14.4) with ESMTP id v26A4qvw036156; Mon, 6 Mar 2017 05:04:52 -0500 Received: from int-mx09.intmail.prod.int.phx2.redhat.com (int-mx09.intmail.prod.int.phx2.redhat.com [10.5.11.22]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id v26A4oCX030981 for ; Mon, 6 Mar 2017 05:04:50 -0500 Received: from mx1.redhat.com (ext-mx05.extmail.prod.ext.phx2.redhat.com [10.5.110.29]) by int-mx09.intmail.prod.int.phx2.redhat.com (8.14.4/8.14.4) with ESMTP id v26A4oxM009535 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-GCM-SHA384 bits=256 verify=NO); Mon, 6 Mar 2017 05:04:50 -0500 Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id C6EA33DD47; Mon, 6 Mar 2017 10:04:48 +0000 (UTC) Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 06 Mar 2017 02:04:47 -0800 Received: from s2600wt.sh.intel.com ([10.239.48.158]) by fmsmga001.fm.intel.com with ESMTP; 06 Mar 2017 02:04:46 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.35,253,1484035200"; d="scan'208";a="1118957038" From: Eli Qiao To: libvir-list@redhat.com Date: Mon, 6 Mar 2017 18:06:38 +0800 Message-Id: <1488794801-13844-10-git-send-email-liyong.qiao@intel.com> In-Reply-To: <1488794801-13844-1-git-send-email-liyong.qiao@intel.com> References: <1488794801-13844-1-git-send-email-liyong.qiao@intel.com> X-Greylist: Sender passed SPF test, Sender IP whitelisted by DNSRBL, ACL 203 matched, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.29]); Mon, 06 Mar 2017 10:04:49 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.29]); Mon, 06 Mar 2017 10:04:49 +0000 (UTC) for IP:'192.55.52.93' DOMAIN:'mga11.intel.com' HELO:'mga11.intel.com' FROM:'liyong.qiao@intel.com' RCPT:'' X-RedHat-Spam-Score: -4.303 (BAYES_50, DCC_REPUT_13_19, RCVD_IN_DNSWL_HI, RCVD_IN_MSPIKE_H2, RP_MATCHES_RCVD, SPF_PASS) 192.55.52.93 mga11.intel.com 192.55.52.93 mga11.intel.com X-Scanned-By: MIMEDefang 2.68 on 10.5.11.22 X-Scanned-By: MIMEDefang 2.78 on 10.5.110.29 X-loop: libvir-list@redhat.com Cc: mkletzan@redhat.com, mtosatti@redhat.com, qiaoliyong@gmail.com Subject: [libvirt] [PATCH resend V10 09/12] Resctrl: concurrence support X-BeenThere: libvir-list@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Development discussions about the libvirt library & tools List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: libvir-list-bounces@redhat.com Errors-To: libvir-list-bounces@redhat.com X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" The internal struct list domainall is a list which are resctral domain status shared by all VMs, especiall the default domain, each VM should access it concomitantly. Ues a mutex to control it. Each bank's cache_left field is also a global shared resource we need to be care, add a mutex for each bank. We need also to add lock to access /sys/fs/resctrl, use flock. Signed-off-by: Eli Qiao --- src/util/virresctrl.c | 93 ++++++++++++++++++++++++++++++++++++++++++++---= ---- src/util/virresctrl.h | 3 ++ 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index 9f0d05f..3c54e44 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -20,7 +20,9 @@ */ #include #include +#include #include +#include =20 #include "virresctrl.h" #include "viralloc.h" @@ -62,6 +64,9 @@ do { \ =20 #define VIR_RESCTRL_GET_SCHEMATA(count) ((1 << count) - 1) =20 +#define VIR_RESCTRL_LOCK(fd, op) flock(fd, op) +#define VIR_RESCTRL_UNLOCK(fd) flock(fd, LOCK_UN) + /** * a virResSchemata represents a schemata object under a resource control * domain. @@ -105,6 +110,8 @@ typedef virResCtrlDomain *virResCtrlDomainPtr; struct _virResCtrlDomain { unsigned int num_domains; virResDomainPtr domains; + + virMutex lock; }; =20 static unsigned int host_id; @@ -176,11 +183,16 @@ static int virResCtrlGetStr(const char *domain_name, = const char *item_name, char =20 CONSTRUCT_RESCTRL_PATH(domain_name, item_name); =20 + if (!virFileExists(path)) + goto cleanup; + if (virFileReadAll(path, MAX_FILE_LEN, ret) < 0) { rc =3D -1; goto cleanup; } =20 + rc =3D 0; + cleanup: VIR_FREE(path); return rc; @@ -687,10 +699,15 @@ static int virResCtrlAppendDomain(virResDomainPtr dom) { virResDomainPtr p =3D domainall.domains; + + virMutexLock(&domainall.lock); + while (p->next !=3D NULL) p =3D p->next; p->next =3D dom; dom->pre =3D p; domainall.num_domains +=3D 1; + + virMutexUnlock(&domainall.lock); return 0; } =20 @@ -713,18 +730,22 @@ virResCtrlCalculateSchemata(int type, { size_t i; int count; + int rc =3D -1; virResDomainPtr p; unsigned int tmp_schemata; unsigned int schemata_sum =3D 0; int pair_type =3D 0; =20 + virMutexLock(&resctrlall[type].cache_banks[sid].lock); + if (resctrlall[type].cache_banks[sid].cache_left < size) { VIR_ERROR(_("Not enough cache left on bank %u"), hostid); - return -1; + goto cleanup; } + if ((count =3D size / resctrlall[type].cache_banks[sid].cache_min) <= =3D 0) { VIR_ERROR(_("Error cache size %llu"), size); - return -1; + goto cleanup; } =20 tmp_schemata =3D VIR_RESCTRL_GET_SCHEMATA(count); @@ -739,7 +760,7 @@ virResCtrlCalculateSchemata(int type, if (type =3D=3D VIR_RDT_RESOURCE_L3CODE) pair_type =3D VIR_RDT_RESOURCE_L3DATA; =20 - for (i =3D 1; i < domainall.num_domains; i ++) { + for (i =3D 1; i < domainall.num_domains; i++) { schemata_sum |=3D p->schematas[type]->schemata_items[sid].schemata; if (pair_type > 0) schemata_sum |=3D p->schematas[pair_type]->schemata_items[sid]= .schemata; @@ -750,7 +771,16 @@ virResCtrlCalculateSchemata(int type, =20 while ((tmp_schemata & schemata_sum) !=3D 0) tmp_schemata =3D tmp_schemata >> 1; - return tmp_schemata; + + resctrlall[type].cache_banks[sid].cache_left -=3D size; + if (pair_type > 0) + resctrlall[pair_type].cache_banks[sid].cache_left =3D resctrlall[t= ype].cache_banks[sid].cache_left; + + rc =3D tmp_schemata; + + cleanup: + virMutexUnlock(&resctrlall[type].cache_banks[sid].lock); + return rc; } =20 int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, @@ -763,8 +793,8 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cache= tune, int pair_type =3D -1; int sid; int schemata; - - virUUIDFormat(uuid, name); + int lockfd; + int rc =3D -1; =20 for (i =3D 0; i < cachetune->n_banks; i++) { VIR_DEBUG("cache_banks %u, %u, %llu, %s", @@ -777,12 +807,21 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cac= hetune, if (cachetune->n_banks < 1) return 0; =20 + virUUIDFormat(uuid, name); + + if ((lockfd =3D open(RESCTRL_DIR, O_RDONLY)) < 0) + goto cleanup; + + if (VIR_RESCTRL_LOCK(lockfd, LOCK_EX) < 0) { + virReportSystemError(errno, _("Unable to lock '%s'"), RESCTRL_DIR); + goto cleanup; + } + p =3D virResCtrlGetDomain(name); if (p =3D=3D NULL) { VIR_DEBUG("no domain name %s found, create new one!", name); p =3D virResCtrlCreateDomain(name); } - if (p !=3D NULL) { =20 virResCtrlAppendDomain(p); @@ -828,19 +867,25 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cac= hetune, if (virResCtrlFlushDomainToSysfs(p) < 0) { VIR_ERROR(_("failed to flush domain %s to sysfs"), name); virResCtrlDestroyDomain(p); - return -1; + goto cleanup; } } else { VIR_ERROR(_("Failed to create a domain in sysfs")); - return -1; + goto cleanup; } =20 virResCtrlRefreshSchemata(); /* after refresh, flush header's schemata changes to sys fs */ - if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0) - VIR_WARN("failed to flush domain to sysfs"); + if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0) { + VIR_ERROR(_("failed to flush domain to sysfs")); + goto cleanup; + } =20 - return 0; + rc =3D 0; + + cleanup: + VIR_RESCTRL_UNLOCK(lockfd); + return rc; } =20 /* Should be called after pid disappeared, we recalculate @@ -856,6 +901,9 @@ int virResCtrlUpdate(unsigned char *uuid) del =3D virResCtrlGetDomain(name); =20 if (del !=3D NULL) { + + virMutexLock(&domainall.lock); + del->pre->next =3D del->next; if (del->next !=3D NULL) del->next->pre =3D del->pre; @@ -864,6 +912,7 @@ int virResCtrlUpdate(unsigned char *uuid) virResCtrlRefreshSchemata(); if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0) VIR_WARN("failed to flush domain to sysfs"); + virMutexUnlock(&domainall.lock); } return 0; } @@ -871,7 +920,7 @@ int virResCtrlUpdate(unsigned char *uuid) int virResCtrlInit(void) { - size_t i =3D 0; + size_t i, j; char *tmp; int rc =3D 0; =20 @@ -898,6 +947,24 @@ virResCtrlInit(void) =20 domainall.domains =3D virResCtrlGetAllDomains(&(domainall.num_domains)= ); =20 + for (i =3D 0; i < VIR_RDT_RESOURCE_LAST; i++) { + if (VIR_RESCTRL_ENABLED(i)) { + for (j =3D 0; j < resctrlall[i].num_banks; j++) { + if (virMutexInit(&resctrlall[i].cache_banks[j].lock) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to initialize mutex")); + return -1; + } + } + } + } + + if (virMutexInit(&domainall.lock) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to initialize mutex")); + return -1; + } + if ((rc =3D virResCtrlRefreshSchemata()) < 0) virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Failed to refresh resource control")); diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index d639de1..968e0dc 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -27,6 +27,7 @@ # include "virbitmap.h" # include "virutil.h" # include "conf/domain_conf.h" +# include "virthread.h" =20 #define MAX_CPU_SOCKET_NUM 8 =20 @@ -49,6 +50,8 @@ struct _virResCacheBank { unsigned long long cache_left; unsigned long long cache_min; virBitmapPtr cpu_mask; + + virMutex lock; }; =20 /** --=20 1.9.1 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list