From nobody Thu Apr 25 12:58:40 2024 Delivered-To: importer2@patchew.org Received-SPF: pass (zohomail.com: domain of vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18; envelope-from=linux-kernel-owner@vger.kernel.org; helo=vger.kernel.org; Authentication-Results: mx.zohomail.com; dkim=fail; spf=pass (zohomail.com: domain of vger.kernel.org designates 23.128.96.18 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org; dmarc=fail(p=none dis=none) header.from=gmail.com Return-Path: Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mx.zohomail.com with SMTP id 1643894004775982.0027036173418; Thu, 3 Feb 2022 05:13:24 -0800 (PST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1350713AbiBCNNX (ORCPT ); Thu, 3 Feb 2022 08:13:23 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:39910 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347799AbiBCNNV (ORCPT ); Thu, 3 Feb 2022 08:13:21 -0500 Received: from mail-oo1-xc2c.google.com (mail-oo1-xc2c.google.com [IPv6:2607:f8b0:4864:20::c2c]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 8C6D2C061714 for ; Thu, 3 Feb 2022 05:13:21 -0800 (PST) Received: by mail-oo1-xc2c.google.com with SMTP id o192-20020a4a2cc9000000b00300af40d795so1442016ooo.13 for ; Thu, 03 Feb 2022 05:13:21 -0800 (PST) Received: from localhost.localdomain ([179.223.196.228]) by smtp.gmail.com with ESMTPSA id n12sm16665537oop.5.2022.02.03.05.13.18 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 03 Feb 2022 05:13:20 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=l1NG9qud1LpaSOVSQkpnHko+6K/HJr+ErHRZPqMtGVU=; b=hLT6n8MMx784Gy81FKXkW2EMnPjpoNPQgm4PQUZCiFZvAvVdslEENx8Py9Zy3IP8BI W9cd9oiTC1l7imttsMxLJTKjlIbqDTVLHdWL9hP5zKwUp9GWe5xRerqT2zBlXtg4DX4x P6Mgx/tf8ZZBiSB3reeUeKJ5XoDIUZKWo1W9D3+THh56NXywdvmhh8/CSWvtgGq2jq6C Xx36DEhpfOXvkpTwujxxwXj+eYQCSnqdoEQx1/zFUo3BMRUgLFZZLwFOMFi9eog15IIZ gJl+9z8Zzb2tAfw22dFhkwwwP5dCW78uYSY5ZiOBu7MATYZ64W6aCT113y+BW5QrH8QT 1U+A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=l1NG9qud1LpaSOVSQkpnHko+6K/HJr+ErHRZPqMtGVU=; b=6W7mJvo1HiY3WjEiETK/6wVIPANLVkvRcNMI8FWRzl7ZIHipdi1YSHVCxBHGhujlsc bnPJ7fWzRYZfhq4hR0deXvMAzAQD2eNiBVFquOjJNKXFMOecczeJrVUGYN5UmBmFC7xt 1wLlsupFmWN7OEuDeoAjwhfJofJKmwFX2HpBYo/2otJDnT2LwgjYdiQ9nzx+iPIXimci 8IKRHE10eh98NxVaAZPEesKJPrB1obU99O8aS5tr0tx7G96wRBBGXsgc9j1SMJRxi39e oZcSFLFRu3M+OsvTHvTPU8sGwEPO132Sehu43XJCfDpJnGQ72VK8rbpWANjtbYGBLlES d60w== X-Gm-Message-State: AOAM533ErRE8nABtAqfM0OLeeDUismEPLzSRoJnXVBIYirqEr93zIwfk NRqr/KF24yVNteAeAfts+JA= X-Google-Smtp-Source: ABdhPJw606awtxkHoaAQoMDNkwgrOoN9kSTAdyPIExCnKxwSn/7RBLxmhGiiS+4Vyxas89WUbwE3MA== X-Received: by 2002:a05:6870:d8ae:: with SMTP id dv46mr198000oab.228.1643894000683; Thu, 03 Feb 2022 05:13:20 -0800 (PST) From: Pedro Demarchi Gomes Cc: Pedro Demarchi Gomes , SeongJae Park , Steven Rostedt , Ingo Molnar , Andrew Morton , linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [PATCH] mm/damon: Add option to monitor only writes Date: Thu, 3 Feb 2022 10:12:36 -0300 Message-Id: <20220203131237.298090-1-pedrodemargomes@gmail.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable To: unlisted-recipients:; (no To-header on input) Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org X-ZohoMail-DKIM: fail (Header signature does not verify) X-ZM-MESSAGEID: 1643894007921100001 Content-Type: text/plain; charset="utf-8" When "writes" is written to /sys/kernel/debug/damon/counter_type damon will= monitor only writes. This patch also adds the actions mergeable and unmergeable to damos schemes= . These actions are used by KSM as explained in [1]. [1] https://www.kernel.org/doc/html/latest/admin-guide/mm/ksm.html#controll= ing-ksm-with-madvise Signed-off-by: Pedro Demarchi Gomes --- include/linux/damon.h | 13 ++++++ include/trace/events/damon.h | 10 ++-- mm/damon/core.c | 91 +++++++++++++++++++++++++----------- mm/damon/dbgfs.c | 72 +++++++++++++++++++++++++++- mm/damon/prmtv-common.c | 88 ++++++++++++++++++++++++++++++++++ mm/damon/prmtv-common.h | 5 ++ mm/damon/vaddr.c | 80 ++++++++++++++++++++++++++++--- 7 files changed, 318 insertions(+), 41 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index b4d4be3cc987..9efe89bbcec8 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -44,11 +44,13 @@ struct damon_region { struct damon_addr_range ar; unsigned long sampling_addr; unsigned int nr_accesses; + unsigned int nr_writes; struct list_head list; =20 unsigned int age; /* private: Internal value for age calculation. */ unsigned int last_nr_accesses; + unsigned int last_nr_writes; }; =20 /** @@ -88,6 +90,8 @@ enum damos_action { DAMOS_PAGEOUT, DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, + DAMOS_MERGEABLE, + DAMOS_UNMERGEABLE, DAMOS_STAT, /* Do nothing but only record the stat */ }; =20 @@ -185,6 +189,11 @@ struct damos_watermarks { bool activated; }; =20 +enum damos_counter_type { + DAMOS_NUMBER_ACCESSES, + DAMOS_NUMBER_WRITES, +}; + /** * struct damos - Represents a Data Access Monitoring-based Operation Sche= me. * @min_sz_region: Minimum size of target regions. @@ -223,6 +232,9 @@ struct damos { unsigned long max_sz_region; unsigned int min_nr_accesses; unsigned int max_nr_accesses; + unsigned int min_nr_writes; + unsigned int max_nr_writes; + enum damos_counter_type counter_type; unsigned int min_age_region; unsigned int max_age_region; enum damos_action action; @@ -386,6 +398,7 @@ struct damon_ctx { struct damon_primitive primitive; struct damon_callback callback; =20 + enum damos_counter_type counter_type; unsigned long min_nr_regions; unsigned long max_nr_regions; struct list_head adaptive_targets; diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h index 2f422f4f1fb9..45573f421707 100644 --- a/include/trace/events/damon.h +++ b/include/trace/events/damon.h @@ -11,17 +11,17 @@ =20 TRACE_EVENT(damon_aggregated, =20 - TP_PROTO(struct damon_target *t, struct damon_region *r, + TP_PROTO(struct damon_ctx *c, struct damon_target *t, struct damon_region= *r, unsigned int nr_regions), =20 - TP_ARGS(t, r, nr_regions), + TP_ARGS(c, t, r, nr_regions), =20 TP_STRUCT__entry( __field(unsigned long, target_id) __field(unsigned int, nr_regions) __field(unsigned long, start) __field(unsigned long, end) - __field(unsigned int, nr_accesses) + __field(unsigned int, nr) ), =20 TP_fast_assign( @@ -29,12 +29,12 @@ TRACE_EVENT(damon_aggregated, __entry->nr_regions =3D nr_regions; __entry->start =3D r->ar.start; __entry->end =3D r->ar.end; - __entry->nr_accesses =3D r->nr_accesses; + __entry->nr =3D c->counter_type =3D=3D DAMOS_NUMBER_WRITES ? r->nr_write= s : r->nr_accesses; ), =20 TP_printk("target_id=3D%lu nr_regions=3D%u %lu-%lu: %u", __entry->target_id, __entry->nr_regions, - __entry->start, __entry->end, __entry->nr_accesses) + __entry->start, __entry->end, __entry->nr) ); =20 #endif /* _TRACE_DAMON_H */ diff --git a/mm/damon/core.c b/mm/damon/core.c index e92497895202..e827231366db 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -45,10 +45,12 @@ struct damon_region *damon_new_region(unsigned long sta= rt, unsigned long end) region->ar.start =3D start; region->ar.end =3D end; region->nr_accesses =3D 0; + region->nr_writes =3D 0; INIT_LIST_HEAD(®ion->list); =20 region->age =3D 0; region->last_nr_accesses =3D 0; + region->last_nr_writes =3D 0; =20 return region; } @@ -89,7 +91,7 @@ void damon_destroy_region(struct damon_region *r, struct = damon_target *t) =20 struct damos *damon_new_scheme( unsigned long min_sz_region, unsigned long max_sz_region, - unsigned int min_nr_accesses, unsigned int max_nr_accesses, + unsigned int min_nr, unsigned int max_nr, unsigned int min_age_region, unsigned int max_age_region, enum damos_action action, struct damos_quota *quota, struct damos_watermarks *wmarks) @@ -101,8 +103,10 @@ struct damos *damon_new_scheme( return NULL; scheme->min_sz_region =3D min_sz_region; scheme->max_sz_region =3D max_sz_region; - scheme->min_nr_accesses =3D min_nr_accesses; - scheme->max_nr_accesses =3D max_nr_accesses; + scheme->max_nr_writes =3D max_nr; + scheme->min_nr_writes =3D min_nr; + scheme->min_nr_accesses =3D min_nr; + scheme->max_nr_accesses =3D max_nr; scheme->min_age_region =3D min_age_region; scheme->max_age_region =3D max_age_region; scheme->action =3D action; @@ -221,6 +225,7 @@ struct damon_ctx *damon_new_ctx(void) ctx->sample_interval =3D 5 * 1000; ctx->aggr_interval =3D 100 * 1000; ctx->primitive_update_interval =3D 60 * 1000 * 1000; + ctx->counter_type =3D DAMOS_NUMBER_ACCESSES; =20 ktime_get_coarse_ts64(&ctx->last_aggregation); ctx->last_primitive_update =3D ctx->last_aggregation; @@ -535,9 +540,11 @@ static void kdamond_reset_aggregated(struct damon_ctx = *c) struct damon_region *r; =20 damon_for_each_region(r, t) { - trace_damon_aggregated(t, r, damon_nr_regions(t)); + trace_damon_aggregated(c, t, r, damon_nr_regions(t)); r->last_nr_accesses =3D r->nr_accesses; r->nr_accesses =3D 0; + r->last_nr_writes =3D r->nr_writes; + r->nr_writes =3D 0; } } } @@ -546,21 +553,27 @@ static void damon_split_region_at(struct damon_ctx *c= tx, struct damon_target *t, struct damon_region *r, unsigned long sz_r); =20 -static bool __damos_valid_target(struct damon_region *r, struct damos *s) +static bool __damos_valid_target(struct damon_ctx *ctx, struct damon_regio= n *r, struct damos *s) { unsigned long sz; - sz =3D r->ar.end - r->ar.start; - return s->min_sz_region <=3D sz && sz <=3D s->max_sz_region && - s->min_nr_accesses <=3D r->nr_accesses && - r->nr_accesses <=3D s->max_nr_accesses && - s->min_age_region <=3D r->age && r->age <=3D s->max_age_region; + + if (ctx->counter_type =3D=3D DAMOS_NUMBER_WRITES) + return s->min_sz_region <=3D sz && sz <=3D s->max_sz_region && + s->min_nr_writes <=3D r->nr_writes && + r->nr_writes <=3D s->max_nr_writes && + s->min_age_region <=3D r->age && r->age <=3D s->max_age_region; + else + return s->min_sz_region <=3D sz && sz <=3D s->max_sz_region && + s->min_nr_accesses <=3D r->nr_accesses && + r->nr_accesses <=3D s->max_nr_accesses && + s->min_age_region <=3D r->age && r->age <=3D s->max_age_region; } =20 static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, struct damon_region *r, struct damos *s) { - bool ret =3D __damos_valid_target(r, s); + bool ret =3D __damos_valid_target(c, r, s); =20 if (!ret || !s->quota.esz || !c->primitive.get_scheme_score) return ret; @@ -707,7 +720,7 @@ static void kdamond_apply_schemes(struct damon_ctx *c) memset(quota->histogram, 0, sizeof(quota->histogram)); damon_for_each_target(t, c) { damon_for_each_region(r, t) { - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(c, r, s)) continue; score =3D c->primitive.get_scheme_score( c, t, r, s); @@ -738,13 +751,18 @@ static void kdamond_apply_schemes(struct damon_ctx *c) /* * Merge two adjacent regions into one region */ -static void damon_merge_two_regions(struct damon_target *t, +static void damon_merge_two_regions(struct damon_ctx *c, struct damon_targ= et *t, struct damon_region *l, struct damon_region *r) { unsigned long sz_l =3D sz_damon_region(l), sz_r =3D sz_damon_region(r); =20 - l->nr_accesses =3D (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / + if (c->counter_type =3D=3D DAMOS_NUMBER_WRITES) + l->nr_writes =3D (l->nr_writes * sz_l + r->nr_writes * sz_r) / (sz_l + sz_r); + else + l->nr_accesses =3D (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / + (sz_l + sz_r); + l->age =3D (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); l->ar.end =3D r->ar.end; damon_destroy_region(r, t); @@ -759,23 +777,39 @@ static void damon_merge_two_regions(struct damon_targ= et *t, * thres '->nr_accesses' diff threshold for the merge * sz_limit size upper limit of each region */ -static void damon_merge_regions_of(struct damon_target *t, unsigned int th= res, +static void damon_merge_regions_of(struct damon_ctx *c, struct damon_targe= t *t, unsigned int thres, unsigned long sz_limit) { struct damon_region *r, *prev =3D NULL, *next; =20 - damon_for_each_region_safe(r, next, t) { - if (diff_of(r->nr_accesses, r->last_nr_accesses) > thres) - r->age =3D 0; - else - r->age++; - - if (prev && prev->ar.end =3D=3D r->ar.start && - diff_of(prev->nr_accesses, r->nr_accesses) <=3D thres && - sz_damon_region(prev) + sz_damon_region(r) <=3D sz_limit) - damon_merge_two_regions(t, prev, r); - else - prev =3D r; + if (c->counter_type =3D=3D DAMOS_NUMBER_WRITES) { + damon_for_each_region_safe(r, next, t) { + if (diff_of(r->nr_writes, r->last_nr_writes) > thres) + r->age =3D 0; + else + r->age++; + + if (prev && prev->ar.end =3D=3D r->ar.start && + diff_of(prev->nr_writes, r->nr_writes) <=3D thres && + sz_damon_region(prev) + sz_damon_region(r) <=3D sz_limit) + damon_merge_two_regions(c, t, prev, r); + else + prev =3D r; + } + } else { + damon_for_each_region_safe(r, next, t) { + if (diff_of(r->nr_accesses, r->last_nr_accesses) > thres) + r->age =3D 0; + else + r->age++; + + if (prev && prev->ar.end =3D=3D r->ar.start && + diff_of(prev->nr_accesses, r->nr_accesses) <=3D thres && + sz_damon_region(prev) + sz_damon_region(r) <=3D sz_limit) + damon_merge_two_regions(c, t, prev, r); + else + prev =3D r; + } } } =20 @@ -796,7 +830,7 @@ static void kdamond_merge_regions(struct damon_ctx *c, = unsigned int threshold, struct damon_target *t; =20 damon_for_each_target(t, c) - damon_merge_regions_of(t, threshold, sz_limit); + damon_merge_regions_of(c, t, threshold, sz_limit); } =20 /* @@ -819,6 +853,7 @@ static void damon_split_region_at(struct damon_ctx *ctx, =20 new->age =3D r->age; new->last_nr_accesses =3D r->last_nr_accesses; + new->last_nr_writes =3D r->last_nr_writes; =20 damon_insert_region(new, r, damon_next_region(r), t); } diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index ad65436756af..6cf2501148f5 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -166,6 +166,8 @@ static bool damos_action_valid(int action) case DAMOS_PAGEOUT: case DAMOS_HUGEPAGE: case DAMOS_NOHUGEPAGE: + case DAMOS_MERGEABLE: + case DAMOS_UNMERGEABLE: case DAMOS_STAT: return true; default: @@ -477,6 +479,66 @@ static ssize_t dbgfs_init_regions_read(struct file *fi= le, char __user *buf, return len; } =20 +static ssize_t dbgfs_counter_type_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx =3D file->private_data; + char *kbuf; + ssize_t ret; + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + mutex_unlock(&ctx->kdamond_lock); + ret =3D -EBUSY; + goto out; + } + ret =3D count; + kbuf =3D user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + if (!strncmp(kbuf, "writes\n", count)) + ctx->counter_type =3D DAMOS_NUMBER_WRITES; + else + ctx->counter_type =3D DAMOS_NUMBER_ACCESSES; + + mutex_unlock(&ctx->kdamond_lock); +out: + kfree(kbuf); + return ret; +} + +static ssize_t dbgfs_counter_type_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx =3D file->private_data; + char *kbuf; + ssize_t len; + + kbuf =3D kmalloc(count, GFP_KERNEL | __GFP_NOWARN); + if (!kbuf) + return -ENOMEM; + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + mutex_unlock(&ctx->kdamond_lock); + len =3D -EBUSY; + goto out; + } + + if (ctx->counter_type =3D=3D DAMOS_NUMBER_WRITES) + len =3D scnprintf(kbuf, count, "writes"); + else + len =3D scnprintf(kbuf, count, "accesses"); + mutex_unlock(&ctx->kdamond_lock); + len =3D simple_read_from_buffer(buf, count, ppos, kbuf, len); + +out: + kfree(kbuf); + return len; +} + + static int add_init_region(struct damon_ctx *c, unsigned long target_id, struct damon_addr_range *ar) { @@ -636,12 +698,18 @@ static const struct file_operations kdamond_pid_fops = =3D { .read =3D dbgfs_kdamond_pid_read, }; =20 +static const struct file_operations counter_type_fops =3D { + .open =3D damon_dbgfs_open, + .read =3D dbgfs_counter_type_read, + .write =3D dbgfs_counter_type_write, +}; + static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx) { const char * const file_names[] =3D {"attrs", "schemes", "target_ids", - "init_regions", "kdamond_pid"}; + "init_regions", "kdamond_pid", "counter_type"}; const struct file_operations *fops[] =3D {&attrs_fops, &schemes_fops, - &target_ids_fops, &init_regions_fops, &kdamond_pid_fops}; + &target_ids_fops, &init_regions_fops, &kdamond_pid_fops, &counter_type_f= ops}; int i; =20 for (i =3D 0; i < ARRAY_SIZE(file_names); i++) diff --git a/mm/damon/prmtv-common.c b/mm/damon/prmtv-common.c index 92a04f5831d6..09ba2b5b895e 100644 --- a/mm/damon/prmtv-common.c +++ b/mm/damon/prmtv-common.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include =20 #include "prmtv-common.h" =20 @@ -58,6 +60,92 @@ void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, = unsigned long addr) put_page(page); } =20 +static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long= addr, pte_t pte) +{ + struct page *page; + + if (!pte_write(pte)) + return false; + if (!is_cow_mapping(vma->vm_flags)) + return false; + if (likely(!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))) + return false; + page =3D vm_normal_page(vma, addr, pte); + if (!page) + return false; + return page_maybe_dma_pinned(page); +} + +static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) +{ + pmd_t old, pmd =3D *pmdp; + + if (pmd_present(pmd)) { + /* See comment in change_huge_pmd() */ + old =3D pmdp_invalidate(vma, addr, pmdp); + if (pmd_dirty(old)) + pmd =3D pmd_mkdirty(pmd); + if (pmd_young(old)) + pmd =3D pmd_mkyoung(pmd); + + pmd =3D pmd_wrprotect(pmd); + pmd =3D pmd_clear_soft_dirty(pmd); + + set_pmd_at(vma->vm_mm, addr, pmdp, pmd); + } else if (is_migration_entry(pmd_to_swp_entry(pmd))) { + pmd =3D pmd_swp_clear_soft_dirty(pmd); + set_pmd_at(vma->vm_mm, addr, pmdp, pmd); + } +} + +static inline void clear_soft_dirty(struct vm_area_struct *vma, + unsigned long addr, pte_t *pte) +{ + /* + * The soft-dirty tracker uses #PF-s to catch writes + * to pages, so write-protect the pte as well. See the + * Documentation/admin-guide/mm/soft-dirty.rst for full description + * of how soft-dirty works. + */ + pte_t ptent =3D *pte; + + if (pte_present(ptent)) { + pte_t old_pte; + + if (pte_is_pinned(vma, addr, ptent)) + return; + old_pte =3D ptep_modify_prot_start(vma, addr, pte); + ptent =3D pte_wrprotect(old_pte); + ptent =3D pte_clear_soft_dirty(ptent); + ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); + } else if (is_swap_pte(ptent)) { + ptent =3D pte_swp_clear_soft_dirty(ptent); + set_pte_at(vma->vm_mm, addr, pte, ptent); + } +} + +void damon_pmd_clean_soft_dirty(struct vm_area_struct *vma, unsigned long = addr, + pmd_t *pmd) +{ + vma->vm_flags &=3D ~VM_SOFTDIRTY; + vma_set_page_prot(vma); + + if (pmd_soft_dirty(*pmd)) + clear_soft_dirty_pmd(vma, addr, pmd); + +} + +void damon_ptep_clean_soft_dirty(struct vm_area_struct *vma, unsigned long= addr, + pte_t *pte) +{ + vma->vm_flags &=3D ~VM_SOFTDIRTY; + vma_set_page_prot(vma); + + if (pte_soft_dirty(*pte)) + clear_soft_dirty(vma, addr, pte); +} + void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE diff --git a/mm/damon/prmtv-common.h b/mm/damon/prmtv-common.h index 61f27037603e..03847d149f0e 100644 --- a/mm/damon/prmtv-common.h +++ b/mm/damon/prmtv-common.h @@ -13,6 +13,11 @@ =20 struct page *damon_get_page(unsigned long pfn); =20 +void damon_ptep_clean_soft_dirty(struct vm_area_struct *vma, unsigned long= addr, + pte_t *pte); +void damon_pmd_clean_soft_dirty(struct vm_area_struct *vma, unsigned long = addr, + pmd_t *pmd); + void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr= ); void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr= ); =20 diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 20a9a9d69eb1..a7d9c9563d1d 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -368,6 +368,44 @@ void damon_va_update(struct damon_ctx *ctx) } } =20 +static int damon_clean_soft_dirty_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t *pte; + spinlock_t *ptl; + + if (pmd_huge(*pmd)) { + ptl =3D pmd_lock(walk->mm, pmd); + if (pmd_huge(*pmd)) { + damon_pmd_clean_soft_dirty(walk->vma, addr, pmd); + spin_unlock(ptl); + return 0; + } + spin_unlock(ptl); + } + + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + return 0; + pte =3D pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte_present(*pte)) + goto out; + damon_ptep_clean_soft_dirty(walk->vma, addr, pte); +out: + pte_unmap_unlock(pte, ptl); + return 0; +} + +static const struct mm_walk_ops damon_clean_soft_dirty_ops =3D { + .pmd_entry =3D damon_clean_soft_dirty_pmd_entry, +}; + +static void damon_va_clean_soft_dirty(struct mm_struct *mm, unsigned long = addr) +{ + mmap_read_lock(mm); + walk_page_range(mm, addr, addr + 1, &damon_clean_soft_dirty_ops, NULL); + mmap_read_unlock(mm); +} + static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long next, struct mm_walk *walk) { @@ -415,7 +453,10 @@ static void damon_va_prepare_access_check(struct damon= _ctx *ctx, { r->sampling_addr =3D damon_rand(r->ar.start, r->ar.end); =20 - damon_va_mkold(mm, r->sampling_addr); + if (ctx->counter_type =3D=3D DAMOS_NUMBER_WRITES) + damon_va_clean_soft_dirty(mm, r->sampling_addr); + else + damon_va_mkold(mm, r->sampling_addr); } =20 void damon_va_prepare_access_checks(struct damon_ctx *ctx) @@ -437,6 +478,7 @@ void damon_va_prepare_access_checks(struct damon_ctx *c= tx) struct damon_young_walk_private { unsigned long *page_sz; bool young; + bool dirty; }; =20 static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, @@ -463,6 +505,10 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned = long addr, *priv->page_sz =3D ((1UL) << HPAGE_PMD_SHIFT); priv->young =3D true; } + if (pmd_soft_dirty(*pmd)) { + *priv->page_sz =3D ((1UL) << HPAGE_PMD_SHIFT); + priv->dirty =3D true; + } put_page(page); huge_out: spin_unlock(ptl); @@ -485,6 +531,10 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned = long addr, *priv->page_sz =3D PAGE_SIZE; priv->young =3D true; } + if (pte_soft_dirty(*pte)) { + *priv->page_sz =3D PAGE_SIZE; + priv->dirty =3D true; + } put_page(page); out: pte_unmap_unlock(pte, ptl); @@ -496,16 +546,19 @@ static const struct mm_walk_ops damon_young_ops =3D { }; =20 static bool damon_va_young(struct mm_struct *mm, unsigned long addr, - unsigned long *page_sz) + unsigned long *page_sz, bool *dirty) { struct damon_young_walk_private arg =3D { .page_sz =3D page_sz, .young =3D false, + .dirty =3D false, }; =20 mmap_read_lock(mm); walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg); mmap_read_unlock(mm); + + *dirty =3D arg.dirty; return arg.young; } =20 @@ -522,18 +575,23 @@ static void damon_va_check_access(struct damon_ctx *c= tx, static unsigned long last_addr; static unsigned long last_page_sz =3D PAGE_SIZE; static bool last_accessed; + static bool last_written; =20 /* If the region is in the last checked page, reuse the result */ if (mm =3D=3D last_mm && (ALIGN_DOWN(last_addr, last_page_sz) =3D=3D ALIGN_DOWN(r->sampling_addr, last_page_sz))) { if (last_accessed) r->nr_accesses++; + if (last_written) + r->nr_writes++; return; } =20 - last_accessed =3D damon_va_young(mm, r->sampling_addr, &last_page_sz); + last_accessed =3D damon_va_young(mm, r->sampling_addr, &last_page_sz, &la= st_written); if (last_accessed) r->nr_accesses++; + if (last_written) + r->nr_writes++; =20 last_mm =3D mm; last_addr =3D r->sampling_addr; @@ -544,7 +602,7 @@ unsigned int damon_va_check_accesses(struct damon_ctx *= ctx) struct damon_target *t; struct mm_struct *mm; struct damon_region *r; - unsigned int max_nr_accesses =3D 0; + unsigned int max_nr =3D 0; =20 damon_for_each_target(t, ctx) { mm =3D damon_get_mm(t); @@ -552,12 +610,15 @@ unsigned int damon_va_check_accesses(struct damon_ctx= *ctx) continue; damon_for_each_region(r, t) { damon_va_check_access(ctx, mm, r); - max_nr_accesses =3D max(r->nr_accesses, max_nr_accesses); + if (ctx->counter_type =3D=3D DAMOS_NUMBER_WRITES) + max_nr =3D max(r->nr_writes, max_nr); + else + max_nr =3D max(r->nr_accesses, max_nr); } mmput(mm); } =20 - return max_nr_accesses; + return max_nr; } =20 /* @@ -597,6 +658,7 @@ static int damos_madvise(struct damon_target *target, s= truct damon_region *r, =20 ret =3D do_madvise(mm, PAGE_ALIGN(r->ar.start), PAGE_ALIGN(r->ar.end - r->ar.start), behavior); + mmput(mm); out: return ret; @@ -624,6 +686,12 @@ int damon_va_apply_scheme(struct damon_ctx *ctx, struc= t damon_target *t, case DAMOS_NOHUGEPAGE: madv_action =3D MADV_NOHUGEPAGE; break; + case DAMOS_MERGEABLE: + madv_action =3D MADV_MERGEABLE; + break; + case DAMOS_UNMERGEABLE: + madv_action =3D MADV_UNMERGEABLE; + break; case DAMOS_STAT: return 0; default: --=20 2.25.1