mm: process reclaim: vmpressure based process reclaim
With this patch, anon pages of inactive tasks can be reclaimed,
depending on memory pressure. Memory pressure is detected
using vmpressure events. 'N' best tasks in terms of anon
size is selected and pages proportional to their tasksize
is reclaimed. The total number of pages reclaimed at each
run of the swap work, can be tuned from userspace, the
default being SWAP_CLUSTER_MAX * 32.
The patch also adds tracepoints to debug and tune the
feature.
echo 1 > /sys/module/process_reclaim/parameters/enable_process_reclaim
to enable the feature.
echo <pages> > /sys/module/process_reclaim/parameters/per_swap_size,
to set the number of pages reclaimed in each scan.
/sys/module/process_reclaim/parameters/reclaim_avg_efficiency, provides
the average efficiency (scan to reclaim ratio) of the algorithm.
/sys/module/process_reclaim/parameters/swap_eff_win, to set the window
period (in unit of number of times reclaim is triggered) to detect
low efficiency runs.
/sys/module/process_reclaim/parameters/swap_opt_eff, to set the optimal
efficiency threshold for low efficiency detection.
Change-Id: I895986f10c997d1715761eaaadc4bbbee60db9d2
Signed-off-by: Vinayak Menon <vinmenon@codeaurora.org>
diff --git a/mm/Makefile b/mm/Makefile
index a7aa9efe..a5b60d8 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -100,3 +100,4 @@
obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o
obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o
obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o
+obj-$(CONFIG_PROCESS_RECLAIM) += process_reclaim.o
diff --git a/mm/process_reclaim.c b/mm/process_reclaim.c
new file mode 100644
index 0000000..adba473
--- /dev/null
+++ b/mm/process_reclaim.c
@@ -0,0 +1,258 @@
+/*
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/swap.h>
+#include <linux/sort.h>
+#include <linux/oom.h>
+#include <linux/sched.h>
+#include <linux/rcupdate.h>
+#include <linux/notifier.h>
+#include <linux/vmpressure.h>
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/process_reclaim.h>
+
+#define MAX_SWAP_TASKS SWAP_CLUSTER_MAX
+
+static void swap_fn(struct work_struct *work);
+DECLARE_WORK(swap_work, swap_fn);
+
+/* User knob to enable/disable process reclaim feature */
+static int enable_process_reclaim;
+module_param_named(enable_process_reclaim, enable_process_reclaim, int, 0644);
+
+/* The max number of pages tried to be reclaimed in a single run */
+int per_swap_size = SWAP_CLUSTER_MAX * 32;
+module_param_named(per_swap_size, per_swap_size, int, 0644);
+
+int reclaim_avg_efficiency;
+module_param_named(reclaim_avg_efficiency, reclaim_avg_efficiency, int, 0444);
+
+/* The vmpressure region where process reclaim operates */
+static unsigned long pressure_min = 50;
+static unsigned long pressure_max = 90;
+module_param_named(pressure_min, pressure_min, ulong, 0644);
+module_param_named(pressure_max, pressure_max, ulong, 0644);
+
+/*
+ * Scheduling process reclaim workqueue unecessarily
+ * when the reclaim efficiency is low does not make
+ * sense. We try to detect a drop in efficiency and
+ * disable reclaim for a time period. This period and the
+ * period for which we monitor a drop in efficiency is
+ * defined by swap_eff_win. swap_opt_eff is the optimal
+ * efficincy used as theshold for this.
+ */
+static int swap_eff_win = 2;
+module_param_named(swap_eff_win, swap_eff_win, int, 0644);
+
+static int swap_opt_eff = 50;
+module_param_named(swap_opt_eff, swap_opt_eff, int, 0644);
+
+static atomic_t skip_reclaim = ATOMIC_INIT(0);
+/* Not atomic since only a single instance of swap_fn run at a time */
+static int monitor_eff;
+
+struct selected_task {
+ struct task_struct *p;
+ int tasksize;
+ short oom_score_adj;
+};
+
+int selected_cmp(const void *a, const void *b)
+{
+ const struct selected_task *x = a;
+ const struct selected_task *y = b;
+ int ret;
+
+ ret = x->tasksize < y->tasksize ? -1 : 1;
+
+ return ret;
+}
+
+static int test_task_flag(struct task_struct *p, int flag)
+{
+ struct task_struct *t = p;
+
+ do {
+ task_lock(t);
+ if (test_tsk_thread_flag(t, flag)) {
+ task_unlock(t);
+ return 1;
+ }
+ task_unlock(t);
+ } while_each_thread(p, t);
+
+ return 0;
+}
+
+static void swap_fn(struct work_struct *work)
+{
+ struct task_struct *tsk;
+ struct reclaim_param rp;
+
+ /* Pick the best MAX_SWAP_TASKS tasks in terms of anon size */
+ struct selected_task selected[MAX_SWAP_TASKS] = {{0, 0, 0},};
+ int si = 0;
+ int i;
+ int tasksize;
+ int total_sz = 0;
+ short min_score_adj = 360;
+ int total_scan = 0;
+ int total_reclaimed = 0;
+ int nr_to_reclaim;
+ int efficiency;
+
+ rcu_read_lock();
+ for_each_process(tsk) {
+ struct task_struct *p;
+ short oom_score_adj;
+
+ if (tsk->flags & PF_KTHREAD)
+ continue;
+
+ /* if task no longer has any memory ignore it */
+ if (test_task_flag(tsk, TIF_MM_RELEASED))
+ continue;
+
+ if (test_task_flag(tsk, TIF_MEMDIE))
+ continue;
+
+ p = find_lock_task_mm(tsk);
+ if (!p)
+ continue;
+
+ oom_score_adj = p->signal->oom_score_adj;
+ if (oom_score_adj < min_score_adj) {
+ task_unlock(p);
+ continue;
+ }
+
+ tasksize = get_mm_counter(p->mm, MM_ANONPAGES);
+ task_unlock(p);
+
+ if (tasksize <= 0)
+ continue;
+
+ if (si == MAX_SWAP_TASKS) {
+ sort(&selected[0], MAX_SWAP_TASKS,
+ sizeof(struct selected_task),
+ &selected_cmp, NULL);
+ if (tasksize < selected[0].tasksize)
+ continue;
+ selected[0].p = p;
+ selected[0].oom_score_adj = oom_score_adj;
+ selected[0].tasksize = tasksize;
+ } else {
+ selected[si].p = p;
+ selected[si].oom_score_adj = oom_score_adj;
+ selected[si].tasksize = tasksize;
+ si++;
+ }
+ }
+
+ for (i = 0; i < si; i++) {
+ get_task_struct(selected[i].p);
+ total_sz += selected[i].tasksize;
+ }
+
+ rcu_read_unlock();
+
+ /* Skip reclaim if total size is too less */
+ if (total_sz < SWAP_CLUSTER_MAX) {
+ for (i = 0; i < si; i++)
+ put_task_struct(selected[i].p);
+ return;
+ }
+
+ while (si--) {
+ nr_to_reclaim =
+ (selected[si].tasksize * per_swap_size) / total_sz;
+ /* scan atleast a page */
+ if (!nr_to_reclaim)
+ nr_to_reclaim = 1;
+
+ if ((test_task_flag(selected[si].p, TIF_MM_RELEASED))
+ || (test_task_flag(selected[si].p, TIF_MEMDIE))) {
+ put_task_struct(selected[si].p);
+ continue;
+ }
+
+ rp = reclaim_task_anon(selected[si].p, nr_to_reclaim);
+
+ trace_process_reclaim(selected[si].tasksize,
+ selected[si].oom_score_adj, rp.nr_scanned,
+ rp.nr_reclaimed, per_swap_size, total_sz,
+ nr_to_reclaim);
+ total_scan += rp.nr_scanned;
+ total_reclaimed += rp.nr_reclaimed;
+ put_task_struct(selected[si].p);
+ }
+
+ if (total_scan) {
+ efficiency = (total_reclaimed * 100) / total_scan;
+
+ if (efficiency < swap_opt_eff) {
+ if (++monitor_eff == swap_eff_win) {
+ atomic_set(&skip_reclaim, swap_eff_win + 1);
+ monitor_eff = 0;
+ }
+ } else {
+ monitor_eff = 0;
+ }
+
+ reclaim_avg_efficiency =
+ (efficiency + reclaim_avg_efficiency) / 2;
+ trace_process_reclaim_eff(efficiency, reclaim_avg_efficiency);
+ }
+}
+
+static int vmpressure_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ unsigned long pressure = action;
+
+ if (!enable_process_reclaim)
+ return 0;
+
+ if (!current_is_kswapd())
+ return 0;
+
+ if (!atomic_dec_and_test(&skip_reclaim))
+ return 0;
+
+ if ((pressure >= pressure_min) && (pressure < pressure_max))
+ if (!work_pending(&swap_work))
+ schedule_work(&swap_work);
+ return 0;
+}
+
+static struct notifier_block vmpr_nb = {
+ .notifier_call = vmpressure_notifier,
+};
+
+static int __init process_reclaim_init(void)
+{
+ vmpressure_notifier_register(&vmpr_nb);
+ return 0;
+}
+
+static void __exit process_reclaim_exit(void)
+{
+ vmpressure_notifier_unregister(&vmpr_nb);
+}
+
+module_init(process_reclaim_init);
+module_exit(process_reclaim_exit);