blob: e53f7d02c17cd682c27f0bef1275c416586a9c26 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * mm/thrash.c
3 *
4 * Copyright (C) 2004, Red Hat, Inc.
5 * Copyright (C) 2004, Rik van Riel <riel@redhat.com>
6 * Released under the GPL, see the file COPYING for details.
7 *
8 * Simple token based thrashing protection, using the algorithm
KOSAKI Motohiroe21c7ff2011-07-25 17:12:06 -07009 * described in: http://www.cse.ohio-state.edu/hpcs/WWW/HTML/publications/abs05-1.html
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080010 *
11 * Sep 2006, Ashwin Chaugule <ashwin.chaugule@celunite.com>
12 * Improved algorithm to pass token:
13 * Each task has a priority which is incremented if it contended
14 * for the token in an interval less than its previous attempt.
15 * If the token is acquired, that task's priority is boosted to prevent
16 * the token from bouncing around too often and to let the task make
17 * some progress in its execution.
Linus Torvalds1da177e2005-04-16 15:20:36 -070018 */
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080019
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/jiffies.h>
21#include <linux/mm.h>
22#include <linux/sched.h>
23#include <linux/swap.h>
KOSAKI Motohiroa4336582011-06-15 15:08:13 -070024#include <linux/memcontrol.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -070026#include <trace/events/vmscan.h>
27
KOSAKI Motohirod7911ef32011-06-15 15:08:15 -070028#define TOKEN_AGING_INTERVAL (0xFF)
29
Linus Torvalds1da177e2005-04-16 15:20:36 -070030static DEFINE_SPINLOCK(swap_token_lock);
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080031struct mm_struct *swap_token_mm;
KOSAKI Motohiroa4336582011-06-15 15:08:13 -070032struct mem_cgroup *swap_token_memcg;
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
KOSAKI Motohiroa4336582011-06-15 15:08:13 -070034#ifdef CONFIG_CGROUP_MEM_RES_CTLR
35static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm)
36{
37 struct mem_cgroup *memcg;
38
39 memcg = try_get_mem_cgroup_from_mm(mm);
40 if (memcg)
41 css_put(mem_cgroup_css(memcg));
42
43 return memcg;
44}
45#else
46static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm)
47{
48 return NULL;
49}
50#endif
51
Hugh Dickinsa5c9b692009-06-23 12:36:58 -070052void grab_swap_token(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -070053{
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080054 int current_interval;
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -070055 unsigned int old_prio = mm->token_priority;
KOSAKI Motohiro53bb01f2011-07-25 17:12:07 -070056 static unsigned int global_faults;
57 static unsigned int last_aging;
Linus Torvalds1da177e2005-04-16 15:20:36 -070058
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080059 global_faults++;
60
Hugh Dickinsa5c9b692009-06-23 12:36:58 -070061 current_interval = global_faults - mm->faultstamp;
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080062
63 if (!spin_trylock(&swap_token_lock))
Linus Torvalds1da177e2005-04-16 15:20:36 -070064 return;
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080065
66 /* First come first served */
KOSAKI Motohiroa4336582011-06-15 15:08:13 -070067 if (!swap_token_mm)
68 goto replace_token;
69
KOSAKI Motohiro45ebb842011-07-25 17:12:08 -070070 /*
71 * Usually, we don't need priority aging because long interval faults
72 * makes priority decrease quickly. But there is one exception. If the
73 * token owner task is sleeping, it never make long interval faults.
74 * Thus, we need a priority aging mechanism instead. The requirements
75 * of priority aging are
76 * 1) An aging interval is reasonable enough long. Too short aging
77 * interval makes quick swap token lost and decrease performance.
78 * 2) The swap token owner task have to get priority aging even if
79 * it's under sleep.
80 */
KOSAKI Motohirod7911ef32011-06-15 15:08:15 -070081 if ((global_faults - last_aging) > TOKEN_AGING_INTERVAL) {
82 swap_token_mm->token_priority /= 2;
83 last_aging = global_faults;
84 }
85
KOSAKI Motohiroa4336582011-06-15 15:08:13 -070086 if (mm == swap_token_mm) {
87 mm->token_priority += 2;
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -070088 goto update_priority;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 }
90
KOSAKI Motohiroa4336582011-06-15 15:08:13 -070091 if (current_interval < mm->last_interval)
92 mm->token_priority++;
93 else {
94 if (likely(mm->token_priority > 0))
95 mm->token_priority--;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 }
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -080097
KOSAKI Motohiroa4336582011-06-15 15:08:13 -070098 /* Check if we deserve the token */
99 if (mm->token_priority > swap_token_mm->token_priority)
100 goto replace_token;
101
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -0700102update_priority:
KOSAKI Motohirod7911ef32011-06-15 15:08:15 -0700103 trace_update_swap_token_priority(mm, old_prio, swap_token_mm);
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -0700104
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -0800105out:
Hugh Dickinsa5c9b692009-06-23 12:36:58 -0700106 mm->faultstamp = global_faults;
107 mm->last_interval = current_interval;
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -0800108 spin_unlock(&swap_token_lock);
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700109 return;
110
111replace_token:
112 mm->token_priority += 2;
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -0700113 trace_replace_swap_token(swap_token_mm, mm);
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700114 swap_token_mm = mm;
115 swap_token_memcg = swap_token_memcg_from_mm(mm);
KOSAKI Motohirod7911ef32011-06-15 15:08:15 -0700116 last_aging = global_faults;
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700117 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118}
119
120/* Called on process exit. */
121void __put_swap_token(struct mm_struct *mm)
122{
123 spin_lock(&swap_token_lock);
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700124 if (likely(mm == swap_token_mm)) {
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -0700125 trace_put_swap_token(swap_token_mm);
Ashwin Chaugule7602bdf2006-12-06 20:31:57 -0800126 swap_token_mm = NULL;
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700127 swap_token_memcg = NULL;
128 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 spin_unlock(&swap_token_lock);
130}
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700131
132static bool match_memcg(struct mem_cgroup *a, struct mem_cgroup *b)
133{
134 if (!a)
135 return true;
136 if (!b)
137 return true;
138 if (a == b)
139 return true;
140 return false;
141}
142
143void disable_swap_token(struct mem_cgroup *memcg)
144{
145 /* memcg reclaim don't disable unrelated mm token. */
146 if (match_memcg(memcg, swap_token_memcg)) {
147 spin_lock(&swap_token_lock);
148 if (match_memcg(memcg, swap_token_memcg)) {
KOSAKI Motohiro83cd81a2011-06-15 15:08:14 -0700149 trace_disable_swap_token(swap_token_mm);
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700150 swap_token_mm = NULL;
151 swap_token_memcg = NULL;
152 }
153 spin_unlock(&swap_token_lock);
154 }
155}