Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * mm/thrash.c |
| 3 | * |
| 4 | * Copyright (C) 2004, Red Hat, Inc. |
| 5 | * Copyright (C) 2004, Rik van Riel <riel@redhat.com> |
| 6 | * Released under the GPL, see the file COPYING for details. |
| 7 | * |
| 8 | * Simple token based thrashing protection, using the algorithm |
| 9 | * described in: http://www.cs.wm.edu/~sjiang/token.pdf |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 10 | * |
| 11 | * Sep 2006, Ashwin Chaugule <ashwin.chaugule@celunite.com> |
| 12 | * Improved algorithm to pass token: |
| 13 | * Each task has a priority which is incremented if it contended |
| 14 | * for the token in an interval less than its previous attempt. |
| 15 | * If the token is acquired, that task's priority is boosted to prevent |
| 16 | * the token from bouncing around too often and to let the task make |
| 17 | * some progress in its execution. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | */ |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 19 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <linux/jiffies.h> |
| 21 | #include <linux/mm.h> |
| 22 | #include <linux/sched.h> |
| 23 | #include <linux/swap.h> |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 24 | #include <linux/memcontrol.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
| 26 | static DEFINE_SPINLOCK(swap_token_lock); |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 27 | struct mm_struct *swap_token_mm; |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 28 | struct mem_cgroup *swap_token_memcg; |
Adrian Bunk | e305005 | 2006-12-06 20:32:43 -0800 | [diff] [blame] | 29 | static unsigned int global_faults; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 31 | #ifdef CONFIG_CGROUP_MEM_RES_CTLR |
| 32 | static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm) |
| 33 | { |
| 34 | struct mem_cgroup *memcg; |
| 35 | |
| 36 | memcg = try_get_mem_cgroup_from_mm(mm); |
| 37 | if (memcg) |
| 38 | css_put(mem_cgroup_css(memcg)); |
| 39 | |
| 40 | return memcg; |
| 41 | } |
| 42 | #else |
| 43 | static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm) |
| 44 | { |
| 45 | return NULL; |
| 46 | } |
| 47 | #endif |
| 48 | |
Hugh Dickins | a5c9b69 | 2009-06-23 12:36:58 -0700 | [diff] [blame] | 49 | void grab_swap_token(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 50 | { |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 51 | int current_interval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 52 | |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 53 | global_faults++; |
| 54 | |
Hugh Dickins | a5c9b69 | 2009-06-23 12:36:58 -0700 | [diff] [blame] | 55 | current_interval = global_faults - mm->faultstamp; |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 56 | |
| 57 | if (!spin_trylock(&swap_token_lock)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 58 | return; |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 59 | |
| 60 | /* First come first served */ |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 61 | if (!swap_token_mm) |
| 62 | goto replace_token; |
| 63 | |
| 64 | if (mm == swap_token_mm) { |
| 65 | mm->token_priority += 2; |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 66 | goto out; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | } |
| 68 | |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 69 | if (current_interval < mm->last_interval) |
| 70 | mm->token_priority++; |
| 71 | else { |
| 72 | if (likely(mm->token_priority > 0)) |
| 73 | mm->token_priority--; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | } |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 75 | |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 76 | /* Check if we deserve the token */ |
| 77 | if (mm->token_priority > swap_token_mm->token_priority) |
| 78 | goto replace_token; |
| 79 | |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 80 | out: |
Hugh Dickins | a5c9b69 | 2009-06-23 12:36:58 -0700 | [diff] [blame] | 81 | mm->faultstamp = global_faults; |
| 82 | mm->last_interval = current_interval; |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 83 | spin_unlock(&swap_token_lock); |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 84 | return; |
| 85 | |
| 86 | replace_token: |
| 87 | mm->token_priority += 2; |
| 88 | swap_token_mm = mm; |
| 89 | swap_token_memcg = swap_token_memcg_from_mm(mm); |
| 90 | goto out; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 91 | } |
| 92 | |
| 93 | /* Called on process exit. */ |
| 94 | void __put_swap_token(struct mm_struct *mm) |
| 95 | { |
| 96 | spin_lock(&swap_token_lock); |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 97 | if (likely(mm == swap_token_mm)) { |
Ashwin Chaugule | 7602bdf | 2006-12-06 20:31:57 -0800 | [diff] [blame] | 98 | swap_token_mm = NULL; |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 99 | swap_token_memcg = NULL; |
| 100 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | spin_unlock(&swap_token_lock); |
| 102 | } |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame^] | 103 | |
| 104 | static bool match_memcg(struct mem_cgroup *a, struct mem_cgroup *b) |
| 105 | { |
| 106 | if (!a) |
| 107 | return true; |
| 108 | if (!b) |
| 109 | return true; |
| 110 | if (a == b) |
| 111 | return true; |
| 112 | return false; |
| 113 | } |
| 114 | |
| 115 | void disable_swap_token(struct mem_cgroup *memcg) |
| 116 | { |
| 117 | /* memcg reclaim don't disable unrelated mm token. */ |
| 118 | if (match_memcg(memcg, swap_token_memcg)) { |
| 119 | spin_lock(&swap_token_lock); |
| 120 | if (match_memcg(memcg, swap_token_memcg)) { |
| 121 | swap_token_mm = NULL; |
| 122 | swap_token_memcg = NULL; |
| 123 | } |
| 124 | spin_unlock(&swap_token_lock); |
| 125 | } |
| 126 | } |