blob: 4dcf852e1e6d8f2e9f0eeca9ee39f620ea972957 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/swap.c
3 *
4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 */
6
7/*
Simon Arlott183ff222007-10-20 01:27:18 +02008 * This file contains the default values for the operation of the
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Linux VM subsystem. Fine-tuning documentation can be found in
10 * Documentation/sysctl/vm.txt.
11 * Started 18.12.91
12 * Swap aging added 23.2.95, Stephen Tweedie.
13 * Buffermem limits added 12.3.98, Rik van Riel.
14 */
15
16#include <linux/mm.h>
17#include <linux/sched.h>
18#include <linux/kernel_stat.h>
19#include <linux/swap.h>
20#include <linux/mman.h>
21#include <linux/pagemap.h>
22#include <linux/pagevec.h>
23#include <linux/init.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040024#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/mm_inline.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/percpu_counter.h>
Dan Williams3565fce2016-01-15 16:56:55 -080027#include <linux/memremap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/percpu.h>
29#include <linux/cpu.h>
30#include <linux/notifier.h>
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -070031#include <linux/backing-dev.h>
Balbir Singh66e17072008-02-07 00:13:56 -080032#include <linux/memcontrol.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090033#include <linux/gfp.h>
Kent Overstreeta27bb332013-05-07 16:19:08 -070034#include <linux/uio.h>
Naoya Horiguchi822fc612015-04-15 16:14:35 -070035#include <linux/hugetlb.h>
Vladimir Davydov33c3fc72015-09-09 15:35:45 -070036#include <linux/page_idle.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Lee Schermerhorn64d65192008-10-18 20:26:52 -070038#include "internal.h"
39
Mel Gormanc6286c92013-07-03 15:02:26 -070040#define CREATE_TRACE_POINTS
41#include <trace/events/pagemap.h>
42
Linus Torvalds1da177e2005-04-16 15:20:36 -070043/* How many pages do we try to swap or page in/out together? */
44int page_cluster;
45
Mel Gorman13f7f782013-07-03 15:02:28 -070046static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
Vegard Nossumf84f95042008-07-23 21:28:14 -070047static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
Minchan Kimcc5993b2015-04-15 16:13:26 -070048static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
Minchan Kim10853a02016-01-15 16:55:11 -080049static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
Ming Lia4a921a2016-05-20 16:57:56 -070050#ifdef CONFIG_SMP
51static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
52#endif
Hisashi Hifumi902aaed2007-10-16 01:24:52 -070053
Adrian Bunkb2213852006-09-25 23:31:02 -070054/*
55 * This path almost never happens for VM activity - pages are normally
56 * freed via pagevecs. But it gets used by networking.
57 */
Harvey Harrison920c7a52008-02-04 22:29:26 -080058static void __page_cache_release(struct page *page)
Adrian Bunkb2213852006-09-25 23:31:02 -070059{
60 if (PageLRU(page)) {
Adrian Bunkb2213852006-09-25 23:31:02 -070061 struct zone *zone = page_zone(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -070062 struct lruvec *lruvec;
63 unsigned long flags;
Adrian Bunkb2213852006-09-25 23:31:02 -070064
Mel Gormana52633d2016-07-28 15:45:28 -070065 spin_lock_irqsave(zone_lru_lock(zone), flags);
Mel Gorman599d0c92016-07-28 15:45:31 -070066 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
Sasha Levin309381fea2014-01-23 15:52:54 -080067 VM_BUG_ON_PAGE(!PageLRU(page), page);
Adrian Bunkb2213852006-09-25 23:31:02 -070068 __ClearPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -070069 del_page_from_lru_list(page, lruvec, page_off_lru(page));
Mel Gormana52633d2016-07-28 15:45:28 -070070 spin_unlock_irqrestore(zone_lru_lock(zone), flags);
Adrian Bunkb2213852006-09-25 23:31:02 -070071 }
Johannes Weiner0a31bc92014-08-08 14:19:22 -070072 mem_cgroup_uncharge(page);
Andrea Arcangeli91807062011-01-13 15:46:32 -080073}
74
75static void __put_single_page(struct page *page)
76{
77 __page_cache_release(page);
Mel Gormanb745bc82014-06-04 16:10:22 -070078 free_hot_cold_page(page, false);
Adrian Bunkb2213852006-09-25 23:31:02 -070079}
80
Andrea Arcangeli91807062011-01-13 15:46:32 -080081static void __put_compound_page(struct page *page)
82{
83 compound_page_dtor *dtor;
84
Naoya Horiguchi822fc612015-04-15 16:14:35 -070085 /*
86 * __page_cache_release() is supposed to be called for thp, not for
87 * hugetlb. This is because hugetlb page does never have PageLRU set
88 * (it's never listed to any LRU lists) and no memcg routines should
89 * be called for hugetlb (it has a separate hugetlb_cgroup.)
90 */
91 if (!PageHuge(page))
92 __page_cache_release(page);
Andrea Arcangeli91807062011-01-13 15:46:32 -080093 dtor = get_compound_page_dtor(page);
94 (*dtor)(page);
95}
96
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -080097void __put_page(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -070098{
Nick Piggin8519fb32006-02-07 12:58:52 -080099 if (unlikely(PageCompound(page)))
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -0800100 __put_compound_page(page);
101 else
Andrea Arcangeli91807062011-01-13 15:46:32 -0800102 __put_single_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103}
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -0800104EXPORT_SYMBOL(__put_page);
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700105
Alexander Zarochentsev1d7ea732006-08-13 23:24:27 -0700106/**
Randy Dunlap76824862008-03-19 17:00:40 -0700107 * put_pages_list() - release a list of pages
108 * @pages: list of pages threaded on page->lru
Alexander Zarochentsev1d7ea732006-08-13 23:24:27 -0700109 *
110 * Release a list of pages which are strung together on page.lru. Currently
111 * used by read_cache_pages() and related error recovery code.
Alexander Zarochentsev1d7ea732006-08-13 23:24:27 -0700112 */
113void put_pages_list(struct list_head *pages)
114{
115 while (!list_empty(pages)) {
116 struct page *victim;
117
118 victim = list_entry(pages->prev, struct page, lru);
119 list_del(&victim->lru);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300120 put_page(victim);
Alexander Zarochentsev1d7ea732006-08-13 23:24:27 -0700121 }
122}
123EXPORT_SYMBOL(put_pages_list);
124
Mel Gorman18022c52012-07-31 16:44:51 -0700125/*
126 * get_kernel_pages() - pin kernel pages in memory
127 * @kiov: An array of struct kvec structures
128 * @nr_segs: number of segments to pin
129 * @write: pinning for read/write, currently ignored
130 * @pages: array that receives pointers to the pages pinned.
131 * Should be at least nr_segs long.
132 *
133 * Returns number of pages pinned. This may be fewer than the number
134 * requested. If nr_pages is 0 or negative, returns 0. If no pages
135 * were pinned, returns -errno. Each page returned must be released
136 * with a put_page() call when it is finished with.
137 */
138int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write,
139 struct page **pages)
140{
141 int seg;
142
143 for (seg = 0; seg < nr_segs; seg++) {
144 if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE))
145 return seg;
146
Mel Gorman5a178112012-07-31 16:45:02 -0700147 pages[seg] = kmap_to_page(kiov[seg].iov_base);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300148 get_page(pages[seg]);
Mel Gorman18022c52012-07-31 16:44:51 -0700149 }
150
151 return seg;
152}
153EXPORT_SYMBOL_GPL(get_kernel_pages);
154
155/*
156 * get_kernel_page() - pin a kernel page in memory
157 * @start: starting kernel address
158 * @write: pinning for read/write, currently ignored
159 * @pages: array that receives pointer to the page pinned.
160 * Must be at least nr_segs long.
161 *
162 * Returns 1 if page is pinned. If the page was not pinned, returns
163 * -errno. The page returned must be released with a put_page() call
164 * when it is finished with.
165 */
166int get_kernel_page(unsigned long start, int write, struct page **pages)
167{
168 const struct kvec kiov = {
169 .iov_base = (void *)start,
170 .iov_len = PAGE_SIZE
171 };
172
173 return get_kernel_pages(&kiov, 1, write, pages);
174}
175EXPORT_SYMBOL_GPL(get_kernel_page);
176
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700177static void pagevec_lru_move_fn(struct pagevec *pvec,
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700178 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),
179 void *arg)
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700180{
181 int i;
Mel Gorman68eb0732016-07-28 15:47:11 -0700182 struct pglist_data *pgdat = NULL;
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700183 struct lruvec *lruvec;
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700184 unsigned long flags = 0;
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700185
186 for (i = 0; i < pagevec_count(pvec); i++) {
187 struct page *page = pvec->pages[i];
Mel Gorman68eb0732016-07-28 15:47:11 -0700188 struct pglist_data *pagepgdat = page_pgdat(page);
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700189
Mel Gorman68eb0732016-07-28 15:47:11 -0700190 if (pagepgdat != pgdat) {
191 if (pgdat)
192 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
193 pgdat = pagepgdat;
194 spin_lock_irqsave(&pgdat->lru_lock, flags);
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700195 }
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700196
Mel Gorman68eb0732016-07-28 15:47:11 -0700197 lruvec = mem_cgroup_page_lruvec(page, pgdat);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700198 (*move_fn)(page, lruvec, arg);
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700199 }
Mel Gorman68eb0732016-07-28 15:47:11 -0700200 if (pgdat)
201 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
Linus Torvalds83896fb2011-01-17 14:42:34 -0800202 release_pages(pvec->pages, pvec->nr, pvec->cold);
203 pagevec_reinit(pvec);
Shaohua Lid8505de2011-01-13 15:47:33 -0800204}
205
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700206static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,
207 void *arg)
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700208{
209 int *pgmoved = arg;
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700210
211 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
212 enum lru_list lru = page_lru_base_type(page);
Johannes Weiner925b7672012-01-12 17:18:15 -0800213 list_move_tail(&page->lru, &lruvec->lists[lru]);
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700214 (*pgmoved)++;
215 }
216}
217
218/*
219 * pagevec_move_tail() must be called with IRQ disabled.
220 * Otherwise this may cause nasty races.
221 */
222static void pagevec_move_tail(struct pagevec *pvec)
223{
224 int pgmoved = 0;
225
226 pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
227 __count_vm_events(PGROTATED, pgmoved);
228}
229
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700230/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 * Writeback is about to end against a page which has been marked for immediate
232 * reclaim. If it still appears to be reclaimable, move it to the tail of the
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700233 * inactive list.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 */
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700235void rotate_reclaimable_page(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236{
Miklos Szerediac6aadb2008-04-28 02:12:38 -0700237 if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700238 !PageUnevictable(page) && PageLRU(page)) {
Miklos Szerediac6aadb2008-04-28 02:12:38 -0700239 struct pagevec *pvec;
240 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300242 get_page(page);
Miklos Szerediac6aadb2008-04-28 02:12:38 -0700243 local_irq_save(flags);
Christoph Lameter7c8e0182014-06-04 16:07:56 -0700244 pvec = this_cpu_ptr(&lru_rotate_pvecs);
Lukasz Odzioba8f182272016-06-24 14:50:01 -0700245 if (!pagevec_add(pvec, page) || PageCompound(page))
Miklos Szerediac6aadb2008-04-28 02:12:38 -0700246 pagevec_move_tail(pvec);
247 local_irq_restore(flags);
248 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249}
250
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700251static void update_page_reclaim_stat(struct lruvec *lruvec,
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800252 int file, int rotated)
253{
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700254 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800255
256 reclaim_stat->recent_scanned[file]++;
257 if (rotated)
258 reclaim_stat->recent_rotated[file]++;
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800259}
260
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700261static void __activate_page(struct page *page, struct lruvec *lruvec,
262 void *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263{
Linus Torvalds7a608572011-01-17 14:42:19 -0800264 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
265 int file = page_is_file_cache(page);
266 int lru = page_lru_base_type(page);
Linus Torvalds7a608572011-01-17 14:42:19 -0800267
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700268 del_page_from_lru_list(page, lruvec, lru);
Linus Torvalds7a608572011-01-17 14:42:19 -0800269 SetPageActive(page);
270 lru += LRU_ACTIVE;
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700271 add_page_to_lru_list(page, lruvec, lru);
Mel Gorman24b7e582014-08-06 16:07:11 -0700272 trace_mm_lru_activate(page);
Linus Torvalds7a608572011-01-17 14:42:19 -0800273
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700274 __count_vm_event(PGACTIVATE);
275 update_page_reclaim_stat(lruvec, file, 1);
Linus Torvalds7a608572011-01-17 14:42:19 -0800276 }
Shaohua Lieb709b02011-05-24 17:12:55 -0700277}
278
279#ifdef CONFIG_SMP
Shaohua Lieb709b02011-05-24 17:12:55 -0700280static void activate_page_drain(int cpu)
281{
282 struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
283
284 if (pagevec_count(pvec))
285 pagevec_lru_move_fn(pvec, __activate_page, NULL);
286}
287
Chris Metcalf5fbc4612013-09-12 15:13:55 -0700288static bool need_activate_page_drain(int cpu)
289{
290 return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0;
291}
292
Shaohua Lieb709b02011-05-24 17:12:55 -0700293void activate_page(struct page *page)
294{
Kirill A. Shutemov800d8c62016-07-26 15:26:18 -0700295 page = compound_head(page);
Shaohua Lieb709b02011-05-24 17:12:55 -0700296 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
297 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
298
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300299 get_page(page);
Lukasz Odzioba8f182272016-06-24 14:50:01 -0700300 if (!pagevec_add(pvec, page) || PageCompound(page))
Shaohua Lieb709b02011-05-24 17:12:55 -0700301 pagevec_lru_move_fn(pvec, __activate_page, NULL);
302 put_cpu_var(activate_page_pvecs);
303 }
304}
305
306#else
307static inline void activate_page_drain(int cpu)
308{
309}
310
Chris Metcalf5fbc4612013-09-12 15:13:55 -0700311static bool need_activate_page_drain(int cpu)
312{
313 return false;
314}
315
Shaohua Lieb709b02011-05-24 17:12:55 -0700316void activate_page(struct page *page)
317{
318 struct zone *zone = page_zone(page);
319
Kirill A. Shutemov800d8c62016-07-26 15:26:18 -0700320 page = compound_head(page);
Mel Gormana52633d2016-07-28 15:45:28 -0700321 spin_lock_irq(zone_lru_lock(zone));
Mel Gorman599d0c92016-07-28 15:45:31 -0700322 __activate_page(page, mem_cgroup_page_lruvec(page, zone->zone_pgdat), NULL);
Mel Gormana52633d2016-07-28 15:45:28 -0700323 spin_unlock_irq(zone_lru_lock(zone));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324}
Shaohua Lieb709b02011-05-24 17:12:55 -0700325#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326
Mel Gorman059285a2013-07-03 15:02:30 -0700327static void __lru_cache_activate_page(struct page *page)
328{
329 struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
330 int i;
331
332 /*
333 * Search backwards on the optimistic assumption that the page being
334 * activated has just been added to this pagevec. Note that only
335 * the local pagevec is examined as a !PageLRU page could be in the
336 * process of being released, reclaimed, migrated or on a remote
337 * pagevec that is currently being drained. Furthermore, marking
338 * a remote pagevec's page PageActive potentially hits a race where
339 * a page is marked PageActive just after it is added to the inactive
340 * list causing accounting errors and BUG_ON checks to trigger.
341 */
342 for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
343 struct page *pagevec_page = pvec->pages[i];
344
345 if (pagevec_page == page) {
346 SetPageActive(page);
347 break;
348 }
349 }
350
351 put_cpu_var(lru_add_pvec);
352}
353
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354/*
355 * Mark a page as having seen activity.
356 *
357 * inactive,unreferenced -> inactive,referenced
358 * inactive,referenced -> active,unreferenced
359 * active,unreferenced -> active,referenced
Hugh Dickinseb39d612014-08-06 16:06:43 -0700360 *
361 * When a newly allocated page is not yet visible, so safe for non-atomic ops,
362 * __SetPageReferenced(page) may be substituted for mark_page_accessed(page).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 */
Harvey Harrison920c7a52008-02-04 22:29:26 -0800364void mark_page_accessed(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365{
Kirill A. Shutemove90309c2016-01-15 16:54:33 -0800366 page = compound_head(page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700367 if (!PageActive(page) && !PageUnevictable(page) &&
Mel Gorman059285a2013-07-03 15:02:30 -0700368 PageReferenced(page)) {
369
370 /*
371 * If the page is on the LRU, queue it for activation via
372 * activate_page_pvecs. Otherwise, assume the page is on a
373 * pagevec, mark it active and it'll be moved to the active
374 * LRU on the next drain.
375 */
376 if (PageLRU(page))
377 activate_page(page);
378 else
379 __lru_cache_activate_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 ClearPageReferenced(page);
Johannes Weinera5289102014-04-03 14:47:51 -0700381 if (page_is_file_cache(page))
382 workingset_activation(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 } else if (!PageReferenced(page)) {
384 SetPageReferenced(page);
385 }
Vladimir Davydov33c3fc72015-09-09 15:35:45 -0700386 if (page_is_idle(page))
387 clear_page_idle(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389EXPORT_SYMBOL(mark_page_accessed);
390
Jianyu Zhan2329d372014-06-04 16:07:31 -0700391static void __lru_cache_add(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392{
Mel Gorman13f7f782013-07-03 15:02:28 -0700393 struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
394
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300395 get_page(page);
Lukasz Odzioba8f182272016-06-24 14:50:01 -0700396 if (!pagevec_add(pvec, page) || PageCompound(page))
Mel Gormana0b8cab32013-07-03 15:02:32 -0700397 __pagevec_lru_add(pvec);
Mel Gorman13f7f782013-07-03 15:02:28 -0700398 put_cpu_var(lru_add_pvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399}
Jianyu Zhan2329d372014-06-04 16:07:31 -0700400
401/**
402 * lru_cache_add: add a page to the page lists
403 * @page: the page to add
404 */
405void lru_cache_add_anon(struct page *page)
406{
Mel Gorman6fb81a12014-06-04 16:10:28 -0700407 if (PageActive(page))
408 ClearPageActive(page);
Jianyu Zhan2329d372014-06-04 16:07:31 -0700409 __lru_cache_add(page);
410}
411
412void lru_cache_add_file(struct page *page)
413{
Mel Gorman6fb81a12014-06-04 16:10:28 -0700414 if (PageActive(page))
415 ClearPageActive(page);
Jianyu Zhan2329d372014-06-04 16:07:31 -0700416 __lru_cache_add(page);
417}
418EXPORT_SYMBOL(lru_cache_add_file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
KOSAKI Motohirof04e9eb2008-10-18 20:26:19 -0700420/**
Mel Gormanc53954a2013-07-03 15:02:34 -0700421 * lru_cache_add - add a page to a page list
KOSAKI Motohirof04e9eb2008-10-18 20:26:19 -0700422 * @page: the page to be added to the LRU.
Jianyu Zhan2329d372014-06-04 16:07:31 -0700423 *
424 * Queue the page for addition to the LRU via pagevec. The decision on whether
425 * to add the page to the [in]active [file|anon] list is deferred until the
426 * pagevec is drained. This gives a chance for the caller of lru_cache_add()
427 * have the page added to the active list using mark_page_accessed().
KOSAKI Motohirof04e9eb2008-10-18 20:26:19 -0700428 */
Mel Gormanc53954a2013-07-03 15:02:34 -0700429void lru_cache_add(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430{
Sasha Levin309381fea2014-01-23 15:52:54 -0800431 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
432 VM_BUG_ON_PAGE(PageLRU(page), page);
Mel Gormanc53954a2013-07-03 15:02:34 -0700433 __lru_cache_add(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434}
435
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700436/**
437 * add_page_to_unevictable_list - add a page to the unevictable list
438 * @page: the page to be added to the unevictable list
439 *
440 * Add page directly to its zone's unevictable list. To avoid races with
441 * tasks that might be making the page evictable, through eg. munlock,
442 * munmap or exit, while it's not on the lru, we want to add the page
443 * while it's locked or otherwise "invisible" to other tasks. This is
444 * difficult to do when using the pagevec cache, so bypass that.
445 */
446void add_page_to_unevictable_list(struct page *page)
447{
Mel Gorman599d0c92016-07-28 15:45:31 -0700448 struct pglist_data *pgdat = page_pgdat(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700449 struct lruvec *lruvec;
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700450
Mel Gorman599d0c92016-07-28 15:45:31 -0700451 spin_lock_irq(&pgdat->lru_lock);
452 lruvec = mem_cgroup_page_lruvec(page, pgdat);
Naoya Horiguchief2a2cb2013-07-31 13:53:37 -0700453 ClearPageActive(page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700454 SetPageUnevictable(page);
455 SetPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700456 add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE);
Mel Gorman599d0c92016-07-28 15:45:31 -0700457 spin_unlock_irq(&pgdat->lru_lock);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700458}
459
Johannes Weiner00501b52014-08-08 14:19:20 -0700460/**
461 * lru_cache_add_active_or_unevictable
462 * @page: the page to be added to LRU
463 * @vma: vma in which page is mapped for determining reclaimability
464 *
465 * Place @page on the active or unevictable LRU list, depending on its
466 * evictability. Note that if the page is not evictable, it goes
467 * directly back onto it's zone's unevictable list, it does NOT use a
468 * per cpu pagevec.
469 */
470void lru_cache_add_active_or_unevictable(struct page *page,
471 struct vm_area_struct *vma)
472{
473 VM_BUG_ON_PAGE(PageLRU(page), page);
474
475 if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) {
476 SetPageActive(page);
477 lru_cache_add(page);
478 return;
479 }
480
481 if (!TestSetPageMlocked(page)) {
482 /*
483 * We use the irq-unsafe __mod_zone_page_stat because this
484 * counter is not modified from interrupt context, and the pte
485 * lock is held(spinlock), which implies preemption disabled.
486 */
487 __mod_zone_page_state(page_zone(page), NR_MLOCK,
488 hpage_nr_pages(page));
489 count_vm_event(UNEVICTABLE_PGMLOCKED);
490 }
491 add_page_to_unevictable_list(page);
492}
493
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700494/*
Minchan Kim31560182011-03-22 16:32:52 -0700495 * If the page can not be invalidated, it is moved to the
496 * inactive list to speed up its reclaim. It is moved to the
497 * head of the list, rather than the tail, to give the flusher
498 * threads some time to write it out, as this is much more
499 * effective than the single-page writeout from reclaim.
Minchan Kim278df9f2011-03-22 16:32:54 -0700500 *
501 * If the page isn't page_mapped and dirty/writeback, the page
502 * could reclaim asap using PG_reclaim.
503 *
504 * 1. active, mapped page -> none
505 * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
506 * 3. inactive, mapped page -> none
507 * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
508 * 5. inactive, clean -> inactive, tail
509 * 6. Others -> none
510 *
511 * In 4, why it moves inactive's head, the VM expects the page would
512 * be write it out by flusher threads as this is much more effective
513 * than the single-page writeout from reclaim.
Minchan Kim31560182011-03-22 16:32:52 -0700514 */
Minchan Kimcc5993b2015-04-15 16:13:26 -0700515static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700516 void *arg)
Minchan Kim31560182011-03-22 16:32:52 -0700517{
518 int lru, file;
Minchan Kim278df9f2011-03-22 16:32:54 -0700519 bool active;
Minchan Kim31560182011-03-22 16:32:52 -0700520
Minchan Kim278df9f2011-03-22 16:32:54 -0700521 if (!PageLRU(page))
Minchan Kim31560182011-03-22 16:32:52 -0700522 return;
523
Minchan Kimbad49d92011-05-11 15:13:30 -0700524 if (PageUnevictable(page))
525 return;
526
Minchan Kim31560182011-03-22 16:32:52 -0700527 /* Some processes are using the page */
528 if (page_mapped(page))
529 return;
530
Minchan Kim278df9f2011-03-22 16:32:54 -0700531 active = PageActive(page);
Minchan Kim31560182011-03-22 16:32:52 -0700532 file = page_is_file_cache(page);
533 lru = page_lru_base_type(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700534
535 del_page_from_lru_list(page, lruvec, lru + active);
Minchan Kim31560182011-03-22 16:32:52 -0700536 ClearPageActive(page);
537 ClearPageReferenced(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700538 add_page_to_lru_list(page, lruvec, lru);
Minchan Kim31560182011-03-22 16:32:52 -0700539
Minchan Kim278df9f2011-03-22 16:32:54 -0700540 if (PageWriteback(page) || PageDirty(page)) {
541 /*
542 * PG_reclaim could be raced with end_page_writeback
543 * It can make readahead confusing. But race window
544 * is _really_ small and it's non-critical problem.
545 */
546 SetPageReclaim(page);
547 } else {
548 /*
549 * The page's writeback ends up during pagevec
550 * We moves tha page into tail of inactive.
551 */
Johannes Weiner925b7672012-01-12 17:18:15 -0800552 list_move_tail(&page->lru, &lruvec->lists[lru]);
Minchan Kim278df9f2011-03-22 16:32:54 -0700553 __count_vm_event(PGROTATED);
554 }
555
556 if (active)
557 __count_vm_event(PGDEACTIVATE);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700558 update_page_reclaim_stat(lruvec, file, 0);
Minchan Kim31560182011-03-22 16:32:52 -0700559}
560
Minchan Kim10853a02016-01-15 16:55:11 -0800561
562static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
563 void *arg)
564{
565 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
566 int file = page_is_file_cache(page);
567 int lru = page_lru_base_type(page);
568
569 del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE);
570 ClearPageActive(page);
571 ClearPageReferenced(page);
572 add_page_to_lru_list(page, lruvec, lru);
573
574 __count_vm_event(PGDEACTIVATE);
575 update_page_reclaim_stat(lruvec, file, 0);
576 }
577}
578
Minchan Kim31560182011-03-22 16:32:52 -0700579/*
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700580 * Drain pages out of the cpu's pagevecs.
581 * Either "cpu" is the current CPU, and preemption has already been
582 * disabled; or "cpu" is being hot-unplugged, and is already dead.
583 */
Konstantin Khlebnikovf0cb3c72012-03-21 16:34:06 -0700584void lru_add_drain_cpu(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585{
Mel Gorman13f7f782013-07-03 15:02:28 -0700586 struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587
Mel Gorman13f7f782013-07-03 15:02:28 -0700588 if (pagevec_count(pvec))
Mel Gormana0b8cab32013-07-03 15:02:32 -0700589 __pagevec_lru_add(pvec);
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700590
591 pvec = &per_cpu(lru_rotate_pvecs, cpu);
592 if (pagevec_count(pvec)) {
593 unsigned long flags;
594
595 /* No harm done if a racing interrupt already did this */
596 local_irq_save(flags);
597 pagevec_move_tail(pvec);
598 local_irq_restore(flags);
599 }
Minchan Kim31560182011-03-22 16:32:52 -0700600
Minchan Kimcc5993b2015-04-15 16:13:26 -0700601 pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
Minchan Kim31560182011-03-22 16:32:52 -0700602 if (pagevec_count(pvec))
Minchan Kimcc5993b2015-04-15 16:13:26 -0700603 pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
Shaohua Lieb709b02011-05-24 17:12:55 -0700604
Minchan Kim10853a02016-01-15 16:55:11 -0800605 pvec = &per_cpu(lru_deactivate_pvecs, cpu);
606 if (pagevec_count(pvec))
607 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
608
Shaohua Lieb709b02011-05-24 17:12:55 -0700609 activate_page_drain(cpu);
Minchan Kim31560182011-03-22 16:32:52 -0700610}
611
612/**
Minchan Kimcc5993b2015-04-15 16:13:26 -0700613 * deactivate_file_page - forcefully deactivate a file page
Minchan Kim31560182011-03-22 16:32:52 -0700614 * @page: page to deactivate
615 *
616 * This function hints the VM that @page is a good reclaim candidate,
617 * for example if its invalidation fails due to the page being dirty
618 * or under writeback.
619 */
Minchan Kimcc5993b2015-04-15 16:13:26 -0700620void deactivate_file_page(struct page *page)
Minchan Kim31560182011-03-22 16:32:52 -0700621{
Minchan Kim821ed6b2011-05-24 17:12:31 -0700622 /*
Minchan Kimcc5993b2015-04-15 16:13:26 -0700623 * In a workload with many unevictable page such as mprotect,
624 * unevictable page deactivation for accelerating reclaim is pointless.
Minchan Kim821ed6b2011-05-24 17:12:31 -0700625 */
626 if (PageUnevictable(page))
627 return;
628
Minchan Kim31560182011-03-22 16:32:52 -0700629 if (likely(get_page_unless_zero(page))) {
Minchan Kimcc5993b2015-04-15 16:13:26 -0700630 struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs);
Minchan Kim31560182011-03-22 16:32:52 -0700631
Lukasz Odzioba8f182272016-06-24 14:50:01 -0700632 if (!pagevec_add(pvec, page) || PageCompound(page))
Minchan Kimcc5993b2015-04-15 16:13:26 -0700633 pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
634 put_cpu_var(lru_deactivate_file_pvecs);
Minchan Kim31560182011-03-22 16:32:52 -0700635 }
Andrew Morton80bfed92006-01-06 00:11:14 -0800636}
637
Minchan Kim10853a02016-01-15 16:55:11 -0800638/**
639 * deactivate_page - deactivate a page
640 * @page: page to deactivate
641 *
642 * deactivate_page() moves @page to the inactive list if @page was on the active
643 * list and was not an unevictable page. This is done to accelerate the reclaim
644 * of @page.
645 */
646void deactivate_page(struct page *page)
647{
648 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
649 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
650
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300651 get_page(page);
Lukasz Odzioba8f182272016-06-24 14:50:01 -0700652 if (!pagevec_add(pvec, page) || PageCompound(page))
Minchan Kim10853a02016-01-15 16:55:11 -0800653 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
654 put_cpu_var(lru_deactivate_pvecs);
655 }
656}
657
Andrew Morton80bfed92006-01-06 00:11:14 -0800658void lru_add_drain(void)
659{
Konstantin Khlebnikovf0cb3c72012-03-21 16:34:06 -0700660 lru_add_drain_cpu(get_cpu());
Andrew Morton80bfed92006-01-06 00:11:14 -0800661 put_cpu();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662}
663
David Howellsc4028952006-11-22 14:57:56 +0000664static void lru_add_drain_per_cpu(struct work_struct *dummy)
Nick Piggin053837f2006-01-18 17:42:27 -0800665{
666 lru_add_drain();
667}
668
Chris Metcalf5fbc4612013-09-12 15:13:55 -0700669static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
670
Wang Sheng-Huif3a932ba2016-06-08 15:33:56 -0700671/*
672 * lru_add_drain_wq is used to do lru_add_drain_all() from a WQ_MEM_RECLAIM
673 * workqueue, aiding in getting memory freed.
674 */
675static struct workqueue_struct *lru_add_drain_wq;
676
677static int __init lru_init(void)
678{
679 lru_add_drain_wq = alloc_workqueue("lru-add-drain", WQ_MEM_RECLAIM, 0);
680
681 if (WARN(!lru_add_drain_wq,
682 "Failed to create workqueue lru_add_drain_wq"))
683 return -ENOMEM;
684
685 return 0;
686}
687early_initcall(lru_init);
688
Chris Metcalf5fbc4612013-09-12 15:13:55 -0700689void lru_add_drain_all(void)
Nick Piggin053837f2006-01-18 17:42:27 -0800690{
Chris Metcalf5fbc4612013-09-12 15:13:55 -0700691 static DEFINE_MUTEX(lock);
692 static struct cpumask has_work;
693 int cpu;
694
695 mutex_lock(&lock);
696 get_online_cpus();
697 cpumask_clear(&has_work);
698
699 for_each_online_cpu(cpu) {
700 struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
701
702 if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
703 pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
Minchan Kimcc5993b2015-04-15 16:13:26 -0700704 pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
Minchan Kim10853a02016-01-15 16:55:11 -0800705 pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
Chris Metcalf5fbc4612013-09-12 15:13:55 -0700706 need_activate_page_drain(cpu)) {
707 INIT_WORK(work, lru_add_drain_per_cpu);
Wang Sheng-Huif3a932ba2016-06-08 15:33:56 -0700708 queue_work_on(cpu, lru_add_drain_wq, work);
Chris Metcalf5fbc4612013-09-12 15:13:55 -0700709 cpumask_set_cpu(cpu, &has_work);
710 }
711 }
712
713 for_each_cpu(cpu, &has_work)
714 flush_work(&per_cpu(lru_add_drain_work, cpu));
715
716 put_online_cpus();
717 mutex_unlock(&lock);
Nick Piggin053837f2006-01-18 17:42:27 -0800718}
719
Michal Hockoaabfb572014-10-09 15:28:52 -0700720/**
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +0300721 * release_pages - batched put_page()
Michal Hockoaabfb572014-10-09 15:28:52 -0700722 * @pages: array of pages to release
723 * @nr: number of pages
724 * @cold: whether the pages are cache cold
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 *
Michal Hockoaabfb572014-10-09 15:28:52 -0700726 * Decrement the reference count on all the pages in @pages. If it
727 * fell to zero, remove the page from the LRU and free it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 */
Mel Gormanb745bc82014-06-04 16:10:22 -0700729void release_pages(struct page **pages, int nr, bool cold)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730{
731 int i;
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -0800732 LIST_HEAD(pages_to_free);
Mel Gorman599d0c92016-07-28 15:45:31 -0700733 struct pglist_data *locked_pgdat = NULL;
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700734 struct lruvec *lruvec;
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700735 unsigned long uninitialized_var(flags);
Michal Hockoaabfb572014-10-09 15:28:52 -0700736 unsigned int uninitialized_var(lock_batch);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 for (i = 0; i < nr; i++) {
739 struct page *page = pages[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740
Michal Hockoaabfb572014-10-09 15:28:52 -0700741 /*
742 * Make sure the IRQ-safe lock-holding time does not get
743 * excessive with a continuous string of pages from the
Mel Gorman599d0c92016-07-28 15:45:31 -0700744 * same pgdat. The lock is held only if pgdat != NULL.
Michal Hockoaabfb572014-10-09 15:28:52 -0700745 */
Mel Gorman599d0c92016-07-28 15:45:31 -0700746 if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) {
747 spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
748 locked_pgdat = NULL;
Michal Hockoaabfb572014-10-09 15:28:52 -0700749 }
750
Aaron Lu6fcb52a2016-10-07 17:00:08 -0700751 if (is_huge_zero_page(page))
Kirill A. Shutemovaa88b682016-04-28 16:18:27 -0700752 continue;
Kirill A. Shutemovaa88b682016-04-28 16:18:27 -0700753
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -0800754 page = compound_head(page);
Nick Pigginb5810032005-10-29 18:16:12 -0700755 if (!put_page_testzero(page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 continue;
757
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -0800758 if (PageCompound(page)) {
Mel Gorman599d0c92016-07-28 15:45:31 -0700759 if (locked_pgdat) {
760 spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
761 locked_pgdat = NULL;
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -0800762 }
763 __put_compound_page(page);
764 continue;
765 }
766
Nick Piggin46453a62006-03-22 00:07:58 -0800767 if (PageLRU(page)) {
Mel Gorman599d0c92016-07-28 15:45:31 -0700768 struct pglist_data *pgdat = page_pgdat(page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700769
Mel Gorman599d0c92016-07-28 15:45:31 -0700770 if (pgdat != locked_pgdat) {
771 if (locked_pgdat)
772 spin_unlock_irqrestore(&locked_pgdat->lru_lock,
Hisashi Hifumi902aaed2007-10-16 01:24:52 -0700773 flags);
Michal Hockoaabfb572014-10-09 15:28:52 -0700774 lock_batch = 0;
Mel Gorman599d0c92016-07-28 15:45:31 -0700775 locked_pgdat = pgdat;
776 spin_lock_irqsave(&locked_pgdat->lru_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 }
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700778
Mel Gorman599d0c92016-07-28 15:45:31 -0700779 lruvec = mem_cgroup_page_lruvec(page, locked_pgdat);
Sasha Levin309381fea2014-01-23 15:52:54 -0800780 VM_BUG_ON_PAGE(!PageLRU(page), page);
Nick Piggin67453912006-03-22 00:08:00 -0800781 __ClearPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700782 del_page_from_lru_list(page, lruvec, page_off_lru(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 }
Nick Piggin46453a62006-03-22 00:07:58 -0800784
Mel Gormanc53954a2013-07-03 15:02:34 -0700785 /* Clear Active bit in case of parallel mark_page_accessed */
Mel Gormane3741b52014-06-04 16:10:26 -0700786 __ClearPageActive(page);
Mel Gormanc53954a2013-07-03 15:02:34 -0700787
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -0800788 list_add(&page->lru, &pages_to_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 }
Mel Gorman599d0c92016-07-28 15:45:31 -0700790 if (locked_pgdat)
791 spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792
Johannes Weiner747db952014-08-08 14:19:24 -0700793 mem_cgroup_uncharge_list(&pages_to_free);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -0800794 free_hot_cold_page_list(&pages_to_free, cold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795}
Miklos Szeredi0be85572010-10-27 15:34:46 -0700796EXPORT_SYMBOL(release_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
798/*
799 * The pages which we're about to release may be in the deferred lru-addition
800 * queues. That would prevent them from really being freed right now. That's
801 * OK from a correctness point of view but is inefficient - those pages may be
802 * cache-warm and we want to give them back to the page allocator ASAP.
803 *
804 * So __pagevec_release() will drain those queues here. __pagevec_lru_add()
805 * and __pagevec_lru_add_active() call release_pages() directly to avoid
806 * mutual recursion.
807 */
808void __pagevec_release(struct pagevec *pvec)
809{
810 lru_add_drain();
811 release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
812 pagevec_reinit(pvec);
813}
Steve French7f285702005-11-01 10:22:55 -0800814EXPORT_SYMBOL(__pagevec_release);
815
Hugh Dickins12d27102012-01-12 17:19:52 -0800816#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800817/* used by __split_huge_page_refcount() */
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700818void lru_add_page_tail(struct page *page, struct page *page_tail,
Shaohua Li5bc7b8a2013-04-29 15:08:36 -0700819 struct lruvec *lruvec, struct list_head *list)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800820{
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800821 const int file = 0;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800822
Sasha Levin309381fea2014-01-23 15:52:54 -0800823 VM_BUG_ON_PAGE(!PageHead(page), page);
824 VM_BUG_ON_PAGE(PageCompound(page_tail), page);
825 VM_BUG_ON_PAGE(PageLRU(page_tail), page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700826 VM_BUG_ON(NR_CPUS != 1 &&
Mel Gorman599d0c92016-07-28 15:45:31 -0700827 !spin_is_locked(&lruvec_pgdat(lruvec)->lru_lock));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800828
Shaohua Li5bc7b8a2013-04-29 15:08:36 -0700829 if (!list)
830 SetPageLRU(page_tail);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800831
Hugh Dickins12d27102012-01-12 17:19:52 -0800832 if (likely(PageLRU(page)))
833 list_add_tail(&page_tail->lru, &page->lru);
Shaohua Li5bc7b8a2013-04-29 15:08:36 -0700834 else if (list) {
835 /* page reclaim is reclaiming a huge page */
836 get_page(page_tail);
837 list_add_tail(&page_tail->lru, list);
838 } else {
Hugh Dickins12d27102012-01-12 17:19:52 -0800839 struct list_head *list_head;
840 /*
841 * Head page has not yet been counted, as an hpage,
842 * so we must account for each subpage individually.
843 *
844 * Use the standard add function to put page_tail on the list,
845 * but then correct its position so they all end up in order.
846 */
Kirill A. Shutemove180cf82013-07-31 13:53:39 -0700847 add_page_to_lru_list(page_tail, lruvec, page_lru(page_tail));
Hugh Dickins12d27102012-01-12 17:19:52 -0800848 list_head = page_tail->lru.prev;
849 list_move_tail(&page_tail->lru, list_head);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800850 }
Hugh Dickins75121022012-03-05 14:59:18 -0800851
852 if (!PageUnevictable(page))
Kirill A. Shutemove180cf82013-07-31 13:53:39 -0700853 update_page_reclaim_stat(lruvec, file, PageActive(page_tail));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800854}
Hugh Dickins12d27102012-01-12 17:19:52 -0800855#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800856
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700857static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
858 void *arg)
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700859{
Mel Gorman13f7f782013-07-03 15:02:28 -0700860 int file = page_is_file_cache(page);
861 int active = PageActive(page);
862 enum lru_list lru = page_lru(page);
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700863
Sasha Levin309381fea2014-01-23 15:52:54 -0800864 VM_BUG_ON_PAGE(PageLRU(page), page);
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700865
866 SetPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700867 add_page_to_lru_list(page, lruvec, lru);
868 update_page_reclaim_stat(lruvec, file, active);
Mel Gorman24b7e582014-08-06 16:07:11 -0700869 trace_mm_lru_insertion(page, lru);
Shaohua Li3dd7ae82011-03-22 16:33:45 -0700870}
871
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873 * Add the passed pages to the LRU, then drop the caller's refcount
874 * on them. Reinitialises the caller's pagevec.
875 */
Mel Gormana0b8cab32013-07-03 15:02:32 -0700876void __pagevec_lru_add(struct pagevec *pvec)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877{
Mel Gormana0b8cab32013-07-03 15:02:32 -0700878 pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879}
Hugh Dickins5095ae832012-01-12 17:19:58 -0800880EXPORT_SYMBOL(__pagevec_lru_add);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882/**
Johannes Weiner0cd61442014-04-03 14:47:46 -0700883 * pagevec_lookup_entries - gang pagecache lookup
884 * @pvec: Where the resulting entries are placed
885 * @mapping: The address_space to search
886 * @start: The starting entry index
887 * @nr_entries: The maximum number of entries
888 * @indices: The cache indices corresponding to the entries in @pvec
889 *
890 * pagevec_lookup_entries() will search for and return a group of up
891 * to @nr_entries pages and shadow entries in the mapping. All
892 * entries are placed in @pvec. pagevec_lookup_entries() takes a
893 * reference against actual pages in @pvec.
894 *
895 * The search returns a group of mapping-contiguous entries with
896 * ascending indexes. There may be holes in the indices due to
897 * not-present entries.
898 *
899 * pagevec_lookup_entries() returns the number of entries which were
900 * found.
901 */
902unsigned pagevec_lookup_entries(struct pagevec *pvec,
903 struct address_space *mapping,
904 pgoff_t start, unsigned nr_pages,
905 pgoff_t *indices)
906{
907 pvec->nr = find_get_entries(mapping, start, nr_pages,
908 pvec->pages, indices);
909 return pagevec_count(pvec);
910}
911
912/**
913 * pagevec_remove_exceptionals - pagevec exceptionals pruning
914 * @pvec: The pagevec to prune
915 *
916 * pagevec_lookup_entries() fills both pages and exceptional radix
917 * tree entries into the pagevec. This function prunes all
918 * exceptionals from @pvec without leaving holes, so that it can be
919 * passed on to page-only pagevec operations.
920 */
921void pagevec_remove_exceptionals(struct pagevec *pvec)
922{
923 int i, j;
924
925 for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
926 struct page *page = pvec->pages[i];
927 if (!radix_tree_exceptional_entry(page))
928 pvec->pages[j++] = page;
929 }
930 pvec->nr = j;
931}
932
933/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934 * pagevec_lookup - gang pagecache lookup
935 * @pvec: Where the resulting pages are placed
936 * @mapping: The address_space to search
937 * @start: The starting page index
938 * @nr_pages: The maximum number of pages
939 *
940 * pagevec_lookup() will search for and return a group of up to @nr_pages pages
941 * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a
942 * reference against the pages in @pvec.
943 *
944 * The search returns a group of mapping-contiguous pages with ascending
945 * indexes. There may be holes in the indices due to not-present pages.
946 *
947 * pagevec_lookup() returns the number of pages which were found.
948 */
949unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
950 pgoff_t start, unsigned nr_pages)
951{
952 pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
953 return pagevec_count(pvec);
954}
Christoph Hellwig78539fd2006-01-11 20:47:41 +1100955EXPORT_SYMBOL(pagevec_lookup);
956
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
958 pgoff_t *index, int tag, unsigned nr_pages)
959{
960 pvec->nr = find_get_pages_tag(mapping, index, tag,
961 nr_pages, pvec->pages);
962 return pagevec_count(pvec);
963}
Steve French7f285702005-11-01 10:22:55 -0800964EXPORT_SYMBOL(pagevec_lookup_tag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966/*
967 * Perform any setup for the swap system
968 */
969void __init swap_setup(void)
970{
Jan Beulich44813742009-09-21 17:03:05 -0700971 unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700972#ifdef CONFIG_SWAP
Shaohua Li33806f02013-02-22 16:34:37 -0800973 int i;
974
Kirill A. Shutemov27ba0642015-02-10 14:09:59 -0800975 for (i = 0; i < MAX_SWAPFILES; i++)
Shaohua Li33806f02013-02-22 16:34:37 -0800976 spin_lock_init(&swapper_spaces[i].tree_lock);
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700977#endif
978
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 /* Use a smaller cluster for small-memory machines */
980 if (megs < 16)
981 page_cluster = 2;
982 else
983 page_cluster = 3;
984 /*
985 * Right now other parts of the system means that we
986 * _really_ don't want to cluster much more
987 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988}