blob: c585e7e65c3795233da2b88be7b71367399ec20b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#include <linux/mm.h>
Davidlohr Bueso615d6e82014-04-07 15:37:25 -07002#include <linux/vmacache.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07003#include <linux/hugetlb.h>
Dave Hansen22e057c2011-03-22 16:33:00 -07004#include <linux/huge_mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/mount.h>
6#include <linux/seq_file.h>
Mauricio Line070ad42005-09-03 15:55:10 -07007#include <linux/highmem.h>
Kees Cook5096add2007-05-08 00:26:04 -07008#include <linux/ptrace.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09009#include <linux/slab.h>
Christoph Lameter6e21c8f2005-09-03 15:54:45 -070010#include <linux/pagemap.h>
11#include <linux/mempolicy.h>
Dave Hansen22e057c2011-03-22 16:33:00 -070012#include <linux/rmap.h>
Matt Mackall85863e42008-02-04 22:29:04 -080013#include <linux/swap.h>
14#include <linux/swapops.h>
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -070015#include <linux/mmu_notifier.h>
Vladimir Davydov33c3fc72015-09-09 15:35:45 -070016#include <linux/page_idle.h>
Vlastimil Babka6a15a372016-01-14 15:19:20 -080017#include <linux/shmem_fs.h>
Minchan Kimff926892013-05-09 16:21:24 +090018#include <linux/mm_inline.h>
Minchan Kim01fdd712013-05-09 16:21:28 +090019#include <linux/ctype.h>
Mauricio Line070ad42005-09-03 15:55:10 -070020
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/elf.h>
22#include <asm/uaccess.h>
Mauricio Line070ad42005-09-03 15:55:10 -070023#include <asm/tlbflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include "internal.h"
25
Eric W. Biedermandf5f8312008-02-08 04:18:33 -080026void task_mem(struct seq_file *m, struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -070027{
Konstantin Khlebnikov84638332016-01-14 15:22:07 -080028 unsigned long text, lib, swap, ptes, pmds, anon, file, shmem;
Hugh Dickins365e9c872005-10-29 18:16:18 -070029 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
30
Jerome Marchand8cee8522016-01-14 15:19:29 -080031 anon = get_mm_counter(mm, MM_ANONPAGES);
32 file = get_mm_counter(mm, MM_FILEPAGES);
33 shmem = get_mm_counter(mm, MM_SHMEMPAGES);
34
Hugh Dickins365e9c872005-10-29 18:16:18 -070035 /*
36 * Note: to minimize their overhead, mm maintains hiwater_vm and
37 * hiwater_rss only when about to *lower* total_vm or rss. Any
38 * collector of these hiwater stats must therefore get total_vm
39 * and rss too, which will usually be the higher. Barriers? not
40 * worth the effort, such snapshots can always be inconsistent.
41 */
42 hiwater_vm = total_vm = mm->total_vm;
43 if (hiwater_vm < mm->hiwater_vm)
44 hiwater_vm = mm->hiwater_vm;
Jerome Marchand8cee8522016-01-14 15:19:29 -080045 hiwater_rss = total_rss = anon + file + shmem;
Hugh Dickins365e9c872005-10-29 18:16:18 -070046 if (hiwater_rss < mm->hiwater_rss)
47 hiwater_rss = mm->hiwater_rss;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
50 lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080051 swap = get_mm_counter(mm, MM_SWAPENTS);
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -080052 ptes = PTRS_PER_PTE * sizeof(pte_t) * atomic_long_read(&mm->nr_ptes);
53 pmds = PTRS_PER_PMD * sizeof(pmd_t) * mm_nr_pmds(mm);
Eric W. Biedermandf5f8312008-02-08 04:18:33 -080054 seq_printf(m,
Hugh Dickins365e9c872005-10-29 18:16:18 -070055 "VmPeak:\t%8lu kB\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 "VmSize:\t%8lu kB\n"
57 "VmLck:\t%8lu kB\n"
Christoph Lameterbc3e53f2011-10-31 17:07:30 -070058 "VmPin:\t%8lu kB\n"
Hugh Dickins365e9c872005-10-29 18:16:18 -070059 "VmHWM:\t%8lu kB\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070060 "VmRSS:\t%8lu kB\n"
Jerome Marchand8cee8522016-01-14 15:19:29 -080061 "RssAnon:\t%8lu kB\n"
62 "RssFile:\t%8lu kB\n"
63 "RssShmem:\t%8lu kB\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070064 "VmData:\t%8lu kB\n"
65 "VmStk:\t%8lu kB\n"
66 "VmExe:\t%8lu kB\n"
67 "VmLib:\t%8lu kB\n"
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080068 "VmPTE:\t%8lu kB\n"
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -080069 "VmPMD:\t%8lu kB\n"
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080070 "VmSwap:\t%8lu kB\n",
Hugh Dickins365e9c872005-10-29 18:16:18 -070071 hiwater_vm << (PAGE_SHIFT-10),
Konstantin Khlebnikov314e51b2012-10-08 16:29:02 -070072 total_vm << (PAGE_SHIFT-10),
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 mm->locked_vm << (PAGE_SHIFT-10),
Christoph Lameterbc3e53f2011-10-31 17:07:30 -070074 mm->pinned_vm << (PAGE_SHIFT-10),
Hugh Dickins365e9c872005-10-29 18:16:18 -070075 hiwater_rss << (PAGE_SHIFT-10),
76 total_rss << (PAGE_SHIFT-10),
Jerome Marchand8cee8522016-01-14 15:19:29 -080077 anon << (PAGE_SHIFT-10),
78 file << (PAGE_SHIFT-10),
79 shmem << (PAGE_SHIFT-10),
Konstantin Khlebnikov84638332016-01-14 15:22:07 -080080 mm->data_vm << (PAGE_SHIFT-10),
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 mm->stack_vm << (PAGE_SHIFT-10), text, lib,
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -080082 ptes >> 10,
83 pmds >> 10,
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080084 swap << (PAGE_SHIFT-10));
Naoya Horiguchi5d317b22015-11-05 18:47:14 -080085 hugetlb_report_usage(m, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -070086}
87
88unsigned long task_vsize(struct mm_struct *mm)
89{
90 return PAGE_SIZE * mm->total_vm;
91}
92
Alexey Dobriyana2ade7b2011-01-12 17:00:32 -080093unsigned long task_statm(struct mm_struct *mm,
94 unsigned long *shared, unsigned long *text,
95 unsigned long *data, unsigned long *resident)
Linus Torvalds1da177e2005-04-16 15:20:36 -070096{
Jerome Marchandeca56ff2016-01-14 15:19:26 -080097 *shared = get_mm_counter(mm, MM_FILEPAGES) +
98 get_mm_counter(mm, MM_SHMEMPAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
100 >> PAGE_SHIFT;
Konstantin Khlebnikov84638332016-01-14 15:22:07 -0800101 *data = mm->data_vm + mm->stack_vm;
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -0800102 *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 return mm->total_vm;
104}
105
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900106#ifdef CONFIG_NUMA
107/*
Oleg Nesterov498f2372014-10-09 15:27:52 -0700108 * Save get_task_policy() for show_numa_map().
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900109 */
110static void hold_task_mempolicy(struct proc_maps_private *priv)
111{
112 struct task_struct *task = priv->task;
113
114 task_lock(task);
Oleg Nesterov498f2372014-10-09 15:27:52 -0700115 priv->task_mempolicy = get_task_policy(task);
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900116 mpol_get(priv->task_mempolicy);
117 task_unlock(task);
118}
119static void release_task_mempolicy(struct proc_maps_private *priv)
120{
121 mpol_put(priv->task_mempolicy);
122}
123#else
124static void hold_task_mempolicy(struct proc_maps_private *priv)
125{
126}
127static void release_task_mempolicy(struct proc_maps_private *priv)
128{
129}
130#endif
131
Colin Cross3e4578f2015-10-27 16:42:08 -0700132static void seq_print_vma_name(struct seq_file *m, struct vm_area_struct *vma)
133{
134 const char __user *name = vma_get_anon_name(vma);
135 struct mm_struct *mm = vma->vm_mm;
136
137 unsigned long page_start_vaddr;
138 unsigned long page_offset;
139 unsigned long num_pages;
140 unsigned long max_len = NAME_MAX;
141 int i;
142
143 page_start_vaddr = (unsigned long)name & PAGE_MASK;
144 page_offset = (unsigned long)name - page_start_vaddr;
145 num_pages = DIV_ROUND_UP(page_offset + max_len, PAGE_SIZE);
146
147 seq_puts(m, "[anon:");
148
149 for (i = 0; i < num_pages; i++) {
150 int len;
151 int write_len;
152 const char *kaddr;
153 long pages_pinned;
154 struct page *page;
155
156 pages_pinned = get_user_pages_remote(current, mm,
157 page_start_vaddr, 1, 0, &page, NULL);
158 if (pages_pinned < 1) {
159 seq_puts(m, "<fault>]");
160 return;
161 }
162
163 kaddr = (const char *)kmap(page);
164 len = min(max_len, PAGE_SIZE - page_offset);
165 write_len = strnlen(kaddr + page_offset, len);
166 seq_write(m, kaddr + page_offset, write_len);
167 kunmap(page);
168 put_page(page);
169
170 /* if strnlen hit a null terminator then we're done */
171 if (write_len != len)
172 break;
173
174 max_len -= len;
175 page_offset = 0;
176 page_start_vaddr += PAGE_SIZE;
177 }
178
179 seq_putc(m, ']');
180}
181
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700182static void vma_stop(struct proc_maps_private *priv)
Mauricio Line070ad42005-09-03 15:55:10 -0700183{
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700184 struct mm_struct *mm = priv->mm;
185
186 release_task_mempolicy(priv);
187 up_read(&mm->mmap_sem);
188 mmput(mm);
Matt Mackalla6198792008-02-04 22:29:03 -0800189}
190
Oleg Nesterovad2a00e2014-10-09 15:25:39 -0700191static struct vm_area_struct *
192m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma)
193{
194 if (vma == priv->tail_vma)
195 return NULL;
196 return vma->vm_next ?: priv->tail_vma;
197}
198
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700199static void m_cache_vma(struct seq_file *m, struct vm_area_struct *vma)
200{
201 if (m->count < m->size) /* vma is copied successfully */
Robert Ho855af072016-10-07 17:02:36 -0700202 m->version = m_next_vma(m->private, vma) ? vma->vm_end : -1UL;
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700203}
204
Oleg Nesterov0c255322014-10-09 15:25:36 -0700205static void *m_start(struct seq_file *m, loff_t *ppos)
Matt Mackalla6198792008-02-04 22:29:03 -0800206{
207 struct proc_maps_private *priv = m->private;
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700208 unsigned long last_addr = m->version;
Matt Mackalla6198792008-02-04 22:29:03 -0800209 struct mm_struct *mm;
Oleg Nesterov0c255322014-10-09 15:25:36 -0700210 struct vm_area_struct *vma;
211 unsigned int pos = *ppos;
Matt Mackalla6198792008-02-04 22:29:03 -0800212
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700213 /* See m_cache_vma(). Zero at the start or after lseek. */
214 if (last_addr == -1UL)
215 return NULL;
216
Oleg Nesterov2c033762014-10-09 15:25:51 -0700217 priv->task = get_proc_task(priv->inode);
Matt Mackalla6198792008-02-04 22:29:03 -0800218 if (!priv->task)
Al Viroec6fd8a2011-02-15 22:22:54 -0500219 return ERR_PTR(-ESRCH);
Matt Mackalla6198792008-02-04 22:29:03 -0800220
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700221 mm = priv->mm;
222 if (!mm || !atomic_inc_not_zero(&mm->mm_users))
223 return NULL;
Oleg Nesterov0c255322014-10-09 15:25:36 -0700224
Oleg Nesterov00f89d22009-07-10 03:27:38 +0200225 down_read(&mm->mmap_sem);
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900226 hold_task_mempolicy(priv);
Oleg Nesterov0c255322014-10-09 15:25:36 -0700227 priv->tail_vma = get_gate_vma(mm);
Matt Mackalla6198792008-02-04 22:29:03 -0800228
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700229 if (last_addr) {
Robert Ho855af072016-10-07 17:02:36 -0700230 vma = find_vma(mm, last_addr - 1);
231 if (vma && vma->vm_start <= last_addr)
232 vma = m_next_vma(priv, vma);
233 if (vma)
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700234 return vma;
235 }
236
237 m->version = 0;
Oleg Nesterov0c255322014-10-09 15:25:36 -0700238 if (pos < mm->map_count) {
Oleg Nesterov557c2d82014-10-09 15:25:43 -0700239 for (vma = mm->mmap; pos; pos--) {
240 m->version = vma->vm_start;
Matt Mackalla6198792008-02-04 22:29:03 -0800241 vma = vma->vm_next;
Oleg Nesterov557c2d82014-10-09 15:25:43 -0700242 }
Oleg Nesterov0c255322014-10-09 15:25:36 -0700243 return vma;
Matt Mackalla6198792008-02-04 22:29:03 -0800244 }
245
Oleg Nesterov557c2d82014-10-09 15:25:43 -0700246 /* we do not bother to update m->version in this case */
Oleg Nesterov0c255322014-10-09 15:25:36 -0700247 if (pos == mm->map_count && priv->tail_vma)
248 return priv->tail_vma;
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700249
250 vma_stop(priv);
251 return NULL;
Matt Mackalla6198792008-02-04 22:29:03 -0800252}
253
254static void *m_next(struct seq_file *m, void *v, loff_t *pos)
255{
256 struct proc_maps_private *priv = m->private;
Oleg Nesterovad2a00e2014-10-09 15:25:39 -0700257 struct vm_area_struct *next;
Matt Mackalla6198792008-02-04 22:29:03 -0800258
259 (*pos)++;
Oleg Nesterovad2a00e2014-10-09 15:25:39 -0700260 next = m_next_vma(priv, v);
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700261 if (!next)
262 vma_stop(priv);
263 return next;
Matt Mackalla6198792008-02-04 22:29:03 -0800264}
265
266static void m_stop(struct seq_file *m, void *v)
267{
268 struct proc_maps_private *priv = m->private;
Matt Mackalla6198792008-02-04 22:29:03 -0800269
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700270 if (!IS_ERR_OR_NULL(v))
271 vma_stop(priv);
Oleg Nesterov0d5f5f42014-10-09 15:25:32 -0700272 if (priv->task) {
Matt Mackalla6198792008-02-04 22:29:03 -0800273 put_task_struct(priv->task);
Oleg Nesterov0d5f5f42014-10-09 15:25:32 -0700274 priv->task = NULL;
275 }
Matt Mackalla6198792008-02-04 22:29:03 -0800276}
277
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -0700278static int proc_maps_open(struct inode *inode, struct file *file,
279 const struct seq_operations *ops, int psize)
280{
281 struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
282
283 if (!priv)
284 return -ENOMEM;
285
Oleg Nesterov2c033762014-10-09 15:25:51 -0700286 priv->inode = inode;
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700287 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
288 if (IS_ERR(priv->mm)) {
289 int err = PTR_ERR(priv->mm);
290
291 seq_release_private(inode, file);
292 return err;
293 }
294
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -0700295 return 0;
296}
297
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700298static int proc_map_release(struct inode *inode, struct file *file)
299{
300 struct seq_file *seq = file->private_data;
301 struct proc_maps_private *priv = seq->private;
302
303 if (priv->mm)
304 mmdrop(priv->mm);
305
306 return seq_release_private(inode, file);
307}
308
Matt Mackalla6198792008-02-04 22:29:03 -0800309static int do_maps_open(struct inode *inode, struct file *file,
Jan Engelhardt03a44822008-02-08 04:21:19 -0800310 const struct seq_operations *ops)
Matt Mackalla6198792008-02-04 22:29:03 -0800311{
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -0700312 return proc_maps_open(inode, file, ops,
313 sizeof(struct proc_maps_private));
Matt Mackalla6198792008-02-04 22:29:03 -0800314}
Mauricio Line070ad42005-09-03 15:55:10 -0700315
Johannes Weiner65376df2016-02-02 16:57:29 -0800316/*
317 * Indicate if the VMA is a stack for the given task; for
318 * /proc/PID/maps that is the stack of the main task.
319 */
320static int is_stack(struct proc_maps_private *priv,
Andy Lutomirskib18cb642016-09-30 10:58:57 -0700321 struct vm_area_struct *vma)
Oleg Nesterov58cb6542014-10-09 15:25:54 -0700322{
Andy Lutomirskib18cb642016-09-30 10:58:57 -0700323 /*
324 * We make no effort to guess what a given thread considers to be
325 * its "stack". It's not even well-defined for programs written
326 * languages like Go.
327 */
328 return vma->vm_start <= vma->vm_mm->start_stack &&
329 vma->vm_end >= vma->vm_mm->start_stack;
Oleg Nesterov58cb6542014-10-09 15:25:54 -0700330}
331
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700332static void
333show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334{
Mauricio Line070ad42005-09-03 15:55:10 -0700335 struct mm_struct *mm = vma->vm_mm;
336 struct file *file = vma->vm_file;
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700337 struct proc_maps_private *priv = m->private;
KOSAKI Motohiroca16d142011-05-26 19:16:19 +0900338 vm_flags_t flags = vma->vm_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 unsigned long ino = 0;
KAMEZAWA Hiroyuki6260a4b2009-04-06 19:00:30 -0700340 unsigned long long pgoff = 0;
Mikulas Patockaa09a79f2011-05-09 13:01:09 +0200341 unsigned long start, end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 dev_t dev = 0;
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700343 const char *name = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344
345 if (file) {
Al Viro496ad9a2013-01-23 17:07:38 -0500346 struct inode *inode = file_inode(vma->vm_file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 dev = inode->i_sb->s_dev;
348 ino = inode->i_ino;
KAMEZAWA Hiroyuki6260a4b2009-04-06 19:00:30 -0700349 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 }
351
Linus Torvaldsd7824372010-08-15 11:35:52 -0700352 /* We don't show the stack guard page in /proc/maps */
353 start = vma->vm_start;
Mikulas Patockaa09a79f2011-05-09 13:01:09 +0200354 end = vma->vm_end;
Linus Torvaldsd7824372010-08-15 11:35:52 -0700355
Tetsuo Handa652586d2013-11-14 14:31:57 -0800356 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
357 seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu ",
Linus Torvaldsd7824372010-08-15 11:35:52 -0700358 start,
Mikulas Patockaa09a79f2011-05-09 13:01:09 +0200359 end,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 flags & VM_READ ? 'r' : '-',
361 flags & VM_WRITE ? 'w' : '-',
362 flags & VM_EXEC ? 'x' : '-',
363 flags & VM_MAYSHARE ? 's' : 'p',
KAMEZAWA Hiroyuki6260a4b2009-04-06 19:00:30 -0700364 pgoff,
Tetsuo Handa652586d2013-11-14 14:31:57 -0800365 MAJOR(dev), MINOR(dev), ino);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366
367 /*
368 * Print the dentry name for named mappings, and a
369 * special [heap] marker for the heap:
370 */
Mauricio Line070ad42005-09-03 15:55:10 -0700371 if (file) {
Tetsuo Handa652586d2013-11-14 14:31:57 -0800372 seq_pad(m, ' ');
Miklos Szeredi2726d562015-06-19 10:30:28 +0200373 seq_file_path(m, file, "\n");
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700374 goto done;
375 }
376
Andy Lutomirski78d683e2014-05-19 15:58:32 -0700377 if (vma->vm_ops && vma->vm_ops->name) {
378 name = vma->vm_ops->name(vma);
379 if (name)
380 goto done;
381 }
382
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700383 name = arch_vma_name(vma);
384 if (!name) {
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700385 if (!mm) {
386 name = "[vdso]";
387 goto done;
388 }
389
390 if (vma->vm_start <= mm->brk &&
391 vma->vm_end >= mm->start_brk) {
392 name = "[heap]";
393 goto done;
394 }
395
Colin Cross3e4578f2015-10-27 16:42:08 -0700396 if (is_stack(priv, vma)) {
Johannes Weiner65376df2016-02-02 16:57:29 -0800397 name = "[stack]";
Colin Cross3e4578f2015-10-27 16:42:08 -0700398 goto done;
399 }
400
401 if (vma_get_anon_name(vma)) {
402 seq_pad(m, ' ');
403 seq_print_vma_name(m, vma);
404 }
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700405 }
406
407done:
408 if (name) {
Tetsuo Handa652586d2013-11-14 14:31:57 -0800409 seq_pad(m, ' ');
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700410 seq_puts(m, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 }
412 seq_putc(m, '\n');
Joe Korty7c88db02008-10-16 15:27:09 +0400413}
414
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700415static int show_map(struct seq_file *m, void *v, int is_pid)
Joe Korty7c88db02008-10-16 15:27:09 +0400416{
Oleg Nesterovebb6cdd2014-10-09 15:25:34 -0700417 show_map_vma(m, v, is_pid);
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700418 m_cache_vma(m, v);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 return 0;
420}
421
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700422static int show_pid_map(struct seq_file *m, void *v)
423{
424 return show_map(m, v, 1);
425}
426
427static int show_tid_map(struct seq_file *m, void *v)
428{
429 return show_map(m, v, 0);
430}
431
Jan Engelhardt03a44822008-02-08 04:21:19 -0800432static const struct seq_operations proc_pid_maps_op = {
Matt Mackalla6198792008-02-04 22:29:03 -0800433 .start = m_start,
434 .next = m_next,
435 .stop = m_stop,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700436 .show = show_pid_map
Matt Mackalla6198792008-02-04 22:29:03 -0800437};
438
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700439static const struct seq_operations proc_tid_maps_op = {
440 .start = m_start,
441 .next = m_next,
442 .stop = m_stop,
443 .show = show_tid_map
444};
445
446static int pid_maps_open(struct inode *inode, struct file *file)
Matt Mackalla6198792008-02-04 22:29:03 -0800447{
448 return do_maps_open(inode, file, &proc_pid_maps_op);
449}
450
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700451static int tid_maps_open(struct inode *inode, struct file *file)
452{
453 return do_maps_open(inode, file, &proc_tid_maps_op);
454}
455
456const struct file_operations proc_pid_maps_operations = {
457 .open = pid_maps_open,
458 .read = seq_read,
459 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700460 .release = proc_map_release,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700461};
462
463const struct file_operations proc_tid_maps_operations = {
464 .open = tid_maps_open,
Matt Mackalla6198792008-02-04 22:29:03 -0800465 .read = seq_read,
466 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700467 .release = proc_map_release,
Matt Mackalla6198792008-02-04 22:29:03 -0800468};
469
470/*
471 * Proportional Set Size(PSS): my share of RSS.
472 *
473 * PSS of a process is the count of pages it has in memory, where each
474 * page is divided by the number of processes sharing it. So if a
475 * process has 1000 pages all to itself, and 1000 shared with one other
476 * process, its PSS will be 1500.
477 *
478 * To keep (accumulated) division errors low, we adopt a 64bit
479 * fixed-point pss counter to minimize division errors. So (pss >>
480 * PSS_SHIFT) would be the real byte count.
481 *
482 * A shift of 12 before division means (assuming 4K page size):
483 * - 1M 3-user-pages add up to 8KB errors;
484 * - supports mapcount up to 2^24, or 16M;
485 * - supports PSS up to 2^52 bytes, or 4PB.
486 */
487#define PSS_SHIFT 12
488
Matt Mackall1e883282008-02-04 22:29:07 -0800489#ifdef CONFIG_PROC_PAGE_MONITOR
Peter Zijlstra214e4712008-04-28 02:12:55 -0700490struct mem_size_stats {
Matt Mackalla6198792008-02-04 22:29:03 -0800491 unsigned long resident;
492 unsigned long shared_clean;
493 unsigned long shared_dirty;
494 unsigned long private_clean;
495 unsigned long private_dirty;
496 unsigned long referenced;
Nikanth Karthikesanb40d4f82010-10-27 15:34:10 -0700497 unsigned long anonymous;
Dave Hansen4031a212011-03-22 16:33:01 -0700498 unsigned long anonymous_thp;
Kirill A. Shutemov65c45372016-07-26 15:26:10 -0700499 unsigned long shmem_thp;
Peter Zijlstra214e4712008-04-28 02:12:55 -0700500 unsigned long swap;
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800501 unsigned long shared_hugetlb;
502 unsigned long private_hugetlb;
Matt Mackalla6198792008-02-04 22:29:03 -0800503 u64 pss;
Minchan Kim8334b962015-09-08 15:00:24 -0700504 u64 swap_pss;
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800505 bool check_shmem_swap;
Matt Mackalla6198792008-02-04 22:29:03 -0800506};
507
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800508static void smaps_account(struct mem_size_stats *mss, struct page *page,
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800509 bool compound, bool young, bool dirty)
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800510{
Kirill A. Shutemovf4be6152016-01-20 14:58:12 -0800511 int i, nr = compound ? 1 << compound_order(page) : 1;
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800512 unsigned long size = nr * PAGE_SIZE;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700513
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800514 if (PageAnon(page))
515 mss->anonymous += size;
516
517 mss->resident += size;
518 /* Accumulate the size in pages that have been accessed. */
Vladimir Davydov33c3fc72015-09-09 15:35:45 -0700519 if (young || page_is_young(page) || PageReferenced(page))
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800520 mss->referenced += size;
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800521
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800522 /*
523 * page_count(page) == 1 guarantees the page is mapped exactly once.
524 * If any subpage of the compound page mapped with PTE it would elevate
525 * page_count().
526 */
527 if (page_count(page) == 1) {
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800528 if (dirty || PageDirty(page))
529 mss->private_dirty += size;
530 else
531 mss->private_clean += size;
532 mss->pss += (u64)size << PSS_SHIFT;
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800533 return;
534 }
535
536 for (i = 0; i < nr; i++, page++) {
537 int mapcount = page_mapcount(page);
538
539 if (mapcount >= 2) {
540 if (dirty || PageDirty(page))
541 mss->shared_dirty += PAGE_SIZE;
542 else
543 mss->shared_clean += PAGE_SIZE;
544 mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
545 } else {
546 if (dirty || PageDirty(page))
547 mss->private_dirty += PAGE_SIZE;
548 else
549 mss->private_clean += PAGE_SIZE;
550 mss->pss += PAGE_SIZE << PSS_SHIFT;
551 }
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800552 }
553}
554
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800555#ifdef CONFIG_SHMEM
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800556static int smaps_pte_hole(unsigned long addr, unsigned long end,
557 struct mm_walk *walk)
558{
559 struct mem_size_stats *mss = walk->private;
560
Vlastimil Babka48131e02016-01-14 15:19:23 -0800561 mss->swap += shmem_partial_swap_usage(
562 walk->vma->vm_file->f_mapping, addr, end);
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800563
564 return 0;
565}
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800566#endif
567
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800568static void smaps_pte_entry(pte_t *pte, unsigned long addr,
569 struct mm_walk *walk)
Dave Hansenae11c4d2011-03-22 16:32:58 -0700570{
571 struct mem_size_stats *mss = walk->private;
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800572 struct vm_area_struct *vma = walk->vma;
Konstantin Khlebnikovb1d4d9e2012-05-31 16:26:20 -0700573 struct page *page = NULL;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700574
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800575 if (pte_present(*pte)) {
576 page = vm_normal_page(vma, addr, *pte);
577 } else if (is_swap_pte(*pte)) {
578 swp_entry_t swpent = pte_to_swp_entry(*pte);
Konstantin Khlebnikovb1d4d9e2012-05-31 16:26:20 -0700579
Minchan Kim8334b962015-09-08 15:00:24 -0700580 if (!non_swap_entry(swpent)) {
581 int mapcount;
582
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800583 mss->swap += PAGE_SIZE;
Minchan Kim8334b962015-09-08 15:00:24 -0700584 mapcount = swp_swapcount(swpent);
585 if (mapcount >= 2) {
586 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
587
588 do_div(pss_delta, mapcount);
589 mss->swap_pss += pss_delta;
590 } else {
591 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
592 }
593 } else if (is_migration_entry(swpent))
Konstantin Khlebnikovb1d4d9e2012-05-31 16:26:20 -0700594 page = migration_entry_to_page(swpent);
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800595 } else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
596 && pte_none(*pte))) {
Vlastimil Babka48131e02016-01-14 15:19:23 -0800597 page = find_get_entry(vma->vm_file->f_mapping,
598 linear_page_index(vma, addr));
599 if (!page)
600 return;
601
602 if (radix_tree_exceptional_entry(page))
603 mss->swap += PAGE_SIZE;
604 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300605 put_page(page);
Vlastimil Babka48131e02016-01-14 15:19:23 -0800606
607 return;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700608 }
609
Dave Hansenae11c4d2011-03-22 16:32:58 -0700610 if (!page)
611 return;
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800612
613 smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte));
Dave Hansenae11c4d2011-03-22 16:32:58 -0700614}
615
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800616#ifdef CONFIG_TRANSPARENT_HUGEPAGE
617static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
618 struct mm_walk *walk)
619{
620 struct mem_size_stats *mss = walk->private;
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800621 struct vm_area_struct *vma = walk->vma;
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800622 struct page *page;
623
624 /* FOLL_DUMP will return -EFAULT on huge zero page */
625 page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
626 if (IS_ERR_OR_NULL(page))
627 return;
Kirill A. Shutemov65c45372016-07-26 15:26:10 -0700628 if (PageAnon(page))
629 mss->anonymous_thp += HPAGE_PMD_SIZE;
630 else if (PageSwapBacked(page))
631 mss->shmem_thp += HPAGE_PMD_SIZE;
Dan Williamsca120cf2016-09-03 10:38:03 -0700632 else if (is_zone_device_page(page))
633 /* pass */;
Kirill A. Shutemov65c45372016-07-26 15:26:10 -0700634 else
635 VM_BUG_ON_PAGE(1, page);
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800636 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd));
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800637}
638#else
639static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
640 struct mm_walk *walk)
641{
642}
643#endif
644
Matt Mackallb3ae5ac2008-02-04 22:29:01 -0800645static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -0700646 struct mm_walk *walk)
Mauricio Line070ad42005-09-03 15:55:10 -0700647{
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800648 struct vm_area_struct *vma = walk->vma;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700649 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700650 spinlock_t *ptl;
Mauricio Line070ad42005-09-03 15:55:10 -0700651
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -0800652 ptl = pmd_trans_huge_lock(pmd, vma);
653 if (ptl) {
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800654 smaps_pmd_entry(pmd, addr, walk);
Kirill A. Shutemovbf929152013-11-14 14:30:54 -0800655 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -0700656 return 0;
Dave Hansen22e057c2011-03-22 16:33:00 -0700657 }
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -0700658
659 if (pmd_trans_unstable(pmd))
660 return 0;
Dave Hansen22e057c2011-03-22 16:33:00 -0700661 /*
662 * The mmap_sem held all the way back in m_start() is what
663 * keeps khugepaged out of here and from collapsing things
664 * in here.
665 */
Hugh Dickins705e87c2005-10-29 18:16:27 -0700666 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Dave Hansenae11c4d2011-03-22 16:32:58 -0700667 for (; addr != end; pte++, addr += PAGE_SIZE)
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800668 smaps_pte_entry(pte, addr, walk);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700669 pte_unmap_unlock(pte - 1, ptl);
670 cond_resched();
Matt Mackallb3ae5ac2008-02-04 22:29:01 -0800671 return 0;
Mauricio Line070ad42005-09-03 15:55:10 -0700672}
673
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800674static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
675{
676 /*
677 * Don't forget to update Documentation/ on changes.
678 */
679 static const char mnemonics[BITS_PER_LONG][2] = {
680 /*
681 * In case if we meet a flag we don't know about.
682 */
683 [0 ... (BITS_PER_LONG-1)] = "??",
684
685 [ilog2(VM_READ)] = "rd",
686 [ilog2(VM_WRITE)] = "wr",
687 [ilog2(VM_EXEC)] = "ex",
688 [ilog2(VM_SHARED)] = "sh",
689 [ilog2(VM_MAYREAD)] = "mr",
690 [ilog2(VM_MAYWRITE)] = "mw",
691 [ilog2(VM_MAYEXEC)] = "me",
692 [ilog2(VM_MAYSHARE)] = "ms",
693 [ilog2(VM_GROWSDOWN)] = "gd",
694 [ilog2(VM_PFNMAP)] = "pf",
695 [ilog2(VM_DENYWRITE)] = "dw",
Qiaowei Ren4aae7e42014-11-14 07:18:25 -0800696#ifdef CONFIG_X86_INTEL_MPX
697 [ilog2(VM_MPX)] = "mp",
698#endif
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800699 [ilog2(VM_LOCKED)] = "lo",
700 [ilog2(VM_IO)] = "io",
701 [ilog2(VM_SEQ_READ)] = "sr",
702 [ilog2(VM_RAND_READ)] = "rr",
703 [ilog2(VM_DONTCOPY)] = "dc",
704 [ilog2(VM_DONTEXPAND)] = "de",
705 [ilog2(VM_ACCOUNT)] = "ac",
706 [ilog2(VM_NORESERVE)] = "nr",
707 [ilog2(VM_HUGETLB)] = "ht",
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800708 [ilog2(VM_ARCH_1)] = "ar",
709 [ilog2(VM_DONTDUMP)] = "dd",
Naoya Horiguchiec8e41a2013-11-12 15:07:49 -0800710#ifdef CONFIG_MEM_SOFT_DIRTY
711 [ilog2(VM_SOFTDIRTY)] = "sd",
712#endif
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800713 [ilog2(VM_MIXEDMAP)] = "mm",
714 [ilog2(VM_HUGEPAGE)] = "hg",
715 [ilog2(VM_NOHUGEPAGE)] = "nh",
716 [ilog2(VM_MERGEABLE)] = "mg",
Andrea Arcangeli16ba6f82015-09-04 15:46:17 -0700717 [ilog2(VM_UFFD_MISSING)]= "um",
718 [ilog2(VM_UFFD_WP)] = "uw",
Dave Hansenc1192f82016-02-12 13:02:27 -0800719#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
720 /* These come out via ProtectionKey: */
721 [ilog2(VM_PKEY_BIT0)] = "",
722 [ilog2(VM_PKEY_BIT1)] = "",
723 [ilog2(VM_PKEY_BIT2)] = "",
724 [ilog2(VM_PKEY_BIT3)] = "",
725#endif
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800726 };
727 size_t i;
728
729 seq_puts(m, "VmFlags: ");
730 for (i = 0; i < BITS_PER_LONG; i++) {
Dave Hansenc1192f82016-02-12 13:02:27 -0800731 if (!mnemonics[i][0])
732 continue;
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800733 if (vma->vm_flags & (1UL << i)) {
734 seq_printf(m, "%c%c ",
735 mnemonics[i][0], mnemonics[i][1]);
736 }
737 }
738 seq_putc(m, '\n');
739}
740
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800741#ifdef CONFIG_HUGETLB_PAGE
742static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
743 unsigned long addr, unsigned long end,
744 struct mm_walk *walk)
745{
746 struct mem_size_stats *mss = walk->private;
747 struct vm_area_struct *vma = walk->vma;
748 struct page *page = NULL;
749
750 if (pte_present(*pte)) {
751 page = vm_normal_page(vma, addr, *pte);
752 } else if (is_swap_pte(*pte)) {
753 swp_entry_t swpent = pte_to_swp_entry(*pte);
754
755 if (is_migration_entry(swpent))
756 page = migration_entry_to_page(swpent);
757 }
758 if (page) {
759 int mapcount = page_mapcount(page);
760
761 if (mapcount >= 2)
762 mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
763 else
764 mss->private_hugetlb += huge_page_size(hstate_vma(vma));
765 }
766 return 0;
767}
768#endif /* HUGETLB_PAGE */
769
Dave Hansenc1192f82016-02-12 13:02:27 -0800770void __weak arch_show_smap(struct seq_file *m, struct vm_area_struct *vma)
771{
772}
773
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700774static int show_smap(struct seq_file *m, void *v, int is_pid)
Mauricio Line070ad42005-09-03 15:55:10 -0700775{
776 struct vm_area_struct *vma = v;
Mauricio Line070ad42005-09-03 15:55:10 -0700777 struct mem_size_stats mss;
Dave Hansen21650092008-06-12 15:21:47 -0700778 struct mm_walk smaps_walk = {
779 .pmd_entry = smaps_pte_range,
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800780#ifdef CONFIG_HUGETLB_PAGE
781 .hugetlb_entry = smaps_hugetlb_range,
782#endif
Dave Hansen21650092008-06-12 15:21:47 -0700783 .mm = vma->vm_mm,
784 .private = &mss,
785 };
Mauricio Line070ad42005-09-03 15:55:10 -0700786
787 memset(&mss, 0, sizeof mss);
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800788
789#ifdef CONFIG_SHMEM
790 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
Vlastimil Babka6a15a372016-01-14 15:19:20 -0800791 /*
792 * For shared or readonly shmem mappings we know that all
793 * swapped out pages belong to the shmem object, and we can
794 * obtain the swap value much more efficiently. For private
795 * writable mappings, we might have COW pages that are
796 * not affected by the parent swapped out pages of the shmem
797 * object, so we have to distinguish them during the page walk.
798 * Unless we know that the shmem object (or the part mapped by
799 * our VMA) has no swapped out pages at all.
800 */
801 unsigned long shmem_swapped = shmem_swap_usage(vma);
802
803 if (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
804 !(vma->vm_flags & VM_WRITE)) {
805 mss.swap = shmem_swapped;
806 } else {
807 mss.check_shmem_swap = true;
808 smaps_walk.pte_hole = smaps_pte_hole;
809 }
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800810 }
811#endif
812
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +0900813 /* mmap_sem is held in m_start */
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800814 walk_page_vma(vma, &smaps_walk);
Matt Mackall4752c362008-02-04 22:29:02 -0800815
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700816 show_map_vma(m, vma, is_pid);
Matt Mackall4752c362008-02-04 22:29:02 -0800817
Colin Cross3e4578f2015-10-27 16:42:08 -0700818 if (vma_get_anon_name(vma)) {
819 seq_puts(m, "Name: ");
820 seq_print_vma_name(m, vma);
821 seq_putc(m, '\n');
822 }
823
Matt Mackall4752c362008-02-04 22:29:02 -0800824 seq_printf(m,
825 "Size: %8lu kB\n"
826 "Rss: %8lu kB\n"
827 "Pss: %8lu kB\n"
828 "Shared_Clean: %8lu kB\n"
829 "Shared_Dirty: %8lu kB\n"
830 "Private_Clean: %8lu kB\n"
831 "Private_Dirty: %8lu kB\n"
Peter Zijlstra214e4712008-04-28 02:12:55 -0700832 "Referenced: %8lu kB\n"
Nikanth Karthikesanb40d4f82010-10-27 15:34:10 -0700833 "Anonymous: %8lu kB\n"
Dave Hansen4031a212011-03-22 16:33:01 -0700834 "AnonHugePages: %8lu kB\n"
Kirill A. Shutemov65c45372016-07-26 15:26:10 -0700835 "ShmemPmdMapped: %8lu kB\n"
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800836 "Shared_Hugetlb: %8lu kB\n"
837 "Private_Hugetlb: %7lu kB\n"
Mel Gorman08fba692009-01-06 14:38:53 -0800838 "Swap: %8lu kB\n"
Minchan Kim8334b962015-09-08 15:00:24 -0700839 "SwapPss: %8lu kB\n"
Mel Gorman33402892009-01-06 14:38:54 -0800840 "KernelPageSize: %8lu kB\n"
Nikanth Karthikesan2d905082011-01-13 15:45:53 -0800841 "MMUPageSize: %8lu kB\n"
842 "Locked: %8lu kB\n",
Matt Mackall4752c362008-02-04 22:29:02 -0800843 (vma->vm_end - vma->vm_start) >> 10,
844 mss.resident >> 10,
845 (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
846 mss.shared_clean >> 10,
847 mss.shared_dirty >> 10,
848 mss.private_clean >> 10,
849 mss.private_dirty >> 10,
Peter Zijlstra214e4712008-04-28 02:12:55 -0700850 mss.referenced >> 10,
Nikanth Karthikesanb40d4f82010-10-27 15:34:10 -0700851 mss.anonymous >> 10,
Dave Hansen4031a212011-03-22 16:33:01 -0700852 mss.anonymous_thp >> 10,
Kirill A. Shutemov65c45372016-07-26 15:26:10 -0700853 mss.shmem_thp >> 10,
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800854 mss.shared_hugetlb >> 10,
855 mss.private_hugetlb >> 10,
Mel Gorman08fba692009-01-06 14:38:53 -0800856 mss.swap >> 10,
Minchan Kim8334b962015-09-08 15:00:24 -0700857 (unsigned long)(mss.swap_pss >> (10 + PSS_SHIFT)),
Mel Gorman33402892009-01-06 14:38:54 -0800858 vma_kernel_pagesize(vma) >> 10,
Nikanth Karthikesan2d905082011-01-13 15:45:53 -0800859 vma_mmu_pagesize(vma) >> 10,
860 (vma->vm_flags & VM_LOCKED) ?
861 (unsigned long)(mss.pss >> (10 + PSS_SHIFT)) : 0);
Matt Mackall4752c362008-02-04 22:29:02 -0800862
Dave Hansenc1192f82016-02-12 13:02:27 -0800863 arch_show_smap(m, vma);
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800864 show_smap_vma_flags(m, vma);
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700865 m_cache_vma(m, vma);
Joe Korty7c88db02008-10-16 15:27:09 +0400866 return 0;
Mauricio Line070ad42005-09-03 15:55:10 -0700867}
868
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700869static int show_pid_smap(struct seq_file *m, void *v)
870{
871 return show_smap(m, v, 1);
872}
873
874static int show_tid_smap(struct seq_file *m, void *v)
875{
876 return show_smap(m, v, 0);
877}
878
Jan Engelhardt03a44822008-02-08 04:21:19 -0800879static const struct seq_operations proc_pid_smaps_op = {
Matt Mackalla6198792008-02-04 22:29:03 -0800880 .start = m_start,
881 .next = m_next,
882 .stop = m_stop,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700883 .show = show_pid_smap
Matt Mackalla6198792008-02-04 22:29:03 -0800884};
885
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700886static const struct seq_operations proc_tid_smaps_op = {
887 .start = m_start,
888 .next = m_next,
889 .stop = m_stop,
890 .show = show_tid_smap
891};
892
893static int pid_smaps_open(struct inode *inode, struct file *file)
Matt Mackalla6198792008-02-04 22:29:03 -0800894{
895 return do_maps_open(inode, file, &proc_pid_smaps_op);
896}
897
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700898static int tid_smaps_open(struct inode *inode, struct file *file)
899{
900 return do_maps_open(inode, file, &proc_tid_smaps_op);
901}
902
903const struct file_operations proc_pid_smaps_operations = {
904 .open = pid_smaps_open,
905 .read = seq_read,
906 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700907 .release = proc_map_release,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700908};
909
910const struct file_operations proc_tid_smaps_operations = {
911 .open = tid_smaps_open,
Matt Mackalla6198792008-02-04 22:29:03 -0800912 .read = seq_read,
913 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700914 .release = proc_map_release,
Matt Mackalla6198792008-02-04 22:29:03 -0800915};
916
Pavel Emelyanov040fa022013-07-03 15:01:16 -0700917enum clear_refs_types {
918 CLEAR_REFS_ALL = 1,
919 CLEAR_REFS_ANON,
920 CLEAR_REFS_MAPPED,
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700921 CLEAR_REFS_SOFT_DIRTY,
Petr Cermak695f0552015-02-12 15:01:00 -0800922 CLEAR_REFS_MM_HIWATER_RSS,
Pavel Emelyanov040fa022013-07-03 15:01:16 -0700923 CLEAR_REFS_LAST,
924};
925
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -0700926struct clear_refs_private {
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700927 enum clear_refs_types type;
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -0700928};
929
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800930#ifdef CONFIG_MEM_SOFT_DIRTY
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700931static inline void clear_soft_dirty(struct vm_area_struct *vma,
932 unsigned long addr, pte_t *pte)
933{
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700934 /*
935 * The soft-dirty tracker uses #PF-s to catch writes
936 * to pages, so write-protect the pte as well. See the
937 * Documentation/vm/soft-dirty.txt for full description
938 * of how soft-dirty works.
939 */
940 pte_t ptent = *pte;
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700941
942 if (pte_present(ptent)) {
Laurent Dufour326c2592015-11-05 18:49:21 -0800943 ptent = ptep_modify_prot_start(vma->vm_mm, addr, pte);
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700944 ptent = pte_wrprotect(ptent);
Martin Schwidefskya7b76172015-04-22 14:20:47 +0200945 ptent = pte_clear_soft_dirty(ptent);
Laurent Dufour326c2592015-11-05 18:49:21 -0800946 ptep_modify_prot_commit(vma->vm_mm, addr, pte, ptent);
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700947 } else if (is_swap_pte(ptent)) {
948 ptent = pte_swp_clear_soft_dirty(ptent);
Laurent Dufour326c2592015-11-05 18:49:21 -0800949 set_pte_at(vma->vm_mm, addr, pte, ptent);
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700950 }
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700951}
Laurent Dufour5d3875a2015-11-05 18:49:24 -0800952#else
953static inline void clear_soft_dirty(struct vm_area_struct *vma,
954 unsigned long addr, pte_t *pte)
955{
956}
957#endif
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700958
Laurent Dufour5d3875a2015-11-05 18:49:24 -0800959#if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800960static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
961 unsigned long addr, pmd_t *pmdp)
962{
Kirill A. Shutemov5c9d0832017-04-13 14:56:28 -0700963 pmd_t pmd = *pmdp;
964
965 /* See comment in change_huge_pmd() */
966 pmdp_invalidate(vma, addr, pmdp);
967 if (pmd_dirty(*pmdp))
968 pmd = pmd_mkdirty(pmd);
969 if (pmd_young(*pmdp))
970 pmd = pmd_mkyoung(pmd);
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800971
972 pmd = pmd_wrprotect(pmd);
Martin Schwidefskya7b76172015-04-22 14:20:47 +0200973 pmd = pmd_clear_soft_dirty(pmd);
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800974
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800975 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
976}
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800977#else
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800978static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
979 unsigned long addr, pmd_t *pmdp)
980{
981}
982#endif
983
Matt Mackalla6198792008-02-04 22:29:03 -0800984static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
Dave Hansen21650092008-06-12 15:21:47 -0700985 unsigned long end, struct mm_walk *walk)
Matt Mackalla6198792008-02-04 22:29:03 -0800986{
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -0700987 struct clear_refs_private *cp = walk->private;
Naoya Horiguchi5c64f522015-02-11 15:27:46 -0800988 struct vm_area_struct *vma = walk->vma;
Matt Mackalla6198792008-02-04 22:29:03 -0800989 pte_t *pte, ptent;
990 spinlock_t *ptl;
991 struct page *page;
992
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -0800993 ptl = pmd_trans_huge_lock(pmd, vma);
994 if (ptl) {
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800995 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
996 clear_soft_dirty_pmd(vma, addr, pmd);
997 goto out;
998 }
999
1000 page = pmd_page(*pmd);
1001
1002 /* Clear accessed and referenced bits. */
1003 pmdp_test_and_clear_young(vma, addr, pmd);
Vladimir Davydov33c3fc72015-09-09 15:35:45 -07001004 test_and_clear_page_young(page);
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -08001005 ClearPageReferenced(page);
1006out:
1007 spin_unlock(ptl);
1008 return 0;
1009 }
1010
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07001011 if (pmd_trans_unstable(pmd))
1012 return 0;
Dave Hansen03319322011-03-22 16:32:56 -07001013
Matt Mackalla6198792008-02-04 22:29:03 -08001014 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1015 for (; addr != end; pte++, addr += PAGE_SIZE) {
1016 ptent = *pte;
Matt Mackalla6198792008-02-04 22:29:03 -08001017
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001018 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1019 clear_soft_dirty(vma, addr, pte);
1020 continue;
1021 }
1022
Cyrill Gorcunov179ef712013-08-13 16:00:49 -07001023 if (!pte_present(ptent))
1024 continue;
1025
Matt Mackalla6198792008-02-04 22:29:03 -08001026 page = vm_normal_page(vma, addr, ptent);
1027 if (!page)
1028 continue;
1029
1030 /* Clear accessed and referenced bits. */
1031 ptep_test_and_clear_young(vma, addr, pte);
Vladimir Davydov33c3fc72015-09-09 15:35:45 -07001032 test_and_clear_page_young(page);
Matt Mackalla6198792008-02-04 22:29:03 -08001033 ClearPageReferenced(page);
1034 }
1035 pte_unmap_unlock(pte - 1, ptl);
1036 cond_resched();
1037 return 0;
1038}
1039
Naoya Horiguchi5c64f522015-02-11 15:27:46 -08001040static int clear_refs_test_walk(unsigned long start, unsigned long end,
1041 struct mm_walk *walk)
1042{
1043 struct clear_refs_private *cp = walk->private;
1044 struct vm_area_struct *vma = walk->vma;
1045
Naoya Horiguchi48684a62015-02-11 15:28:06 -08001046 if (vma->vm_flags & VM_PFNMAP)
1047 return 1;
1048
Naoya Horiguchi5c64f522015-02-11 15:27:46 -08001049 /*
1050 * Writing 1 to /proc/pid/clear_refs affects all pages.
1051 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1052 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1053 * Writing 4 to /proc/pid/clear_refs affects all pages.
1054 */
1055 if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1056 return 1;
1057 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1058 return 1;
1059 return 0;
1060}
1061
Matt Mackallf248dcb2008-02-04 22:29:03 -08001062static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1063 size_t count, loff_t *ppos)
David Rientjesb813e932007-05-06 14:49:24 -07001064{
Matt Mackallf248dcb2008-02-04 22:29:03 -08001065 struct task_struct *task;
Vincent Lifb92a4b2009-09-22 16:45:36 -07001066 char buffer[PROC_NUMBUF];
Matt Mackallf248dcb2008-02-04 22:29:03 -08001067 struct mm_struct *mm;
David Rientjesb813e932007-05-06 14:49:24 -07001068 struct vm_area_struct *vma;
Pavel Emelyanov040fa022013-07-03 15:01:16 -07001069 enum clear_refs_types type;
1070 int itype;
Alexey Dobriyan0a8cb8e2011-05-26 16:25:50 -07001071 int rv;
David Rientjesb813e932007-05-06 14:49:24 -07001072
Matt Mackallf248dcb2008-02-04 22:29:03 -08001073 memset(buffer, 0, sizeof(buffer));
1074 if (count > sizeof(buffer) - 1)
1075 count = sizeof(buffer) - 1;
1076 if (copy_from_user(buffer, buf, count))
1077 return -EFAULT;
Pavel Emelyanov040fa022013-07-03 15:01:16 -07001078 rv = kstrtoint(strstrip(buffer), 10, &itype);
Alexey Dobriyan0a8cb8e2011-05-26 16:25:50 -07001079 if (rv < 0)
1080 return rv;
Pavel Emelyanov040fa022013-07-03 15:01:16 -07001081 type = (enum clear_refs_types)itype;
1082 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
Matt Mackallf248dcb2008-02-04 22:29:03 -08001083 return -EINVAL;
Pavel Emelyanov541c2372013-07-03 15:01:22 -07001084
Al Viro496ad9a2013-01-23 17:07:38 -05001085 task = get_proc_task(file_inode(file));
Matt Mackallf248dcb2008-02-04 22:29:03 -08001086 if (!task)
1087 return -ESRCH;
1088 mm = get_task_mm(task);
1089 if (mm) {
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -07001090 struct clear_refs_private cp = {
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001091 .type = type,
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -07001092 };
Andrew Morton20cbc972008-07-05 12:29:05 -07001093 struct mm_walk clear_refs_walk = {
1094 .pmd_entry = clear_refs_pte_range,
Naoya Horiguchi5c64f522015-02-11 15:27:46 -08001095 .test_walk = clear_refs_test_walk,
Andrew Morton20cbc972008-07-05 12:29:05 -07001096 .mm = mm,
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -07001097 .private = &cp,
Andrew Morton20cbc972008-07-05 12:29:05 -07001098 };
Petr Cermak695f0552015-02-12 15:01:00 -08001099
1100 if (type == CLEAR_REFS_MM_HIWATER_RSS) {
Michal Hocko4e801532016-05-23 16:25:45 -07001101 if (down_write_killable(&mm->mmap_sem)) {
1102 count = -EINTR;
1103 goto out_mm;
1104 }
1105
Petr Cermak695f0552015-02-12 15:01:00 -08001106 /*
1107 * Writing 5 to /proc/pid/clear_refs resets the peak
1108 * resident set size to this mm's current rss value.
1109 */
Petr Cermak695f0552015-02-12 15:01:00 -08001110 reset_mm_hiwater_rss(mm);
1111 up_write(&mm->mmap_sem);
1112 goto out_mm;
1113 }
1114
Matt Mackallf248dcb2008-02-04 22:29:03 -08001115 down_read(&mm->mmap_sem);
Peter Feiner64e45502014-10-13 15:55:46 -07001116 if (type == CLEAR_REFS_SOFT_DIRTY) {
1117 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1118 if (!(vma->vm_flags & VM_SOFTDIRTY))
1119 continue;
1120 up_read(&mm->mmap_sem);
Michal Hocko4e801532016-05-23 16:25:45 -07001121 if (down_write_killable(&mm->mmap_sem)) {
1122 count = -EINTR;
1123 goto out_mm;
1124 }
Peter Feiner64e45502014-10-13 15:55:46 -07001125 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1126 vma->vm_flags &= ~VM_SOFTDIRTY;
1127 vma_set_page_prot(vma);
1128 }
1129 downgrade_write(&mm->mmap_sem);
1130 break;
1131 }
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001132 mmu_notifier_invalidate_range_start(mm, 0, -1);
Peter Feiner64e45502014-10-13 15:55:46 -07001133 }
James Morse0f302062016-10-07 17:00:06 -07001134 walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001135 if (type == CLEAR_REFS_SOFT_DIRTY)
1136 mmu_notifier_invalidate_range_end(mm, 0, -1);
Matt Mackallf248dcb2008-02-04 22:29:03 -08001137 flush_tlb_mm(mm);
1138 up_read(&mm->mmap_sem);
Petr Cermak695f0552015-02-12 15:01:00 -08001139out_mm:
Matt Mackallf248dcb2008-02-04 22:29:03 -08001140 mmput(mm);
1141 }
1142 put_task_struct(task);
Vincent Lifb92a4b2009-09-22 16:45:36 -07001143
1144 return count;
David Rientjesb813e932007-05-06 14:49:24 -07001145}
1146
Matt Mackallf248dcb2008-02-04 22:29:03 -08001147const struct file_operations proc_clear_refs_operations = {
1148 .write = clear_refs_write,
Arnd Bergmann6038f372010-08-15 18:52:59 +02001149 .llseek = noop_llseek,
Matt Mackallf248dcb2008-02-04 22:29:03 -08001150};
1151
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001152typedef struct {
1153 u64 pme;
1154} pagemap_entry_t;
1155
Matt Mackall85863e42008-02-04 22:29:04 -08001156struct pagemapread {
yonghua zheng8c829622013-08-13 16:01:03 -07001157 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001158 pagemap_entry_t *buffer;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001159 bool show_pfn;
Matt Mackall85863e42008-02-04 22:29:04 -08001160};
1161
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001162#define PAGEMAP_WALK_SIZE (PMD_SIZE)
1163#define PAGEMAP_WALK_MASK (PMD_MASK)
1164
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001165#define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1166#define PM_PFRAME_BITS 55
1167#define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1168#define PM_SOFT_DIRTY BIT_ULL(55)
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001169#define PM_MMAP_EXCLUSIVE BIT_ULL(56)
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001170#define PM_FILE BIT_ULL(61)
1171#define PM_SWAP BIT_ULL(62)
1172#define PM_PRESENT BIT_ULL(63)
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001173
Matt Mackall85863e42008-02-04 22:29:04 -08001174#define PM_END_OF_BUFFER 1
1175
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001176static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001177{
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001178 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001179}
1180
1181static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
Matt Mackall85863e42008-02-04 22:29:04 -08001182 struct pagemapread *pm)
1183{
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001184 pm->buffer[pm->pos++] = *pme;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001185 if (pm->pos >= pm->len)
Thomas Tuttleaae86792008-06-05 22:46:31 -07001186 return PM_END_OF_BUFFER;
Matt Mackall85863e42008-02-04 22:29:04 -08001187 return 0;
1188}
1189
1190static int pagemap_pte_hole(unsigned long start, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -07001191 struct mm_walk *walk)
Matt Mackall85863e42008-02-04 22:29:04 -08001192{
Dave Hansen21650092008-06-12 15:21:47 -07001193 struct pagemapread *pm = walk->private;
Peter Feiner68b5a652014-08-06 16:08:09 -07001194 unsigned long addr = start;
Matt Mackall85863e42008-02-04 22:29:04 -08001195 int err = 0;
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001196
Peter Feiner68b5a652014-08-06 16:08:09 -07001197 while (addr < end) {
1198 struct vm_area_struct *vma = find_vma(walk->mm, addr);
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001199 pagemap_entry_t pme = make_pme(0, 0);
Peter Feiner87e6d492014-09-25 16:05:18 -07001200 /* End of address space hole, which we mark as non-present. */
1201 unsigned long hole_end;
Peter Feiner68b5a652014-08-06 16:08:09 -07001202
Peter Feiner87e6d492014-09-25 16:05:18 -07001203 if (vma)
1204 hole_end = min(end, vma->vm_start);
1205 else
1206 hole_end = end;
1207
1208 for (; addr < hole_end; addr += PAGE_SIZE) {
1209 err = add_to_pagemap(addr, &pme, pm);
1210 if (err)
1211 goto out;
Peter Feiner68b5a652014-08-06 16:08:09 -07001212 }
1213
Peter Feiner87e6d492014-09-25 16:05:18 -07001214 if (!vma)
1215 break;
1216
1217 /* Addresses in the VMA. */
1218 if (vma->vm_flags & VM_SOFTDIRTY)
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001219 pme = make_pme(0, PM_SOFT_DIRTY);
Peter Feiner87e6d492014-09-25 16:05:18 -07001220 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
Peter Feiner68b5a652014-08-06 16:08:09 -07001221 err = add_to_pagemap(addr, &pme, pm);
1222 if (err)
1223 goto out;
1224 }
Matt Mackall85863e42008-02-04 22:29:04 -08001225 }
Peter Feiner68b5a652014-08-06 16:08:09 -07001226out:
Matt Mackall85863e42008-02-04 22:29:04 -08001227 return err;
1228}
1229
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001230static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001231 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
Matt Mackall85863e42008-02-04 22:29:04 -08001232{
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001233 u64 frame = 0, flags = 0;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001234 struct page *page = NULL;
Matt Mackall85863e42008-02-04 22:29:04 -08001235
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001236 if (pte_present(pte)) {
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001237 if (pm->show_pfn)
1238 frame = pte_pfn(pte);
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001239 flags |= PM_PRESENT;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001240 page = vm_normal_page(vma, addr, pte);
Cyrill Gorcunove9cdd6e2013-10-16 13:46:53 -07001241 if (pte_soft_dirty(pte))
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001242 flags |= PM_SOFT_DIRTY;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001243 } else if (is_swap_pte(pte)) {
Cyrill Gorcunov179ef712013-08-13 16:00:49 -07001244 swp_entry_t entry;
1245 if (pte_swp_soft_dirty(pte))
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001246 flags |= PM_SOFT_DIRTY;
Cyrill Gorcunov179ef712013-08-13 16:00:49 -07001247 entry = pte_to_swp_entry(pte);
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001248 frame = swp_type(entry) |
1249 (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001250 flags |= PM_SWAP;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001251 if (is_migration_entry(entry))
1252 page = migration_entry_to_page(entry);
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001253 }
1254
1255 if (page && !PageAnon(page))
1256 flags |= PM_FILE;
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001257 if (page && page_mapcount(page) == 1)
1258 flags |= PM_MMAP_EXCLUSIVE;
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001259 if (vma->vm_flags & VM_SOFTDIRTY)
1260 flags |= PM_SOFT_DIRTY;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001261
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001262 return make_pme(frame, flags);
Dave Hansenbcf80392008-06-12 15:21:48 -07001263}
1264
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001265static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -07001266 struct mm_walk *walk)
Matt Mackall85863e42008-02-04 22:29:04 -08001267{
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001268 struct vm_area_struct *vma = walk->vma;
Dave Hansen21650092008-06-12 15:21:47 -07001269 struct pagemapread *pm = walk->private;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001270 spinlock_t *ptl;
Konstantin Khlebnikov05fbf352015-02-11 15:27:31 -08001271 pte_t *pte, *orig_pte;
Matt Mackall85863e42008-02-04 22:29:04 -08001272 int err = 0;
1273
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001274#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001275 ptl = pmd_trans_huge_lock(pmdp, vma);
1276 if (ptl) {
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001277 u64 flags = 0, frame = 0;
1278 pmd_t pmd = *pmdp;
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001279
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001280 if ((vma->vm_flags & VM_SOFTDIRTY) || pmd_soft_dirty(pmd))
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001281 flags |= PM_SOFT_DIRTY;
Cyrill Gorcunovd9104d12013-09-11 14:22:24 -07001282
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001283 /*
1284 * Currently pmd for thp is always present because thp
1285 * can not be swapped-out, migrated, or HWPOISONed
1286 * (split in such cases instead.)
1287 * This if-check is just to prepare for future implementation.
1288 */
1289 if (pmd_present(pmd)) {
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001290 struct page *page = pmd_page(pmd);
1291
1292 if (page_mapcount(page) == 1)
1293 flags |= PM_MMAP_EXCLUSIVE;
1294
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001295 flags |= PM_PRESENT;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001296 if (pm->show_pfn)
1297 frame = pmd_pfn(pmd) +
1298 ((addr & ~PMD_MASK) >> PAGE_SHIFT);
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001299 }
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001300
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001301 for (; addr != end; addr += PAGE_SIZE) {
1302 pagemap_entry_t pme = make_pme(frame, flags);
1303
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001304 err = add_to_pagemap(addr, &pme, pm);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001305 if (err)
1306 break;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001307 if (pm->show_pfn && (flags & PM_PRESENT))
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001308 frame++;
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001309 }
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001310 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001311 return err;
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001312 }
1313
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001314 if (pmd_trans_unstable(pmdp))
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07001315 return 0;
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001316#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
Dave Hansenbcf80392008-06-12 15:21:48 -07001317
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001318 /*
1319 * We can assume that @vma always points to a valid one and @end never
1320 * goes beyond vma->vm_end.
1321 */
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001322 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001323 for (; addr < end; pte++, addr += PAGE_SIZE) {
1324 pagemap_entry_t pme;
Peter Feiner81d0fa62014-10-09 15:28:32 -07001325
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001326 pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001327 err = add_to_pagemap(addr, &pme, pm);
Konstantin Khlebnikov05fbf352015-02-11 15:27:31 -08001328 if (err)
Peter Feiner81d0fa62014-10-09 15:28:32 -07001329 break;
Matt Mackall85863e42008-02-04 22:29:04 -08001330 }
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001331 pte_unmap_unlock(orig_pte, ptl);
Matt Mackall85863e42008-02-04 22:29:04 -08001332
1333 cond_resched();
1334
1335 return err;
1336}
1337
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001338#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi116354d2010-04-06 14:35:04 -07001339/* This function walks within one hugetlb entry in the single call */
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001340static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
Naoya Horiguchi116354d2010-04-06 14:35:04 -07001341 unsigned long addr, unsigned long end,
1342 struct mm_walk *walk)
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001343{
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001344 struct pagemapread *pm = walk->private;
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001345 struct vm_area_struct *vma = walk->vma;
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001346 u64 flags = 0, frame = 0;
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001347 int err = 0;
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001348 pte_t pte;
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001349
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001350 if (vma->vm_flags & VM_SOFTDIRTY)
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001351 flags |= PM_SOFT_DIRTY;
Cyrill Gorcunovd9104d12013-09-11 14:22:24 -07001352
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001353 pte = huge_ptep_get(ptep);
1354 if (pte_present(pte)) {
1355 struct page *page = pte_page(pte);
1356
1357 if (!PageAnon(page))
1358 flags |= PM_FILE;
1359
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001360 if (page_mapcount(page) == 1)
1361 flags |= PM_MMAP_EXCLUSIVE;
1362
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001363 flags |= PM_PRESENT;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001364 if (pm->show_pfn)
1365 frame = pte_pfn(pte) +
1366 ((addr & ~hmask) >> PAGE_SHIFT);
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001367 }
1368
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001369 for (; addr != end; addr += PAGE_SIZE) {
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001370 pagemap_entry_t pme = make_pme(frame, flags);
1371
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001372 err = add_to_pagemap(addr, &pme, pm);
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001373 if (err)
1374 return err;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001375 if (pm->show_pfn && (flags & PM_PRESENT))
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001376 frame++;
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001377 }
1378
1379 cond_resched();
1380
1381 return err;
1382}
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001383#endif /* HUGETLB_PAGE */
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001384
Matt Mackall85863e42008-02-04 22:29:04 -08001385/*
1386 * /proc/pid/pagemap - an array mapping virtual pages to pfns
1387 *
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001388 * For each page in the address space, this file contains one 64-bit entry
1389 * consisting of the following:
1390 *
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001391 * Bits 0-54 page frame number (PFN) if present
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001392 * Bits 0-4 swap type if swapped
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001393 * Bits 5-54 swap offset if swapped
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001394 * Bit 55 pte is soft-dirty (see Documentation/vm/soft-dirty.txt)
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001395 * Bit 56 page exclusively mapped
1396 * Bits 57-60 zero
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001397 * Bit 61 page is file-page or shared-anon
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001398 * Bit 62 page swapped
1399 * Bit 63 page present
1400 *
1401 * If the page is not present but in swap, then the PFN contains an
1402 * encoding of the swap file number and the page's offset into the
1403 * swap. Unmapped pages return a null PFN. This allows determining
Matt Mackall85863e42008-02-04 22:29:04 -08001404 * precisely which pages are mapped (or in swap) and comparing mapped
1405 * pages between processes.
1406 *
1407 * Efficient users of this interface will use /proc/pid/maps to
1408 * determine which areas of memory are actually mapped and llseek to
1409 * skip over unmapped regions.
1410 */
1411static ssize_t pagemap_read(struct file *file, char __user *buf,
1412 size_t count, loff_t *ppos)
1413{
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001414 struct mm_struct *mm = file->private_data;
Matt Mackall85863e42008-02-04 22:29:04 -08001415 struct pagemapread pm;
Alexey Dobriyanee1e6ab2008-07-21 14:21:36 -07001416 struct mm_walk pagemap_walk = {};
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001417 unsigned long src;
1418 unsigned long svpfn;
1419 unsigned long start_vaddr;
1420 unsigned long end_vaddr;
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001421 int ret = 0, copied = 0;
Matt Mackall85863e42008-02-04 22:29:04 -08001422
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001423 if (!mm || !atomic_inc_not_zero(&mm->mm_users))
Matt Mackall85863e42008-02-04 22:29:04 -08001424 goto out;
1425
Matt Mackall85863e42008-02-04 22:29:04 -08001426 ret = -EINVAL;
1427 /* file position must be aligned */
Thomas Tuttleaae86792008-06-05 22:46:31 -07001428 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001429 goto out_mm;
Matt Mackall85863e42008-02-04 22:29:04 -08001430
1431 ret = 0;
Vitaly Mayatskikh08161782009-04-30 15:08:18 -07001432 if (!count)
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001433 goto out_mm;
Vitaly Mayatskikh08161782009-04-30 15:08:18 -07001434
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001435 /* do not disclose physical addresses: attack vector */
1436 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1437
yonghua zheng8c829622013-08-13 16:01:03 -07001438 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1439 pm.buffer = kmalloc(pm.len * PM_ENTRY_BYTES, GFP_TEMPORARY);
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001440 ret = -ENOMEM;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001441 if (!pm.buffer)
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001442 goto out_mm;
Matt Mackall85863e42008-02-04 22:29:04 -08001443
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001444 pagemap_walk.pmd_entry = pagemap_pmd_range;
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001445 pagemap_walk.pte_hole = pagemap_pte_hole;
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001446#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001447 pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001448#endif
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001449 pagemap_walk.mm = mm;
1450 pagemap_walk.private = &pm;
Matt Mackall85863e42008-02-04 22:29:04 -08001451
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001452 src = *ppos;
1453 svpfn = src / PM_ENTRY_BYTES;
1454 start_vaddr = svpfn << PAGE_SHIFT;
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001455 end_vaddr = mm->task_size;
Matt Mackall85863e42008-02-04 22:29:04 -08001456
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001457 /* watch out for wraparound */
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001458 if (svpfn > mm->task_size >> PAGE_SHIFT)
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001459 start_vaddr = end_vaddr;
1460
1461 /*
1462 * The odds are that this will stop walking way
1463 * before end_vaddr, because the length of the
1464 * user buffer is tracked in "pm", and the walk
1465 * will stop when we hit the end of the buffer.
1466 */
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001467 ret = 0;
1468 while (count && (start_vaddr < end_vaddr)) {
1469 int len;
1470 unsigned long end;
Matt Mackall85863e42008-02-04 22:29:04 -08001471
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001472 pm.pos = 0;
Naoya Horiguchiea251c12010-11-24 12:57:13 -08001473 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001474 /* overflow ? */
1475 if (end < start_vaddr || end > end_vaddr)
1476 end = end_vaddr;
1477 down_read(&mm->mmap_sem);
1478 ret = walk_page_range(start_vaddr, end, &pagemap_walk);
1479 up_read(&mm->mmap_sem);
1480 start_vaddr = end;
1481
1482 len = min(count, PM_ENTRY_BYTES * pm.pos);
Dan Carpenter309361e02010-04-06 13:45:39 +03001483 if (copy_to_user(buf, pm.buffer, len)) {
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001484 ret = -EFAULT;
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001485 goto out_free;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001486 }
1487 copied += len;
1488 buf += len;
1489 count -= len;
Matt Mackall85863e42008-02-04 22:29:04 -08001490 }
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001491 *ppos += copied;
1492 if (!ret || ret == PM_END_OF_BUFFER)
1493 ret = copied;
1494
KOSAKI Motohiro98bc93e2011-05-26 16:25:53 -07001495out_free:
1496 kfree(pm.buffer);
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001497out_mm:
1498 mmput(mm);
Matt Mackall85863e42008-02-04 22:29:04 -08001499out:
1500 return ret;
1501}
1502
Pavel Emelyanov541c2372013-07-03 15:01:22 -07001503static int pagemap_open(struct inode *inode, struct file *file)
1504{
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001505 struct mm_struct *mm;
1506
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001507 mm = proc_mem_open(inode, PTRACE_MODE_READ);
1508 if (IS_ERR(mm))
1509 return PTR_ERR(mm);
1510 file->private_data = mm;
1511 return 0;
1512}
1513
1514static int pagemap_release(struct inode *inode, struct file *file)
1515{
1516 struct mm_struct *mm = file->private_data;
1517
1518 if (mm)
1519 mmdrop(mm);
Pavel Emelyanov541c2372013-07-03 15:01:22 -07001520 return 0;
1521}
1522
Matt Mackall85863e42008-02-04 22:29:04 -08001523const struct file_operations proc_pagemap_operations = {
1524 .llseek = mem_lseek, /* borrow this */
1525 .read = pagemap_read,
Pavel Emelyanov541c2372013-07-03 15:01:22 -07001526 .open = pagemap_open,
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001527 .release = pagemap_release,
Matt Mackall85863e42008-02-04 22:29:04 -08001528};
Matt Mackall1e883282008-02-04 22:29:07 -08001529#endif /* CONFIG_PROC_PAGE_MONITOR */
Matt Mackall85863e42008-02-04 22:29:04 -08001530
Minchan Kimff926892013-05-09 16:21:24 +09001531#ifdef CONFIG_PROCESS_RECLAIM
1532static int reclaim_pte_range(pmd_t *pmd, unsigned long addr,
1533 unsigned long end, struct mm_walk *walk)
1534{
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301535 struct reclaim_param *rp = walk->private;
1536 struct vm_area_struct *vma = rp->vma;
Minchan Kimff926892013-05-09 16:21:24 +09001537 pte_t *pte, ptent;
1538 spinlock_t *ptl;
1539 struct page *page;
1540 LIST_HEAD(page_list);
1541 int isolated;
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301542 int reclaimed;
Minchan Kimff926892013-05-09 16:21:24 +09001543
1544 split_huge_pmd(vma, addr, pmd);
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301545 if (pmd_trans_unstable(pmd) || !rp->nr_to_reclaim)
Minchan Kimff926892013-05-09 16:21:24 +09001546 return 0;
1547cont:
1548 isolated = 0;
1549 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1550 for (; addr != end; pte++, addr += PAGE_SIZE) {
1551 ptent = *pte;
1552 if (!pte_present(ptent))
1553 continue;
1554
1555 page = vm_normal_page(vma, addr, ptent);
1556 if (!page)
1557 continue;
1558
1559 if (isolate_lru_page(page))
1560 continue;
1561
1562 list_add(&page->lru, &page_list);
1563 inc_node_page_state(page, NR_ISOLATED_ANON +
1564 page_is_file_cache(page));
1565 isolated++;
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301566 rp->nr_scanned++;
1567 if ((isolated >= SWAP_CLUSTER_MAX) || !rp->nr_to_reclaim)
Minchan Kimff926892013-05-09 16:21:24 +09001568 break;
1569 }
1570 pte_unmap_unlock(pte - 1, ptl);
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301571 reclaimed = reclaim_pages_from_list(&page_list, vma);
1572 rp->nr_reclaimed += reclaimed;
1573 rp->nr_to_reclaim -= reclaimed;
1574 if (rp->nr_to_reclaim < 0)
1575 rp->nr_to_reclaim = 0;
1576
1577 if (rp->nr_to_reclaim && (addr != end))
Minchan Kimff926892013-05-09 16:21:24 +09001578 goto cont;
1579
1580 cond_resched();
1581 return 0;
1582}
1583
1584enum reclaim_type {
1585 RECLAIM_FILE,
1586 RECLAIM_ANON,
1587 RECLAIM_ALL,
1588 RECLAIM_RANGE,
1589};
1590
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301591struct reclaim_param reclaim_task_anon(struct task_struct *task,
1592 int nr_to_reclaim)
1593{
1594 struct mm_struct *mm;
1595 struct vm_area_struct *vma;
1596 struct mm_walk reclaim_walk = {};
1597 struct reclaim_param rp;
1598
1599 rp.nr_reclaimed = 0;
1600 rp.nr_scanned = 0;
1601 get_task_struct(task);
1602 mm = get_task_mm(task);
1603 if (!mm)
1604 goto out;
1605
1606 reclaim_walk.mm = mm;
1607 reclaim_walk.pmd_entry = reclaim_pte_range;
1608
1609 rp.nr_to_reclaim = nr_to_reclaim;
1610 reclaim_walk.private = &rp;
1611
1612 down_read(&mm->mmap_sem);
1613 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1614 if (is_vm_hugetlb_page(vma))
1615 continue;
1616
1617 if (vma->vm_file)
1618 continue;
1619
Vinayak Menone65af6b2017-05-26 16:29:55 +05301620 if (vma->vm_flags & VM_LOCKED)
1621 continue;
1622
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301623 if (!rp.nr_to_reclaim)
1624 break;
1625
1626 rp.vma = vma;
1627 walk_page_range(vma->vm_start, vma->vm_end,
1628 &reclaim_walk);
1629 }
1630
1631 flush_tlb_mm(mm);
1632 up_read(&mm->mmap_sem);
1633 mmput(mm);
1634out:
1635 put_task_struct(task);
1636 return rp;
1637}
1638
Minchan Kimff926892013-05-09 16:21:24 +09001639static ssize_t reclaim_write(struct file *file, const char __user *buf,
1640 size_t count, loff_t *ppos)
1641{
1642 struct task_struct *task;
Minchan Kim01fdd712013-05-09 16:21:28 +09001643 char buffer[200];
Minchan Kimff926892013-05-09 16:21:24 +09001644 struct mm_struct *mm;
1645 struct vm_area_struct *vma;
1646 enum reclaim_type type;
1647 char *type_buf;
Minchan Kim01fdd712013-05-09 16:21:28 +09001648 struct mm_walk reclaim_walk = {};
1649 unsigned long start = 0;
1650 unsigned long end = 0;
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301651 struct reclaim_param rp;
Minchan Kimff926892013-05-09 16:21:24 +09001652
1653 memset(buffer, 0, sizeof(buffer));
1654 if (count > sizeof(buffer) - 1)
1655 count = sizeof(buffer) - 1;
1656
1657 if (copy_from_user(buffer, buf, count))
1658 return -EFAULT;
1659
1660 type_buf = strstrip(buffer);
1661 if (!strcmp(type_buf, "file"))
1662 type = RECLAIM_FILE;
1663 else if (!strcmp(type_buf, "anon"))
1664 type = RECLAIM_ANON;
1665 else if (!strcmp(type_buf, "all"))
1666 type = RECLAIM_ALL;
Minchan Kim01fdd712013-05-09 16:21:28 +09001667 else if (isdigit(*type_buf))
1668 type = RECLAIM_RANGE;
Minchan Kimff926892013-05-09 16:21:24 +09001669 else
Minchan Kim01fdd712013-05-09 16:21:28 +09001670 goto out_err;
1671
1672 if (type == RECLAIM_RANGE) {
1673 char *token;
1674 unsigned long long len, len_in, tmp;
1675 token = strsep(&type_buf, " ");
1676 if (!token)
1677 goto out_err;
1678 tmp = memparse(token, &token);
1679 if (tmp & ~PAGE_MASK || tmp > ULONG_MAX)
1680 goto out_err;
1681 start = tmp;
1682
1683 token = strsep(&type_buf, " ");
1684 if (!token)
1685 goto out_err;
1686 len_in = memparse(token, &token);
1687 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
1688 if (len > ULONG_MAX)
1689 goto out_err;
1690 /*
1691 * Check to see whether len was rounded up from small -ve
1692 * to zero.
1693 */
1694 if (len_in && !len)
1695 goto out_err;
1696
1697 end = start + len;
1698 if (end < start)
1699 goto out_err;
1700 }
Minchan Kimff926892013-05-09 16:21:24 +09001701
1702 task = get_proc_task(file->f_path.dentry->d_inode);
1703 if (!task)
1704 return -ESRCH;
1705
1706 mm = get_task_mm(task);
Minchan Kim01fdd712013-05-09 16:21:28 +09001707 if (!mm)
1708 goto out;
Minchan Kimff926892013-05-09 16:21:24 +09001709
Minchan Kim01fdd712013-05-09 16:21:28 +09001710 reclaim_walk.mm = mm;
1711 reclaim_walk.pmd_entry = reclaim_pte_range;
1712
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301713 rp.nr_to_reclaim = INT_MAX;
1714 rp.nr_reclaimed = 0;
1715 reclaim_walk.private = &rp;
1716
Minchan Kim01fdd712013-05-09 16:21:28 +09001717 down_read(&mm->mmap_sem);
1718 if (type == RECLAIM_RANGE) {
1719 vma = find_vma(mm, start);
1720 while (vma) {
1721 if (vma->vm_start > end)
1722 break;
1723 if (is_vm_hugetlb_page(vma))
1724 continue;
1725
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301726 rp.vma = vma;
Minchan Kim01fdd712013-05-09 16:21:28 +09001727 walk_page_range(max(vma->vm_start, start),
1728 min(vma->vm_end, end),
1729 &reclaim_walk);
1730 vma = vma->vm_next;
1731 }
1732 } else {
1733 for (vma = mm->mmap; vma; vma = vma->vm_next) {
Minchan Kimff926892013-05-09 16:21:24 +09001734 if (is_vm_hugetlb_page(vma))
1735 continue;
1736
1737 if (type == RECLAIM_ANON && vma->vm_file)
1738 continue;
Minchan Kim01fdd712013-05-09 16:21:28 +09001739
Minchan Kimff926892013-05-09 16:21:24 +09001740 if (type == RECLAIM_FILE && !vma->vm_file)
1741 continue;
1742
Vinayak Menon1bddbcf2015-03-20 14:15:39 +05301743 rp.vma = vma;
Minchan Kimff926892013-05-09 16:21:24 +09001744 walk_page_range(vma->vm_start, vma->vm_end,
Minchan Kim01fdd712013-05-09 16:21:28 +09001745 &reclaim_walk);
Minchan Kimff926892013-05-09 16:21:24 +09001746 }
Minchan Kimff926892013-05-09 16:21:24 +09001747 }
Minchan Kimff926892013-05-09 16:21:24 +09001748
Minchan Kim01fdd712013-05-09 16:21:28 +09001749 flush_tlb_mm(mm);
1750 up_read(&mm->mmap_sem);
1751 mmput(mm);
1752out:
1753 put_task_struct(task);
Minchan Kimff926892013-05-09 16:21:24 +09001754 return count;
Minchan Kim01fdd712013-05-09 16:21:28 +09001755
1756out_err:
1757 return -EINVAL;
Minchan Kimff926892013-05-09 16:21:24 +09001758}
1759
1760const struct file_operations proc_reclaim_operations = {
1761 .write = reclaim_write,
1762 .llseek = noop_llseek,
1763};
1764#endif
1765
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001766#ifdef CONFIG_NUMA
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001767
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001768struct numa_maps {
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001769 unsigned long pages;
1770 unsigned long anon;
1771 unsigned long active;
1772 unsigned long writeback;
1773 unsigned long mapcount_max;
1774 unsigned long dirty;
1775 unsigned long swapcache;
1776 unsigned long node[MAX_NUMNODES];
1777};
1778
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001779struct numa_maps_private {
1780 struct proc_maps_private proc_maps;
1781 struct numa_maps md;
1782};
1783
Dave Hanseneb4866d2011-09-20 15:19:38 -07001784static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1785 unsigned long nr_pages)
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001786{
1787 int count = page_mapcount(page);
1788
Dave Hanseneb4866d2011-09-20 15:19:38 -07001789 md->pages += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001790 if (pte_dirty || PageDirty(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001791 md->dirty += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001792
1793 if (PageSwapCache(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001794 md->swapcache += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001795
1796 if (PageActive(page) || PageUnevictable(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001797 md->active += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001798
1799 if (PageWriteback(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001800 md->writeback += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001801
1802 if (PageAnon(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001803 md->anon += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001804
1805 if (count > md->mapcount_max)
1806 md->mapcount_max = count;
1807
Dave Hanseneb4866d2011-09-20 15:19:38 -07001808 md->node[page_to_nid(page)] += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001809}
1810
Dave Hansen3200a8a2011-09-20 15:19:39 -07001811static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1812 unsigned long addr)
1813{
1814 struct page *page;
1815 int nid;
1816
1817 if (!pte_present(pte))
1818 return NULL;
1819
1820 page = vm_normal_page(vma, addr, pte);
1821 if (!page)
1822 return NULL;
1823
1824 if (PageReserved(page))
1825 return NULL;
1826
1827 nid = page_to_nid(page);
Lai Jiangshan4ff1b2c2012-12-12 13:51:25 -08001828 if (!node_isset(nid, node_states[N_MEMORY]))
Dave Hansen3200a8a2011-09-20 15:19:39 -07001829 return NULL;
1830
1831 return page;
1832}
1833
Gerald Schaefer28093f92016-04-28 16:18:35 -07001834#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1835static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1836 struct vm_area_struct *vma,
1837 unsigned long addr)
1838{
1839 struct page *page;
1840 int nid;
1841
1842 if (!pmd_present(pmd))
1843 return NULL;
1844
1845 page = vm_normal_page_pmd(vma, addr, pmd);
1846 if (!page)
1847 return NULL;
1848
1849 if (PageReserved(page))
1850 return NULL;
1851
1852 nid = page_to_nid(page);
1853 if (!node_isset(nid, node_states[N_MEMORY]))
1854 return NULL;
1855
1856 return page;
1857}
1858#endif
1859
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001860static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1861 unsigned long end, struct mm_walk *walk)
1862{
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001863 struct numa_maps *md = walk->private;
1864 struct vm_area_struct *vma = walk->vma;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001865 spinlock_t *ptl;
1866 pte_t *orig_pte;
1867 pte_t *pte;
1868
Gerald Schaefer28093f92016-04-28 16:18:35 -07001869#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001870 ptl = pmd_trans_huge_lock(pmd, vma);
1871 if (ptl) {
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001872 struct page *page;
1873
Gerald Schaefer28093f92016-04-28 16:18:35 -07001874 page = can_gather_numa_stats_pmd(*pmd, vma, addr);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001875 if (page)
Gerald Schaefer28093f92016-04-28 16:18:35 -07001876 gather_stats(page, md, pmd_dirty(*pmd),
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001877 HPAGE_PMD_SIZE/PAGE_SIZE);
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001878 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001879 return 0;
Dave Hansen32ef4382011-09-20 15:19:41 -07001880 }
1881
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07001882 if (pmd_trans_unstable(pmd))
1883 return 0;
Gerald Schaefer28093f92016-04-28 16:18:35 -07001884#endif
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001885 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1886 do {
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001887 struct page *page = can_gather_numa_stats(*pte, vma, addr);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001888 if (!page)
1889 continue;
Dave Hanseneb4866d2011-09-20 15:19:38 -07001890 gather_stats(page, md, pte_dirty(*pte), 1);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001891
1892 } while (pte++, addr += PAGE_SIZE, addr != end);
1893 pte_unmap_unlock(orig_pte, ptl);
1894 return 0;
1895}
1896#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi632fd602015-02-11 15:27:51 -08001897static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001898 unsigned long addr, unsigned long end, struct mm_walk *walk)
1899{
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001900 pte_t huge_pte = huge_ptep_get(pte);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001901 struct numa_maps *md;
1902 struct page *page;
1903
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001904 if (!pte_present(huge_pte))
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001905 return 0;
1906
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001907 page = pte_page(huge_pte);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001908 if (!page)
1909 return 0;
1910
1911 md = walk->private;
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001912 gather_stats(page, md, pte_dirty(huge_pte), 1);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001913 return 0;
1914}
1915
1916#else
Naoya Horiguchi632fd602015-02-11 15:27:51 -08001917static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001918 unsigned long addr, unsigned long end, struct mm_walk *walk)
1919{
1920 return 0;
1921}
1922#endif
1923
1924/*
1925 * Display pages allocated per node and memory policy via /proc.
1926 */
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001927static int show_numa_map(struct seq_file *m, void *v, int is_pid)
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001928{
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001929 struct numa_maps_private *numa_priv = m->private;
1930 struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001931 struct vm_area_struct *vma = v;
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001932 struct numa_maps *md = &numa_priv->md;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001933 struct file *file = vma->vm_file;
1934 struct mm_struct *mm = vma->vm_mm;
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001935 struct mm_walk walk = {
1936 .hugetlb_entry = gather_hugetlb_stats,
1937 .pmd_entry = gather_pte_stats,
1938 .private = md,
1939 .mm = mm,
1940 };
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001941 struct mempolicy *pol;
David Rientjes948927e2013-11-12 15:07:28 -08001942 char buffer[64];
1943 int nid;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001944
1945 if (!mm)
1946 return 0;
1947
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001948 /* Ensure we start with an empty set of numa_maps statistics. */
1949 memset(md, 0, sizeof(*md));
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001950
Oleg Nesterov498f2372014-10-09 15:27:52 -07001951 pol = __get_vma_policy(vma, vma->vm_start);
1952 if (pol) {
1953 mpol_to_str(buffer, sizeof(buffer), pol);
1954 mpol_cond_put(pol);
1955 } else {
1956 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1957 }
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001958
1959 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1960
1961 if (file) {
Fabian Frederick17c2b4e2014-06-06 14:37:03 -07001962 seq_puts(m, " file=");
Miklos Szeredi2726d562015-06-19 10:30:28 +02001963 seq_file_path(m, file, "\n\t= ");
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001964 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
Fabian Frederick17c2b4e2014-06-06 14:37:03 -07001965 seq_puts(m, " heap");
Andy Lutomirskib18cb642016-09-30 10:58:57 -07001966 } else if (is_stack(proc_priv, vma)) {
Johannes Weiner65376df2016-02-02 16:57:29 -08001967 seq_puts(m, " stack");
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001968 }
1969
Andrew Mortonfc360bd2011-10-31 17:06:32 -07001970 if (is_vm_hugetlb_page(vma))
Fabian Frederick17c2b4e2014-06-06 14:37:03 -07001971 seq_puts(m, " huge");
Andrew Mortonfc360bd2011-10-31 17:06:32 -07001972
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001973 /* mmap_sem is held by m_start */
1974 walk_page_vma(vma, &walk);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001975
1976 if (!md->pages)
1977 goto out;
1978
1979 if (md->anon)
1980 seq_printf(m, " anon=%lu", md->anon);
1981
1982 if (md->dirty)
1983 seq_printf(m, " dirty=%lu", md->dirty);
1984
1985 if (md->pages != md->anon && md->pages != md->dirty)
1986 seq_printf(m, " mapped=%lu", md->pages);
1987
1988 if (md->mapcount_max > 1)
1989 seq_printf(m, " mapmax=%lu", md->mapcount_max);
1990
1991 if (md->swapcache)
1992 seq_printf(m, " swapcache=%lu", md->swapcache);
1993
1994 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1995 seq_printf(m, " active=%lu", md->active);
1996
1997 if (md->writeback)
1998 seq_printf(m, " writeback=%lu", md->writeback);
1999
David Rientjes948927e2013-11-12 15:07:28 -08002000 for_each_node_state(nid, N_MEMORY)
2001 if (md->node[nid])
2002 seq_printf(m, " N%d=%lu", nid, md->node[nid]);
Rafael Aquini198d1592015-02-12 15:01:08 -08002003
2004 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07002005out:
2006 seq_putc(m, '\n');
Oleg Nesterovb8c20a92014-10-09 15:25:41 -07002007 m_cache_vma(m, vma);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07002008 return 0;
2009}
Stephen Wilson5b52fc82011-05-24 17:12:49 -07002010
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07002011static int show_pid_numa_map(struct seq_file *m, void *v)
2012{
2013 return show_numa_map(m, v, 1);
2014}
2015
2016static int show_tid_numa_map(struct seq_file *m, void *v)
2017{
2018 return show_numa_map(m, v, 0);
2019}
2020
Jan Engelhardt03a44822008-02-08 04:21:19 -08002021static const struct seq_operations proc_pid_numa_maps_op = {
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07002022 .start = m_start,
2023 .next = m_next,
2024 .stop = m_stop,
2025 .show = show_pid_numa_map,
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07002026};
Eric W. Biederman662795d2006-06-26 00:25:48 -07002027
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07002028static const struct seq_operations proc_tid_numa_maps_op = {
2029 .start = m_start,
2030 .next = m_next,
2031 .stop = m_stop,
2032 .show = show_tid_numa_map,
2033};
2034
2035static int numa_maps_open(struct inode *inode, struct file *file,
2036 const struct seq_operations *ops)
Eric W. Biederman662795d2006-06-26 00:25:48 -07002037{
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -07002038 return proc_maps_open(inode, file, ops,
2039 sizeof(struct numa_maps_private));
Eric W. Biederman662795d2006-06-26 00:25:48 -07002040}
2041
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07002042static int pid_numa_maps_open(struct inode *inode, struct file *file)
2043{
2044 return numa_maps_open(inode, file, &proc_pid_numa_maps_op);
2045}
2046
2047static int tid_numa_maps_open(struct inode *inode, struct file *file)
2048{
2049 return numa_maps_open(inode, file, &proc_tid_numa_maps_op);
2050}
2051
2052const struct file_operations proc_pid_numa_maps_operations = {
2053 .open = pid_numa_maps_open,
2054 .read = seq_read,
2055 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -07002056 .release = proc_map_release,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07002057};
2058
2059const struct file_operations proc_tid_numa_maps_operations = {
2060 .open = tid_numa_maps_open,
Eric W. Biederman662795d2006-06-26 00:25:48 -07002061 .read = seq_read,
2062 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -07002063 .release = proc_map_release,
Eric W. Biederman662795d2006-06-26 00:25:48 -07002064};
Stephen Wilsonf69ff942011-05-24 17:12:47 -07002065#endif /* CONFIG_NUMA */