blob: 254698856b8fc2723dae61e3dd20b6c8176129e7 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.h - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -08009 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 */
19
20#ifndef _LINUX_MEMCONTROL_H
21#define _LINUX_MEMCONTROL_H
Hirokazu Takahashif8d665422009-01-07 18:08:02 -080022#include <linux/cgroup.h>
Ying Han456f9982011-05-26 16:25:38 -070023#include <linux/vm_event_item.h>
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080024#include <linux/hardirq.h>
Glauber Costaa8964b92012-12-18 14:22:09 -080025#include <linux/jump_label.h>
Michal Hocko33398cf2015-09-08 15:01:02 -070026#include <linux/page_counter.h>
27#include <linux/vmpressure.h>
28#include <linux/eventfd.h>
29#include <linux/mmzone.h>
30#include <linux/writeback.h>
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -070031#include <linux/page-flags.h>
Ying Han456f9982011-05-26 16:25:38 -070032
Pavel Emelianov78fb7462008-02-07 00:13:51 -080033struct mem_cgroup;
Balbir Singh8697d332008-02-07 00:13:59 -080034struct page;
35struct mm_struct;
Glauber Costa2633d7a2012-12-18 14:22:34 -080036struct kmem_cache;
Pavel Emelianov78fb7462008-02-07 00:13:51 -080037
Sha Zhengju68b48762013-09-12 15:13:50 -070038/*
39 * The corresponding mem_cgroup_stat_names is defined in mm/memcontrol.c,
40 * These two lists should keep in accord with each other.
41 */
42enum mem_cgroup_stat_index {
43 /*
44 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
45 */
46 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
47 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
48 MEM_CGROUP_STAT_RSS_HUGE, /* # of pages charged as anon huge */
49 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
Greg Thelenc4843a72015-05-22 17:13:16 -040050 MEM_CGROUP_STAT_DIRTY, /* # of dirty pages in page cache */
Sha Zhengju3ea67d02013-09-12 15:13:53 -070051 MEM_CGROUP_STAT_WRITEBACK, /* # of pages under writeback */
Sha Zhengju68b48762013-09-12 15:13:50 -070052 MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
53 MEM_CGROUP_STAT_NSTATS,
Johannes Weinerb2807f02016-01-20 15:03:22 -080054 /* default hierarchy stats */
Andy Lutomirskiefdc9492016-07-28 15:48:17 -070055 MEMCG_KERNEL_STACK_KB = MEM_CGROUP_STAT_NSTATS,
Vladimir Davydov27ee57c2016-03-17 14:17:35 -070056 MEMCG_SLAB_RECLAIMABLE,
57 MEMCG_SLAB_UNRECLAIMABLE,
Vladimir Davydov12580e42016-03-17 14:17:38 -070058 MEMCG_SOCK,
Johannes Weinerb2807f02016-01-20 15:03:22 -080059 MEMCG_NR_STAT,
Greg Thelen2a7106f2011-01-13 15:47:37 -080060};
61
Johannes Weiner56600482012-01-12 17:17:59 -080062struct mem_cgroup_reclaim_cookie {
Mel Gormanef8f2322016-07-28 15:46:05 -070063 pg_data_t *pgdat;
Johannes Weiner56600482012-01-12 17:17:59 -080064 int priority;
65 unsigned int generation;
66};
67
Johannes Weiner241994ed2015-02-11 15:26:06 -080068enum mem_cgroup_events_index {
69 MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
70 MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
71 MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
72 MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
73 MEM_CGROUP_EVENTS_NSTATS,
74 /* default hierarchy events */
75 MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS,
76 MEMCG_HIGH,
77 MEMCG_MAX,
78 MEMCG_OOM,
79 MEMCG_NR_EVENTS,
80};
81
Michal Hocko33398cf2015-09-08 15:01:02 -070082/*
83 * Per memcg event counter is incremented at every pagein/pageout. With THP,
84 * it will be incremated by the number of pages. This counter is used for
85 * for trigger some periodic events. This is straightforward and better
86 * than using jiffies etc. to handle periodic memcg event.
87 */
88enum mem_cgroup_events_target {
89 MEM_CGROUP_TARGET_THRESH,
90 MEM_CGROUP_TARGET_SOFTLIMIT,
91 MEM_CGROUP_TARGET_NUMAINFO,
92 MEM_CGROUP_NTARGETS,
93};
94
Andrew Mortonc255a452012-07-31 16:43:02 -070095#ifdef CONFIG_MEMCG
Johannes Weiner23047a92016-03-15 14:57:16 -070096
97#define MEM_CGROUP_ID_SHIFT 16
98#define MEM_CGROUP_ID_MAX USHRT_MAX
99
Johannes Weiner73f576c2016-07-20 15:44:57 -0700100struct mem_cgroup_id {
101 int id;
102 atomic_t ref;
103};
104
Michal Hocko33398cf2015-09-08 15:01:02 -0700105struct mem_cgroup_stat_cpu {
Johannes Weinerb2807f02016-01-20 15:03:22 -0800106 long count[MEMCG_NR_STAT];
Michal Hocko33398cf2015-09-08 15:01:02 -0700107 unsigned long events[MEMCG_NR_EVENTS];
108 unsigned long nr_page_events;
109 unsigned long targets[MEM_CGROUP_NTARGETS];
110};
111
112struct mem_cgroup_reclaim_iter {
113 struct mem_cgroup *position;
114 /* scan generation, increased every round-trip */
115 unsigned int generation;
116};
117
118/*
119 * per-zone information in memory controller.
120 */
Mel Gormanef8f2322016-07-28 15:46:05 -0700121struct mem_cgroup_per_node {
Michal Hocko33398cf2015-09-08 15:01:02 -0700122 struct lruvec lruvec;
Michal Hockob4536f0c82017-01-10 16:58:04 -0800123 unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
Michal Hocko33398cf2015-09-08 15:01:02 -0700124
125 struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1];
126
127 struct rb_node tree_node; /* RB tree node */
128 unsigned long usage_in_excess;/* Set to the value by which */
129 /* the soft limit is exceeded*/
130 bool on_tree;
131 struct mem_cgroup *memcg; /* Back pointer, we cannot */
132 /* use container_of */
133};
134
Michal Hocko33398cf2015-09-08 15:01:02 -0700135struct mem_cgroup_threshold {
136 struct eventfd_ctx *eventfd;
137 unsigned long threshold;
138};
139
140/* For threshold */
141struct mem_cgroup_threshold_ary {
142 /* An array index points to threshold just below or equal to usage. */
143 int current_threshold;
144 /* Size of entries[] */
145 unsigned int size;
146 /* Array of thresholds */
147 struct mem_cgroup_threshold entries[0];
148};
149
150struct mem_cgroup_thresholds {
151 /* Primary thresholds array */
152 struct mem_cgroup_threshold_ary *primary;
153 /*
154 * Spare threshold array.
155 * This is needed to make mem_cgroup_unregister_event() "never fail".
156 * It must be able to store at least primary->size - 1 entries.
157 */
158 struct mem_cgroup_threshold_ary *spare;
159};
160
Johannes Weiner567e9ab2016-01-20 15:02:24 -0800161enum memcg_kmem_state {
162 KMEM_NONE,
163 KMEM_ALLOCATED,
164 KMEM_ONLINE,
165};
166
Michal Hocko33398cf2015-09-08 15:01:02 -0700167/*
168 * The memory controller data structure. The memory controller controls both
169 * page cache and RSS per cgroup. We would eventually like to provide
170 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
171 * to help the administrator determine what knobs to tune.
172 */
173struct mem_cgroup {
174 struct cgroup_subsys_state css;
175
Johannes Weiner73f576c2016-07-20 15:44:57 -0700176 /* Private memcg ID. Used to ID objects that outlive the cgroup */
177 struct mem_cgroup_id id;
178
Michal Hocko33398cf2015-09-08 15:01:02 -0700179 /* Accounted resources */
180 struct page_counter memory;
Vladimir Davydov37e84352016-01-20 15:02:56 -0800181 struct page_counter swap;
Johannes Weiner0db15292016-01-20 15:02:50 -0800182
183 /* Legacy consumer-oriented counters */
Michal Hocko33398cf2015-09-08 15:01:02 -0700184 struct page_counter memsw;
185 struct page_counter kmem;
Johannes Weiner0db15292016-01-20 15:02:50 -0800186 struct page_counter tcpmem;
Michal Hocko33398cf2015-09-08 15:01:02 -0700187
188 /* Normal memory consumption range */
189 unsigned long low;
190 unsigned long high;
191
Johannes Weinerf7e1cb62016-01-14 15:21:29 -0800192 /* Range enforcement for interrupt charges */
193 struct work_struct high_work;
194
Michal Hocko33398cf2015-09-08 15:01:02 -0700195 unsigned long soft_limit;
196
197 /* vmpressure notifications */
198 struct vmpressure vmpressure;
199
Michal Hocko33398cf2015-09-08 15:01:02 -0700200 /*
201 * Should the accounting and control be hierarchical, per subtree?
202 */
203 bool use_hierarchy;
204
205 /* protected by memcg_oom_lock */
206 bool oom_lock;
207 int under_oom;
208
209 int swappiness;
210 /* OOM-Killer disable */
211 int oom_kill_disable;
212
Tejun Heo472912a2015-09-18 18:01:59 -0400213 /* handle for "memory.events" */
214 struct cgroup_file events_file;
215
Michal Hocko33398cf2015-09-08 15:01:02 -0700216 /* protect arrays of thresholds */
217 struct mutex thresholds_lock;
218
219 /* thresholds for memory usage. RCU-protected */
220 struct mem_cgroup_thresholds thresholds;
221
222 /* thresholds for mem+swap usage. RCU-protected */
223 struct mem_cgroup_thresholds memsw_thresholds;
224
225 /* For oom notifier event fd */
226 struct list_head oom_notify;
227
228 /*
229 * Should we move charges of a task when a task is moved into this
230 * mem_cgroup ? And what type of charges should we move ?
231 */
232 unsigned long move_charge_at_immigrate;
233 /*
234 * set > 0 if pages under this cgroup are moving to other cgroup.
235 */
236 atomic_t moving_account;
237 /* taken only while moving_account > 0 */
238 spinlock_t move_lock;
239 struct task_struct *move_lock_task;
240 unsigned long move_lock_flags;
241 /*
242 * percpu counter.
243 */
244 struct mem_cgroup_stat_cpu __percpu *stat;
Michal Hocko33398cf2015-09-08 15:01:02 -0700245
Johannes Weinerd886f4e2016-01-20 15:02:47 -0800246 unsigned long socket_pressure;
247
248 /* Legacy tcp memory accounting */
Johannes Weiner0db15292016-01-20 15:02:50 -0800249 bool tcpmem_active;
250 int tcpmem_pressure;
Johannes Weinerd886f4e2016-01-20 15:02:47 -0800251
Johannes Weiner127424c2016-01-20 15:02:32 -0800252#ifndef CONFIG_SLOB
Michal Hocko33398cf2015-09-08 15:01:02 -0700253 /* Index in the kmem_cache->memcg_params.memcg_caches array */
254 int kmemcg_id;
Johannes Weiner567e9ab2016-01-20 15:02:24 -0800255 enum memcg_kmem_state kmem_state;
Michal Hocko33398cf2015-09-08 15:01:02 -0700256#endif
257
258 int last_scanned_node;
259#if MAX_NUMNODES > 1
260 nodemask_t scan_nodes;
261 atomic_t numainfo_events;
262 atomic_t numainfo_updating;
263#endif
264
265#ifdef CONFIG_CGROUP_WRITEBACK
266 struct list_head cgwb_list;
267 struct wb_domain cgwb_domain;
268#endif
269
270 /* List of events which userspace want to receive */
271 struct list_head event_list;
272 spinlock_t event_list_lock;
273
274 struct mem_cgroup_per_node *nodeinfo[0];
275 /* WARNING: nodeinfo must be the last member here */
276};
Johannes Weiner7d828602016-01-14 15:20:56 -0800277
278extern struct mem_cgroup *root_mem_cgroup;
Tejun Heo56161632015-05-22 17:13:20 -0400279
Johannes Weiner23047a92016-03-15 14:57:16 -0700280static inline bool mem_cgroup_disabled(void)
281{
282 return !cgroup_subsys_enabled(memory_cgrp_subsys);
283}
284
Michal Hocko33398cf2015-09-08 15:01:02 -0700285/**
286 * mem_cgroup_events - count memory events against a cgroup
287 * @memcg: the memory cgroup
288 * @idx: the event index
289 * @nr: the number of events to account for
290 */
291static inline void mem_cgroup_events(struct mem_cgroup *memcg,
Johannes Weiner241994ed2015-02-11 15:26:06 -0800292 enum mem_cgroup_events_index idx,
Michal Hocko33398cf2015-09-08 15:01:02 -0700293 unsigned int nr)
294{
295 this_cpu_add(memcg->stat->events[idx], nr);
Tejun Heo472912a2015-09-18 18:01:59 -0400296 cgroup_file_notify(&memcg->events_file);
Michal Hocko33398cf2015-09-08 15:01:02 -0700297}
Johannes Weiner241994ed2015-02-11 15:26:06 -0800298
299bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);
300
Johannes Weiner00501b52014-08-08 14:19:20 -0700301int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800302 gfp_t gfp_mask, struct mem_cgroup **memcgp,
303 bool compound);
Johannes Weiner00501b52014-08-08 14:19:20 -0700304void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800305 bool lrucare, bool compound);
306void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
307 bool compound);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700308void mem_cgroup_uncharge(struct page *page);
Johannes Weiner747db952014-08-08 14:19:24 -0700309void mem_cgroup_uncharge_list(struct list_head *page_list);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700310
Johannes Weiner6a93ca82016-03-15 14:57:19 -0700311void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700312
Mel Gormanef8f2322016-07-28 15:46:05 -0700313static struct mem_cgroup_per_node *
314mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
Johannes Weiner55779ec2016-07-28 15:45:10 -0700315{
Mel Gormanef8f2322016-07-28 15:46:05 -0700316 return memcg->nodeinfo[nid];
Johannes Weiner55779ec2016-07-28 15:45:10 -0700317}
318
319/**
Mel Gormana9dd0a82016-07-28 15:46:02 -0700320 * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
321 * @node: node of the wanted lruvec
Johannes Weiner55779ec2016-07-28 15:45:10 -0700322 * @memcg: memcg of the wanted lruvec
323 *
Mel Gormana9dd0a82016-07-28 15:46:02 -0700324 * Returns the lru list vector holding pages for a given @node or a given
325 * @memcg and @zone. This can be the node lruvec, if the memory controller
Johannes Weiner55779ec2016-07-28 15:45:10 -0700326 * is disabled.
327 */
Mel Gormana9dd0a82016-07-28 15:46:02 -0700328static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
Mel Gormanef8f2322016-07-28 15:46:05 -0700329 struct mem_cgroup *memcg)
Johannes Weiner55779ec2016-07-28 15:45:10 -0700330{
Mel Gormanef8f2322016-07-28 15:46:05 -0700331 struct mem_cgroup_per_node *mz;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700332 struct lruvec *lruvec;
333
334 if (mem_cgroup_disabled()) {
Mel Gormana9dd0a82016-07-28 15:46:02 -0700335 lruvec = node_lruvec(pgdat);
Johannes Weiner55779ec2016-07-28 15:45:10 -0700336 goto out;
337 }
338
Mel Gormanef8f2322016-07-28 15:46:05 -0700339 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
Johannes Weiner55779ec2016-07-28 15:45:10 -0700340 lruvec = &mz->lruvec;
341out:
342 /*
343 * Since a node can be onlined after the mem_cgroup was created,
Mel Gorman599d0c92016-07-28 15:45:31 -0700344 * we have to be prepared to initialize lruvec->pgdat here;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700345 * and if offlined then reonlined, we need to reinitialize it.
346 */
Mel Gormanef8f2322016-07-28 15:46:05 -0700347 if (unlikely(lruvec->pgdat != pgdat))
348 lruvec->pgdat = pgdat;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700349 return lruvec;
350}
351
Mel Gorman599d0c92016-07-28 15:45:31 -0700352struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -0800353
Johannes Weiner2314b422014-12-10 15:44:33 -0800354bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
Michal Hocko64219992015-09-08 15:01:07 -0700355struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
Vladimir Davydove993d902015-09-09 15:35:35 -0700356
Michal Hocko33398cf2015-09-08 15:01:02 -0700357static inline
358struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
359 return css ? container_of(css, struct mem_cgroup, css) : NULL;
360}
361
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800362#define mem_cgroup_from_counter(counter, member) \
363 container_of(counter, struct mem_cgroup, member)
364
Michal Hocko33398cf2015-09-08 15:01:02 -0700365struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
366 struct mem_cgroup *,
367 struct mem_cgroup_reclaim_cookie *);
368void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700369int mem_cgroup_scan_tasks(struct mem_cgroup *,
370 int (*)(struct task_struct *, void *), void *);
Michal Hocko33398cf2015-09-08 15:01:02 -0700371
Johannes Weiner23047a92016-03-15 14:57:16 -0700372static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
373{
374 if (mem_cgroup_disabled())
375 return 0;
376
Johannes Weiner73f576c2016-07-20 15:44:57 -0700377 return memcg->id.id;
Johannes Weiner23047a92016-03-15 14:57:16 -0700378}
Johannes Weiner73f576c2016-07-20 15:44:57 -0700379struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
Johannes Weiner23047a92016-03-15 14:57:16 -0700380
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800381/**
382 * parent_mem_cgroup - find the accounting parent of a memcg
383 * @memcg: memcg whose parent to find
384 *
385 * Returns the parent memcg, or NULL if this is the root or the memory
386 * controller is in legacy no-hierarchy mode.
387 */
388static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
389{
390 if (!memcg->memory.parent)
391 return NULL;
392 return mem_cgroup_from_counter(memcg->memory.parent, memory);
393}
394
Michal Hocko33398cf2015-09-08 15:01:02 -0700395static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
396 struct mem_cgroup *root)
397{
398 if (root == memcg)
399 return true;
400 if (!root->use_hierarchy)
401 return false;
402 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
403}
Glauber Costae1aab162011-12-11 21:47:03 +0000404
Johannes Weiner2314b422014-12-10 15:44:33 -0800405static inline bool mm_match_cgroup(struct mm_struct *mm,
406 struct mem_cgroup *memcg)
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800407{
Johannes Weiner587af302012-10-08 16:34:12 -0700408 struct mem_cgroup *task_memcg;
Johannes Weiner413918b2014-12-10 15:44:30 -0800409 bool match = false;
Johannes Weinerc3ac9a82012-05-29 15:06:25 -0700410
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800411 rcu_read_lock();
Johannes Weiner587af302012-10-08 16:34:12 -0700412 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
Johannes Weiner413918b2014-12-10 15:44:30 -0800413 if (task_memcg)
Johannes Weiner2314b422014-12-10 15:44:33 -0800414 match = mem_cgroup_is_descendant(task_memcg, memcg);
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800415 rcu_read_unlock();
Johannes Weinerc3ac9a82012-05-29 15:06:25 -0700416 return match;
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800417}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800418
Michal Hocko64219992015-09-08 15:01:07 -0700419struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700420ino_t page_cgroup_ino(struct page *page);
Wu Fengguangd3242362009-12-16 12:19:59 +0100421
Vladimir Davydoveb01aaa2016-01-20 15:03:02 -0800422static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
423{
424 if (mem_cgroup_disabled())
425 return true;
426 return !!(memcg->css.flags & CSS_ONLINE);
427}
428
KAMEZAWA Hiroyuki58ae83d2008-02-07 00:14:32 -0800429/*
430 * For memory reclaim.
431 */
Ying Han889976d2011-05-26 16:25:33 -0700432int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
Michal Hocko33398cf2015-09-08 15:01:02 -0700433
434void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
Michal Hockob4536f0c82017-01-10 16:58:04 -0800435 int zid, int nr_pages);
Michal Hocko33398cf2015-09-08 15:01:02 -0700436
Vladimir Davydov0a6b76d2016-03-17 14:18:42 -0700437unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
438 int nid, unsigned int lru_mask);
439
Michal Hocko33398cf2015-09-08 15:01:02 -0700440static inline
441unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
442{
Mel Gormanef8f2322016-07-28 15:46:05 -0700443 struct mem_cgroup_per_node *mz;
Michal Hockob4536f0c82017-01-10 16:58:04 -0800444 unsigned long nr_pages = 0;
445 int zid;
Michal Hocko33398cf2015-09-08 15:01:02 -0700446
Mel Gormanef8f2322016-07-28 15:46:05 -0700447 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Michal Hockob4536f0c82017-01-10 16:58:04 -0800448 for (zid = 0; zid < MAX_NR_ZONES; zid++)
449 nr_pages += mz->lru_zone_size[zid][lru];
450 return nr_pages;
451}
452
453static inline
454unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
455 enum lru_list lru, int zone_idx)
456{
457 struct mem_cgroup_per_node *mz;
458
459 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
460 return mz->lru_zone_size[zone_idx][lru];
Michal Hocko33398cf2015-09-08 15:01:02 -0700461}
462
Tejun Heob23afb92015-11-05 18:46:11 -0800463void mem_cgroup_handle_over_high(void);
464
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700465unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg);
466
Michal Hocko64219992015-09-08 15:01:07 -0700467void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
468 struct task_struct *p);
KAMEZAWA Hiroyuki58ae83d2008-02-07 00:14:32 -0800469
Johannes Weiner49426422013-10-16 13:46:59 -0700470static inline void mem_cgroup_oom_enable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700471{
Tejun Heo626ebc42015-11-05 18:46:09 -0800472 WARN_ON(current->memcg_may_oom);
473 current->memcg_may_oom = 1;
Johannes Weiner519e5242013-09-12 15:13:42 -0700474}
475
Johannes Weiner49426422013-10-16 13:46:59 -0700476static inline void mem_cgroup_oom_disable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700477{
Tejun Heo626ebc42015-11-05 18:46:09 -0800478 WARN_ON(!current->memcg_may_oom);
479 current->memcg_may_oom = 0;
Johannes Weiner519e5242013-09-12 15:13:42 -0700480}
481
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700482static inline bool task_in_memcg_oom(struct task_struct *p)
483{
Tejun Heo626ebc42015-11-05 18:46:09 -0800484 return p->memcg_in_oom;
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700485}
486
Johannes Weiner49426422013-10-16 13:46:59 -0700487bool mem_cgroup_oom_synchronize(bool wait);
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700488
Andrew Mortonc255a452012-07-31 16:43:02 -0700489#ifdef CONFIG_MEMCG_SWAP
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -0800490extern int do_swap_account;
491#endif
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800492
Johannes Weiner62cccb82016-03-15 14:57:22 -0700493void lock_page_memcg(struct page *page);
494void unlock_page_memcg(struct page *page);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -0700495
Michal Hocko33398cf2015-09-08 15:01:02 -0700496/**
497 * mem_cgroup_update_page_stat - update page state statistics
Johannes Weiner62cccb82016-03-15 14:57:22 -0700498 * @page: the page
Michal Hocko33398cf2015-09-08 15:01:02 -0700499 * @idx: page state item to account
500 * @val: number of pages (positive or negative)
501 *
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700502 * The @page must be locked or the caller must use lock_page_memcg()
503 * to prevent double accounting when the page is concurrently being
504 * moved to another memcg:
Johannes Weiner81f8c3a2016-03-15 14:57:04 -0700505 *
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700506 * lock_page(page) or lock_page_memcg(page)
Johannes Weiner81f8c3a2016-03-15 14:57:04 -0700507 * if (TestClearPageState(page))
Johannes Weiner62cccb82016-03-15 14:57:22 -0700508 * mem_cgroup_update_page_stat(page, state, -1);
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700509 * unlock_page(page) or unlock_page_memcg(page)
Michal Hocko33398cf2015-09-08 15:01:02 -0700510 */
Johannes Weiner62cccb82016-03-15 14:57:22 -0700511static inline void mem_cgroup_update_page_stat(struct page *page,
Michal Hocko33398cf2015-09-08 15:01:02 -0700512 enum mem_cgroup_stat_index idx, int val)
513{
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700514 VM_BUG_ON(!(rcu_read_lock_held() || PageLocked(page)));
Michal Hocko33398cf2015-09-08 15:01:02 -0700515
Johannes Weiner62cccb82016-03-15 14:57:22 -0700516 if (page->mem_cgroup)
517 this_cpu_add(page->mem_cgroup->stat->count[idx], val);
Michal Hocko33398cf2015-09-08 15:01:02 -0700518}
519
Johannes Weiner62cccb82016-03-15 14:57:22 -0700520static inline void mem_cgroup_inc_page_stat(struct page *page,
Sha Zhengju68b48762013-09-12 15:13:50 -0700521 enum mem_cgroup_stat_index idx)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800522{
Johannes Weiner62cccb82016-03-15 14:57:22 -0700523 mem_cgroup_update_page_stat(page, idx, 1);
Greg Thelen2a7106f2011-01-13 15:47:37 -0800524}
525
Johannes Weiner62cccb82016-03-15 14:57:22 -0700526static inline void mem_cgroup_dec_page_stat(struct page *page,
Sha Zhengju68b48762013-09-12 15:13:50 -0700527 enum mem_cgroup_stat_index idx)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800528{
Johannes Weiner62cccb82016-03-15 14:57:22 -0700529 mem_cgroup_update_page_stat(page, idx, -1);
Greg Thelen2a7106f2011-01-13 15:47:37 -0800530}
531
Mel Gormanef8f2322016-07-28 15:46:05 -0700532unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -0700533 gfp_t gfp_mask,
534 unsigned long *total_scanned);
David Rientjesa63d83f2010-08-09 17:19:46 -0700535
David Rientjes68ae5642012-12-12 13:51:57 -0800536static inline void mem_cgroup_count_vm_event(struct mm_struct *mm,
537 enum vm_event_item idx)
538{
Michal Hocko33398cf2015-09-08 15:01:02 -0700539 struct mem_cgroup *memcg;
540
David Rientjes68ae5642012-12-12 13:51:57 -0800541 if (mem_cgroup_disabled())
542 return;
Michal Hocko33398cf2015-09-08 15:01:02 -0700543
544 rcu_read_lock();
545 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
546 if (unlikely(!memcg))
547 goto out;
548
549 switch (idx) {
550 case PGFAULT:
551 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
552 break;
553 case PGMAJFAULT:
554 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
555 break;
556 default:
557 BUG();
558 }
559out:
560 rcu_read_unlock();
David Rientjes68ae5642012-12-12 13:51:57 -0800561}
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800562#ifdef CONFIG_TRANSPARENT_HUGEPAGE
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -0800563void mem_cgroup_split_huge_fixup(struct page *head);
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800564#endif
565
Andrew Mortonc255a452012-07-31 16:43:02 -0700566#else /* CONFIG_MEMCG */
Johannes Weiner23047a92016-03-15 14:57:16 -0700567
568#define MEM_CGROUP_ID_SHIFT 0
569#define MEM_CGROUP_ID_MAX 0
570
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800571struct mem_cgroup;
572
Johannes Weiner23047a92016-03-15 14:57:16 -0700573static inline bool mem_cgroup_disabled(void)
574{
575 return true;
576}
577
Johannes Weiner241994ed2015-02-11 15:26:06 -0800578static inline void mem_cgroup_events(struct mem_cgroup *memcg,
579 enum mem_cgroup_events_index idx,
580 unsigned int nr)
581{
582}
583
584static inline bool mem_cgroup_low(struct mem_cgroup *root,
585 struct mem_cgroup *memcg)
586{
587 return false;
588}
589
Johannes Weiner00501b52014-08-08 14:19:20 -0700590static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
591 gfp_t gfp_mask,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800592 struct mem_cgroup **memcgp,
593 bool compound)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800594{
Johannes Weiner00501b52014-08-08 14:19:20 -0700595 *memcgp = NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800596 return 0;
597}
598
Johannes Weiner00501b52014-08-08 14:19:20 -0700599static inline void mem_cgroup_commit_charge(struct page *page,
600 struct mem_cgroup *memcg,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800601 bool lrucare, bool compound)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800602{
603}
604
Johannes Weiner00501b52014-08-08 14:19:20 -0700605static inline void mem_cgroup_cancel_charge(struct page *page,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800606 struct mem_cgroup *memcg,
607 bool compound)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800608{
609}
610
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700611static inline void mem_cgroup_uncharge(struct page *page)
612{
613}
614
Johannes Weiner747db952014-08-08 14:19:24 -0700615static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -0800616{
617}
618
Johannes Weiner6a93ca82016-03-15 14:57:19 -0700619static inline void mem_cgroup_migrate(struct page *old, struct page *new)
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700620{
621}
622
Mel Gormana9dd0a82016-07-28 15:46:02 -0700623static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
Mel Gormanef8f2322016-07-28 15:46:05 -0700624 struct mem_cgroup *memcg)
Johannes Weiner925b7672012-01-12 17:18:15 -0800625{
Mel Gormana9dd0a82016-07-28 15:46:02 -0700626 return node_lruvec(pgdat);
Johannes Weiner925b7672012-01-12 17:18:15 -0800627}
628
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700629static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
Mel Gorman599d0c92016-07-28 15:45:31 -0700630 struct pglist_data *pgdat)
Minchan Kim3f58a822011-03-22 16:32:53 -0700631{
Mel Gorman599d0c92016-07-28 15:45:31 -0700632 return &pgdat->lruvec;
Balbir Singh66e17072008-02-07 00:13:56 -0800633}
634
Johannes Weiner587af302012-10-08 16:34:12 -0700635static inline bool mm_match_cgroup(struct mm_struct *mm,
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700636 struct mem_cgroup *memcg)
Balbir Singhbed71612008-02-07 00:14:01 -0800637{
Johannes Weiner587af302012-10-08 16:34:12 -0700638 return true;
Balbir Singhbed71612008-02-07 00:14:01 -0800639}
640
David Rientjesffbdccf2013-07-03 15:01:23 -0700641static inline bool task_in_mem_cgroup(struct task_struct *task,
642 const struct mem_cgroup *memcg)
David Rientjes4c4a2212008-02-07 00:14:06 -0800643{
David Rientjesffbdccf2013-07-03 15:01:23 -0700644 return true;
David Rientjes4c4a2212008-02-07 00:14:06 -0800645}
646
Johannes Weiner56600482012-01-12 17:17:59 -0800647static inline struct mem_cgroup *
648mem_cgroup_iter(struct mem_cgroup *root,
649 struct mem_cgroup *prev,
650 struct mem_cgroup_reclaim_cookie *reclaim)
651{
652 return NULL;
653}
654
655static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
656 struct mem_cgroup *prev)
657{
658}
659
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700660static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
661 int (*fn)(struct task_struct *, void *), void *arg)
662{
663 return 0;
664}
665
Johannes Weiner23047a92016-03-15 14:57:16 -0700666static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800667{
Johannes Weiner23047a92016-03-15 14:57:16 -0700668 return 0;
669}
670
671static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
672{
673 WARN_ON_ONCE(id);
674 /* XXX: This should always return root_mem_cgroup */
675 return NULL;
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800676}
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -0800677
Vladimir Davydoveb01aaa2016-01-20 15:03:02 -0800678static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800679{
Yaowei Bai13308ca2015-11-05 18:47:40 -0800680 return true;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800681}
682
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800683static inline unsigned long
Hugh Dickins4d7dcca2012-05-29 15:07:08 -0700684mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800685{
686 return 0;
687}
Michal Hockob4536f0c82017-01-10 16:58:04 -0800688static inline
689unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
690 enum lru_list lru, int zone_idx)
691{
692 return 0;
693}
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800694
Vladimir Davydov0a6b76d2016-03-17 14:18:42 -0700695static inline unsigned long
696mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
697 int nid, unsigned int lru_mask)
698{
699 return 0;
700}
701
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700702static inline unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
703{
704 return 0;
705}
706
Balbir Singhe2224322009-04-02 16:57:39 -0700707static inline void
708mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
709{
710}
711
Johannes Weiner62cccb82016-03-15 14:57:22 -0700712static inline void lock_page_memcg(struct page *page)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -0700713{
714}
715
Johannes Weiner62cccb82016-03-15 14:57:22 -0700716static inline void unlock_page_memcg(struct page *page)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -0700717{
718}
719
Tejun Heob23afb92015-11-05 18:46:11 -0800720static inline void mem_cgroup_handle_over_high(void)
721{
722}
723
Johannes Weiner49426422013-10-16 13:46:59 -0700724static inline void mem_cgroup_oom_enable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700725{
726}
727
Johannes Weiner49426422013-10-16 13:46:59 -0700728static inline void mem_cgroup_oom_disable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700729{
730}
731
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700732static inline bool task_in_memcg_oom(struct task_struct *p)
733{
734 return false;
735}
736
Johannes Weiner49426422013-10-16 13:46:59 -0700737static inline bool mem_cgroup_oom_synchronize(bool wait)
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700738{
739 return false;
740}
741
Johannes Weiner62cccb82016-03-15 14:57:22 -0700742static inline void mem_cgroup_inc_page_stat(struct page *page,
Sha Zhengju68b48762013-09-12 15:13:50 -0700743 enum mem_cgroup_stat_index idx)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800744{
745}
746
Johannes Weiner62cccb82016-03-15 14:57:22 -0700747static inline void mem_cgroup_dec_page_stat(struct page *page,
Sha Zhengju68b48762013-09-12 15:13:50 -0700748 enum mem_cgroup_stat_index idx)
Balbir Singhd69b0422009-06-17 16:26:34 -0700749{
750}
751
Balbir Singh4e416952009-09-23 15:56:39 -0700752static inline
Mel Gormanef8f2322016-07-28 15:46:05 -0700753unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -0700754 gfp_t gfp_mask,
755 unsigned long *total_scanned)
Balbir Singh4e416952009-09-23 15:56:39 -0700756{
Andrew Morton0608f432013-09-24 15:27:41 -0700757 return 0;
Balbir Singh4e416952009-09-23 15:56:39 -0700758}
759
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -0800760static inline void mem_cgroup_split_huge_fixup(struct page *head)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800761{
762}
763
Ying Han456f9982011-05-26 16:25:38 -0700764static inline
765void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
766{
767}
Andrew Mortonc255a452012-07-31 16:43:02 -0700768#endif /* CONFIG_MEMCG */
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800769
Tejun Heo52ebea72015-05-22 17:13:37 -0400770#ifdef CONFIG_CGROUP_WRITEBACK
Tejun Heo841710a2015-05-22 18:23:33 -0400771
Tejun Heo52ebea72015-05-22 17:13:37 -0400772struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
Tejun Heo841710a2015-05-22 18:23:33 -0400773struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
Tejun Heoc5edf9c2015-09-29 13:04:26 -0400774void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
775 unsigned long *pheadroom, unsigned long *pdirty,
776 unsigned long *pwriteback);
Tejun Heo841710a2015-05-22 18:23:33 -0400777
778#else /* CONFIG_CGROUP_WRITEBACK */
779
780static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
781{
782 return NULL;
783}
784
Tejun Heoc2aa7232015-05-22 18:23:35 -0400785static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
Tejun Heoc5edf9c2015-09-29 13:04:26 -0400786 unsigned long *pfilepages,
787 unsigned long *pheadroom,
Tejun Heoc2aa7232015-05-22 18:23:35 -0400788 unsigned long *pdirty,
789 unsigned long *pwriteback)
790{
791}
792
Tejun Heo841710a2015-05-22 18:23:33 -0400793#endif /* CONFIG_CGROUP_WRITEBACK */
Tejun Heo52ebea72015-05-22 17:13:37 -0400794
Glauber Costae1aab162011-12-11 21:47:03 +0000795struct sock;
Johannes Weinerbaac50b2016-01-14 15:21:17 -0800796bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
797void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
Johannes Weinerd886f4e2016-01-20 15:02:47 -0800798#ifdef CONFIG_MEMCG
Johannes Weineref129472016-01-14 15:21:34 -0800799extern struct static_key_false memcg_sockets_enabled_key;
800#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
Johannes Weiner2d758072016-10-07 17:00:58 -0700801void mem_cgroup_sk_alloc(struct sock *sk);
802void mem_cgroup_sk_free(struct sock *sk);
Johannes Weinerbaac50b2016-01-14 15:21:17 -0800803static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
Johannes Weinere8056052016-01-14 15:21:14 -0800804{
Johannes Weiner0db15292016-01-20 15:02:50 -0800805 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800806 return true;
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800807 do {
808 if (time_before(jiffies, memcg->socket_pressure))
809 return true;
810 } while ((memcg = parent_mem_cgroup(memcg)));
811 return false;
Johannes Weinere8056052016-01-14 15:21:14 -0800812}
813#else
Johannes Weiner80e95fe2016-01-14 15:21:20 -0800814#define mem_cgroup_sockets_enabled 0
Johannes Weiner2d758072016-10-07 17:00:58 -0700815static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
816static inline void mem_cgroup_sk_free(struct sock *sk) { };
Johannes Weinerbaac50b2016-01-14 15:21:17 -0800817static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
Johannes Weinere8056052016-01-14 15:21:14 -0800818{
819 return false;
820}
821#endif
Glauber Costa7ae1e1d2012-12-18 14:21:56 -0800822
Vladimir Davydov45264772016-07-26 15:24:21 -0700823struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
824void memcg_kmem_put_cache(struct kmem_cache *cachep);
825int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
826 struct mem_cgroup *memcg);
827int memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
828void memcg_kmem_uncharge(struct page *page, int order);
829
Johannes Weiner127424c2016-01-20 15:02:32 -0800830#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
Johannes Weineref129472016-01-14 15:21:34 -0800831extern struct static_key_false memcg_kmem_enabled_key;
Glauber Costa749c5412012-12-18 14:23:01 -0800832
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800833extern int memcg_nr_cache_ids;
Michal Hocko64219992015-09-08 15:01:07 -0700834void memcg_get_cache_ids(void);
835void memcg_put_cache_ids(void);
Glauber Costaebe945c2012-12-18 14:23:10 -0800836
837/*
838 * Helper macro to loop through all memcg-specific caches. Callers must still
839 * check if the cache is valid (it is either valid or NULL).
840 * the slab_mutex must be held when looping through those caches
841 */
Glauber Costa749c5412012-12-18 14:23:01 -0800842#define for_each_memcg_cache_index(_idx) \
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800843 for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
Glauber Costa749c5412012-12-18 14:23:01 -0800844
Glauber Costa7ae1e1d2012-12-18 14:21:56 -0800845static inline bool memcg_kmem_enabled(void)
846{
Johannes Weineref129472016-01-14 15:21:34 -0800847 return static_branch_unlikely(&memcg_kmem_enabled_key);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -0800848}
849
850/*
Jesper Dangaard Brouer9f706d62016-03-15 14:54:03 -0700851 * helper for accessing a memcg's index. It will be used as an index in the
Michal Hocko33398cf2015-09-08 15:01:02 -0700852 * child cache array in kmem_cache, and also to derive its name. This function
853 * will return -1 when this is not a kmem-limited memcg.
854 */
855static inline int memcg_cache_id(struct mem_cgroup *memcg)
856{
857 return memcg ? memcg->kmemcg_id : -1;
858}
Vladimir Davydov5722d092014-04-07 15:39:24 -0700859
Vladimir Davydov27ee57c2016-03-17 14:17:35 -0700860/**
861 * memcg_kmem_update_page_stat - update kmem page state statistics
862 * @page: the page
863 * @idx: page state item to account
864 * @val: number of pages (positive or negative)
865 */
866static inline void memcg_kmem_update_page_stat(struct page *page,
867 enum mem_cgroup_stat_index idx, int val)
868{
869 if (memcg_kmem_enabled() && page->mem_cgroup)
870 this_cpu_add(page->mem_cgroup->stat->count[idx], val);
871}
872
Glauber Costa7ae1e1d2012-12-18 14:21:56 -0800873#else
Glauber Costa749c5412012-12-18 14:23:01 -0800874#define for_each_memcg_cache_index(_idx) \
875 for (; NULL; )
876
Glauber Costab9ce5ef2012-12-18 14:22:46 -0800877static inline bool memcg_kmem_enabled(void)
878{
879 return false;
880}
881
Glauber Costa2633d7a2012-12-18 14:22:34 -0800882static inline int memcg_cache_id(struct mem_cgroup *memcg)
883{
884 return -1;
885}
886
Vladimir Davydov05257a12015-02-12 14:59:01 -0800887static inline void memcg_get_cache_ids(void)
888{
889}
890
891static inline void memcg_put_cache_ids(void)
892{
893}
894
Vladimir Davydov27ee57c2016-03-17 14:17:35 -0700895static inline void memcg_kmem_update_page_stat(struct page *page,
896 enum mem_cgroup_stat_index idx, int val)
897{
898}
Johannes Weiner127424c2016-01-20 15:02:32 -0800899#endif /* CONFIG_MEMCG && !CONFIG_SLOB */
900
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800901#endif /* _LINUX_MEMCONTROL_H */