blob: 1d05cb9d363d0bfadd6a9c58efccd0551d0bd7f2 [file] [log] [blame]
Dave Chinnera38e4082013-08-28 10:17:58 +10001/*
2 * Copyright (c) 2013 Red Hat, Inc. and Parallels Inc. All rights reserved.
3 * Authors: David Chinner and Glauber Costa
4 *
5 * Generic LRU infrastructure
6 */
7#include <linux/kernel.h>
8#include <linux/module.h>
Dave Chinner3b1d58a2013-08-28 10:18:00 +10009#include <linux/mm.h>
Dave Chinnera38e4082013-08-28 10:17:58 +100010#include <linux/list_lru.h>
Glauber Costa5ca302c2013-08-28 10:18:18 +100011#include <linux/slab.h>
Vladimir Davydovc0a5b562015-02-12 14:59:07 -080012#include <linux/mutex.h>
Vladimir Davydov60d3fd32015-02-12 14:59:10 -080013#include <linux/memcontrol.h>
Vladimir Davydovc0a5b562015-02-12 14:59:07 -080014
Johannes Weiner127424c2016-01-20 15:02:32 -080015#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
Vladimir Davydovc0a5b562015-02-12 14:59:07 -080016static LIST_HEAD(list_lrus);
17static DEFINE_MUTEX(list_lrus_mutex);
18
19static void list_lru_register(struct list_lru *lru)
20{
21 mutex_lock(&list_lrus_mutex);
22 list_add(&lru->list, &list_lrus);
23 mutex_unlock(&list_lrus_mutex);
24}
25
26static void list_lru_unregister(struct list_lru *lru)
27{
28 mutex_lock(&list_lrus_mutex);
29 list_del(&lru->list);
30 mutex_unlock(&list_lrus_mutex);
31}
32#else
33static void list_lru_register(struct list_lru *lru)
34{
35}
36
37static void list_lru_unregister(struct list_lru *lru)
38{
39}
Johannes Weiner127424c2016-01-20 15:02:32 -080040#endif /* CONFIG_MEMCG && !CONFIG_SLOB */
Dave Chinnera38e4082013-08-28 10:17:58 +100041
Johannes Weiner127424c2016-01-20 15:02:32 -080042#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
Vladimir Davydov60d3fd32015-02-12 14:59:10 -080043static inline bool list_lru_memcg_aware(struct list_lru *lru)
44{
Raghavendra K T145949a2015-11-05 18:46:26 -080045 /*
46 * This needs node 0 to be always present, even
47 * in the systems supporting sparse numa ids.
48 */
Vladimir Davydov60d3fd32015-02-12 14:59:10 -080049 return !!lru->node[0].memcg_lrus;
50}
51
52static inline struct list_lru_one *
53list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)
54{
55 /*
56 * The lock protects the array of per cgroup lists from relocation
57 * (see memcg_update_list_lru_node).
58 */
59 lockdep_assert_held(&nlru->lock);
60 if (nlru->memcg_lrus && idx >= 0)
61 return nlru->memcg_lrus->lru[idx];
62
63 return &nlru->lru;
64}
65
Vladimir Davydovdf406552015-11-05 18:49:04 -080066static __always_inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr)
67{
68 struct page *page;
69
70 if (!memcg_kmem_enabled())
71 return NULL;
72 page = virt_to_head_page(ptr);
73 return page->mem_cgroup;
74}
75
Vladimir Davydov60d3fd32015-02-12 14:59:10 -080076static inline struct list_lru_one *
77list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)
78{
79 struct mem_cgroup *memcg;
80
81 if (!nlru->memcg_lrus)
82 return &nlru->lru;
83
84 memcg = mem_cgroup_from_kmem(ptr);
85 if (!memcg)
86 return &nlru->lru;
87
88 return list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));
89}
90#else
91static inline bool list_lru_memcg_aware(struct list_lru *lru)
92{
93 return false;
94}
95
96static inline struct list_lru_one *
97list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)
98{
99 return &nlru->lru;
100}
101
102static inline struct list_lru_one *
103list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)
104{
105 return &nlru->lru;
106}
Johannes Weiner127424c2016-01-20 15:02:32 -0800107#endif /* CONFIG_MEMCG && !CONFIG_SLOB */
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800108
Dave Chinnera38e4082013-08-28 10:17:58 +1000109bool list_lru_add(struct list_lru *lru, struct list_head *item)
110{
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000111 int nid = page_to_nid(virt_to_page(item));
112 struct list_lru_node *nlru = &lru->node[nid];
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800113 struct list_lru_one *l;
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000114
115 spin_lock(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000116 if (list_empty(item)) {
Jeff Layton26f5d762015-09-08 15:03:44 -0700117 l = list_lru_from_kmem(nlru, item);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800118 list_add_tail(item, &l->list);
119 l->nr_items++;
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000120 spin_unlock(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000121 return true;
122 }
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000123 spin_unlock(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000124 return false;
125}
126EXPORT_SYMBOL_GPL(list_lru_add);
127
128bool list_lru_del(struct list_lru *lru, struct list_head *item)
129{
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000130 int nid = page_to_nid(virt_to_page(item));
131 struct list_lru_node *nlru = &lru->node[nid];
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800132 struct list_lru_one *l;
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000133
134 spin_lock(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000135 if (!list_empty(item)) {
Jeff Layton26f5d762015-09-08 15:03:44 -0700136 l = list_lru_from_kmem(nlru, item);
Dave Chinnera38e4082013-08-28 10:17:58 +1000137 list_del_init(item);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800138 l->nr_items--;
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000139 spin_unlock(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000140 return true;
141 }
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000142 spin_unlock(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000143 return false;
144}
145EXPORT_SYMBOL_GPL(list_lru_del);
146
Vladimir Davydov3f97b162015-02-12 14:59:35 -0800147void list_lru_isolate(struct list_lru_one *list, struct list_head *item)
148{
149 list_del_init(item);
150 list->nr_items--;
151}
152EXPORT_SYMBOL_GPL(list_lru_isolate);
153
154void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item,
155 struct list_head *head)
156{
157 list_move(item, head);
158 list->nr_items--;
159}
160EXPORT_SYMBOL_GPL(list_lru_isolate_move);
161
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800162static unsigned long __list_lru_count_one(struct list_lru *lru,
163 int nid, int memcg_idx)
Dave Chinnera38e4082013-08-28 10:17:58 +1000164{
Glauber Costa6a4f4962013-08-28 10:18:02 +1000165 struct list_lru_node *nlru = &lru->node[nid];
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800166 struct list_lru_one *l;
167 unsigned long count;
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000168
Glauber Costa6a4f4962013-08-28 10:18:02 +1000169 spin_lock(&nlru->lock);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800170 l = list_lru_from_memcg_idx(nlru, memcg_idx);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800171 count = l->nr_items;
Glauber Costa6a4f4962013-08-28 10:18:02 +1000172 spin_unlock(&nlru->lock);
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000173
174 return count;
175}
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800176
177unsigned long list_lru_count_one(struct list_lru *lru,
178 int nid, struct mem_cgroup *memcg)
179{
180 return __list_lru_count_one(lru, nid, memcg_cache_id(memcg));
181}
182EXPORT_SYMBOL_GPL(list_lru_count_one);
183
184unsigned long list_lru_count_node(struct list_lru *lru, int nid)
185{
186 long count = 0;
187 int memcg_idx;
188
189 count += __list_lru_count_one(lru, nid, -1);
190 if (list_lru_memcg_aware(lru)) {
191 for_each_memcg_cache_index(memcg_idx)
192 count += __list_lru_count_one(lru, nid, memcg_idx);
193 }
194 return count;
195}
Glauber Costa6a4f4962013-08-28 10:18:02 +1000196EXPORT_SYMBOL_GPL(list_lru_count_node);
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000197
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800198static unsigned long
199__list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
200 list_lru_walk_cb isolate, void *cb_arg,
201 unsigned long *nr_to_walk)
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000202{
203
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800204 struct list_lru_node *nlru = &lru->node[nid];
205 struct list_lru_one *l;
Dave Chinnera38e4082013-08-28 10:17:58 +1000206 struct list_head *item, *n;
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000207 unsigned long isolated = 0;
Dave Chinnera38e4082013-08-28 10:17:58 +1000208
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000209 spin_lock(&nlru->lock);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800210 l = list_lru_from_memcg_idx(nlru, memcg_idx);
Dave Chinnera38e4082013-08-28 10:17:58 +1000211restart:
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800212 list_for_each_safe(item, n, &l->list) {
Dave Chinnera38e4082013-08-28 10:17:58 +1000213 enum lru_status ret;
Dave Chinner5cedf7212013-08-28 10:18:01 +1000214
215 /*
216 * decrement nr_to_walk first so that we don't livelock if we
217 * get stuck on large numbesr of LRU_RETRY items
218 */
Russell Kingc56b0972013-10-30 14:16:16 +0000219 if (!*nr_to_walk)
Dave Chinner5cedf7212013-08-28 10:18:01 +1000220 break;
Russell Kingc56b0972013-10-30 14:16:16 +0000221 --*nr_to_walk;
Dave Chinner5cedf7212013-08-28 10:18:01 +1000222
Vladimir Davydov3f97b162015-02-12 14:59:35 -0800223 ret = isolate(item, l, &nlru->lock, cb_arg);
Dave Chinnera38e4082013-08-28 10:17:58 +1000224 switch (ret) {
Johannes Weiner449dd692014-04-03 14:47:56 -0700225 case LRU_REMOVED_RETRY:
226 assert_spin_locked(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000227 case LRU_REMOVED:
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000228 isolated++;
Johannes Weiner449dd692014-04-03 14:47:56 -0700229 /*
230 * If the lru lock has been dropped, our list
231 * traversal is now invalid and so we have to
232 * restart from scratch.
233 */
234 if (ret == LRU_REMOVED_RETRY)
235 goto restart;
Dave Chinnera38e4082013-08-28 10:17:58 +1000236 break;
237 case LRU_ROTATE:
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800238 list_move_tail(item, &l->list);
Dave Chinnera38e4082013-08-28 10:17:58 +1000239 break;
240 case LRU_SKIP:
241 break;
242 case LRU_RETRY:
Dave Chinner5cedf7212013-08-28 10:18:01 +1000243 /*
244 * The lru lock has been dropped, our list traversal is
245 * now invalid and so we have to restart from scratch.
246 */
Johannes Weiner449dd692014-04-03 14:47:56 -0700247 assert_spin_locked(&nlru->lock);
Dave Chinnera38e4082013-08-28 10:17:58 +1000248 goto restart;
249 default:
250 BUG();
251 }
Dave Chinnera38e4082013-08-28 10:17:58 +1000252 }
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000253
254 spin_unlock(&nlru->lock);
255 return isolated;
256}
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800257
258unsigned long
259list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
260 list_lru_walk_cb isolate, void *cb_arg,
261 unsigned long *nr_to_walk)
262{
263 return __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),
264 isolate, cb_arg, nr_to_walk);
265}
266EXPORT_SYMBOL_GPL(list_lru_walk_one);
267
268unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
269 list_lru_walk_cb isolate, void *cb_arg,
270 unsigned long *nr_to_walk)
271{
272 long isolated = 0;
273 int memcg_idx;
274
275 isolated += __list_lru_walk_one(lru, nid, -1, isolate, cb_arg,
276 nr_to_walk);
277 if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {
278 for_each_memcg_cache_index(memcg_idx) {
279 isolated += __list_lru_walk_one(lru, nid, memcg_idx,
280 isolate, cb_arg, nr_to_walk);
281 if (*nr_to_walk <= 0)
282 break;
283 }
284 }
285 return isolated;
286}
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000287EXPORT_SYMBOL_GPL(list_lru_walk_node);
288
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800289static void init_one_lru(struct list_lru_one *l)
290{
291 INIT_LIST_HEAD(&l->list);
292 l->nr_items = 0;
293}
294
Johannes Weiner127424c2016-01-20 15:02:32 -0800295#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800296static void __memcg_destroy_list_lru_node(struct list_lru_memcg *memcg_lrus,
297 int begin, int end)
298{
299 int i;
300
301 for (i = begin; i < end; i++)
302 kfree(memcg_lrus->lru[i]);
303}
304
305static int __memcg_init_list_lru_node(struct list_lru_memcg *memcg_lrus,
306 int begin, int end)
307{
308 int i;
309
310 for (i = begin; i < end; i++) {
311 struct list_lru_one *l;
312
313 l = kmalloc(sizeof(struct list_lru_one), GFP_KERNEL);
314 if (!l)
315 goto fail;
316
317 init_one_lru(l);
318 memcg_lrus->lru[i] = l;
319 }
320 return 0;
321fail:
322 __memcg_destroy_list_lru_node(memcg_lrus, begin, i - 1);
323 return -ENOMEM;
324}
325
326static int memcg_init_list_lru_node(struct list_lru_node *nlru)
327{
328 int size = memcg_nr_cache_ids;
329
330 nlru->memcg_lrus = kmalloc(size * sizeof(void *), GFP_KERNEL);
331 if (!nlru->memcg_lrus)
332 return -ENOMEM;
333
334 if (__memcg_init_list_lru_node(nlru->memcg_lrus, 0, size)) {
335 kfree(nlru->memcg_lrus);
336 return -ENOMEM;
337 }
338
339 return 0;
340}
341
342static void memcg_destroy_list_lru_node(struct list_lru_node *nlru)
343{
344 __memcg_destroy_list_lru_node(nlru->memcg_lrus, 0, memcg_nr_cache_ids);
345 kfree(nlru->memcg_lrus);
346}
347
348static int memcg_update_list_lru_node(struct list_lru_node *nlru,
349 int old_size, int new_size)
350{
351 struct list_lru_memcg *old, *new;
352
353 BUG_ON(old_size > new_size);
354
355 old = nlru->memcg_lrus;
356 new = kmalloc(new_size * sizeof(void *), GFP_KERNEL);
357 if (!new)
358 return -ENOMEM;
359
360 if (__memcg_init_list_lru_node(new, old_size, new_size)) {
361 kfree(new);
362 return -ENOMEM;
363 }
364
365 memcpy(new, old, old_size * sizeof(void *));
366
367 /*
368 * The lock guarantees that we won't race with a reader
369 * (see list_lru_from_memcg_idx).
370 *
371 * Since list_lru_{add,del} may be called under an IRQ-safe lock,
372 * we have to use IRQ-safe primitives here to avoid deadlock.
373 */
374 spin_lock_irq(&nlru->lock);
375 nlru->memcg_lrus = new;
376 spin_unlock_irq(&nlru->lock);
377
378 kfree(old);
379 return 0;
380}
381
382static void memcg_cancel_update_list_lru_node(struct list_lru_node *nlru,
383 int old_size, int new_size)
384{
385 /* do not bother shrinking the array back to the old size, because we
386 * cannot handle allocation failures here */
387 __memcg_destroy_list_lru_node(nlru->memcg_lrus, old_size, new_size);
388}
389
390static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
391{
392 int i;
393
Raghavendra K T145949a2015-11-05 18:46:26 -0800394 if (!memcg_aware)
395 return 0;
396
397 for_each_node(i) {
398 if (memcg_init_list_lru_node(&lru->node[i]))
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800399 goto fail;
400 }
401 return 0;
402fail:
Raghavendra K T145949a2015-11-05 18:46:26 -0800403 for (i = i - 1; i >= 0; i--) {
404 if (!lru->node[i].memcg_lrus)
405 continue;
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800406 memcg_destroy_list_lru_node(&lru->node[i]);
Raghavendra K T145949a2015-11-05 18:46:26 -0800407 }
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800408 return -ENOMEM;
409}
410
411static void memcg_destroy_list_lru(struct list_lru *lru)
412{
413 int i;
414
415 if (!list_lru_memcg_aware(lru))
416 return;
417
Raghavendra K T145949a2015-11-05 18:46:26 -0800418 for_each_node(i)
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800419 memcg_destroy_list_lru_node(&lru->node[i]);
420}
421
422static int memcg_update_list_lru(struct list_lru *lru,
423 int old_size, int new_size)
424{
425 int i;
426
427 if (!list_lru_memcg_aware(lru))
428 return 0;
429
Raghavendra K T145949a2015-11-05 18:46:26 -0800430 for_each_node(i) {
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800431 if (memcg_update_list_lru_node(&lru->node[i],
432 old_size, new_size))
433 goto fail;
434 }
435 return 0;
436fail:
Raghavendra K T145949a2015-11-05 18:46:26 -0800437 for (i = i - 1; i >= 0; i--) {
438 if (!lru->node[i].memcg_lrus)
439 continue;
440
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800441 memcg_cancel_update_list_lru_node(&lru->node[i],
442 old_size, new_size);
Raghavendra K T145949a2015-11-05 18:46:26 -0800443 }
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800444 return -ENOMEM;
445}
446
447static void memcg_cancel_update_list_lru(struct list_lru *lru,
448 int old_size, int new_size)
449{
450 int i;
451
452 if (!list_lru_memcg_aware(lru))
453 return;
454
Raghavendra K T145949a2015-11-05 18:46:26 -0800455 for_each_node(i)
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800456 memcg_cancel_update_list_lru_node(&lru->node[i],
457 old_size, new_size);
458}
459
460int memcg_update_all_list_lrus(int new_size)
461{
462 int ret = 0;
463 struct list_lru *lru;
464 int old_size = memcg_nr_cache_ids;
465
466 mutex_lock(&list_lrus_mutex);
467 list_for_each_entry(lru, &list_lrus, list) {
468 ret = memcg_update_list_lru(lru, old_size, new_size);
469 if (ret)
470 goto fail;
471 }
472out:
473 mutex_unlock(&list_lrus_mutex);
474 return ret;
475fail:
476 list_for_each_entry_continue_reverse(lru, &list_lrus, list)
477 memcg_cancel_update_list_lru(lru, old_size, new_size);
478 goto out;
479}
Vladimir Davydov2788cf02015-02-12 14:59:38 -0800480
481static void memcg_drain_list_lru_node(struct list_lru_node *nlru,
482 int src_idx, int dst_idx)
483{
484 struct list_lru_one *src, *dst;
485
486 /*
487 * Since list_lru_{add,del} may be called under an IRQ-safe lock,
488 * we have to use IRQ-safe primitives here to avoid deadlock.
489 */
490 spin_lock_irq(&nlru->lock);
491
492 src = list_lru_from_memcg_idx(nlru, src_idx);
493 dst = list_lru_from_memcg_idx(nlru, dst_idx);
494
495 list_splice_init(&src->list, &dst->list);
496 dst->nr_items += src->nr_items;
497 src->nr_items = 0;
498
499 spin_unlock_irq(&nlru->lock);
500}
501
502static void memcg_drain_list_lru(struct list_lru *lru,
503 int src_idx, int dst_idx)
504{
505 int i;
506
507 if (!list_lru_memcg_aware(lru))
508 return;
509
Raghavendra K T145949a2015-11-05 18:46:26 -0800510 for_each_node(i)
Vladimir Davydov2788cf02015-02-12 14:59:38 -0800511 memcg_drain_list_lru_node(&lru->node[i], src_idx, dst_idx);
512}
513
514void memcg_drain_all_list_lrus(int src_idx, int dst_idx)
515{
516 struct list_lru *lru;
517
518 mutex_lock(&list_lrus_mutex);
519 list_for_each_entry(lru, &list_lrus, list)
520 memcg_drain_list_lru(lru, src_idx, dst_idx);
521 mutex_unlock(&list_lrus_mutex);
522}
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800523#else
524static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
525{
526 return 0;
527}
528
529static void memcg_destroy_list_lru(struct list_lru *lru)
530{
531}
Johannes Weiner127424c2016-01-20 15:02:32 -0800532#endif /* CONFIG_MEMCG && !CONFIG_SLOB */
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800533
534int __list_lru_init(struct list_lru *lru, bool memcg_aware,
535 struct lock_class_key *key)
Dave Chinnera38e4082013-08-28 10:17:58 +1000536{
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000537 int i;
Glauber Costa5ca302c2013-08-28 10:18:18 +1000538 size_t size = sizeof(*lru->node) * nr_node_ids;
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800539 int err = -ENOMEM;
540
541 memcg_get_cache_ids();
Glauber Costa5ca302c2013-08-28 10:18:18 +1000542
543 lru->node = kzalloc(size, GFP_KERNEL);
544 if (!lru->node)
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800545 goto out;
Dave Chinnera38e4082013-08-28 10:17:58 +1000546
Raghavendra K T145949a2015-11-05 18:46:26 -0800547 for_each_node(i) {
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000548 spin_lock_init(&lru->node[i].lock);
Johannes Weiner449dd692014-04-03 14:47:56 -0700549 if (key)
550 lockdep_set_class(&lru->node[i].lock, key);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800551 init_one_lru(&lru->node[i].lru);
Dave Chinner3b1d58a2013-08-28 10:18:00 +1000552 }
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800553
554 err = memcg_init_list_lru(lru, memcg_aware);
555 if (err) {
556 kfree(lru->node);
557 goto out;
558 }
559
Vladimir Davydovc0a5b562015-02-12 14:59:07 -0800560 list_lru_register(lru);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800561out:
562 memcg_put_cache_ids();
563 return err;
Dave Chinnera38e4082013-08-28 10:17:58 +1000564}
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800565EXPORT_SYMBOL_GPL(__list_lru_init);
Glauber Costa5ca302c2013-08-28 10:18:18 +1000566
567void list_lru_destroy(struct list_lru *lru)
568{
Vladimir Davydovc0a5b562015-02-12 14:59:07 -0800569 /* Already destroyed or not yet initialized? */
570 if (!lru->node)
571 return;
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800572
573 memcg_get_cache_ids();
574
Vladimir Davydovc0a5b562015-02-12 14:59:07 -0800575 list_lru_unregister(lru);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800576
577 memcg_destroy_list_lru(lru);
Glauber Costa5ca302c2013-08-28 10:18:18 +1000578 kfree(lru->node);
Vladimir Davydovc0a5b562015-02-12 14:59:07 -0800579 lru->node = NULL;
Vladimir Davydov60d3fd32015-02-12 14:59:10 -0800580
581 memcg_put_cache_ids();
Glauber Costa5ca302c2013-08-28 10:18:18 +1000582}
583EXPORT_SYMBOL_GPL(list_lru_destroy);