memcg: free memcg_caches slot on css offline
We need to look up a kmem_cache in ->memcg_params.memcg_caches arrays only
on allocations, so there is no need to have the array entries set until
css free - we can clear them on css offline. This will allow us to reuse
array entries more efficiently and avoid costly array relocations.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Dave Chinner <david@fromorbit.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 6087b1f..0873bcc 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -440,18 +440,8 @@
*need_rcu_barrier = true;
#ifdef CONFIG_MEMCG_KMEM
- if (!is_root_cache(s)) {
- int idx;
- struct memcg_cache_array *arr;
-
- idx = memcg_cache_id(s->memcg_params.memcg);
- arr = rcu_dereference_protected(s->memcg_params.root_cache->
- memcg_params.memcg_caches,
- lockdep_is_held(&slab_mutex));
- BUG_ON(arr->entries[idx] != s);
- arr->entries[idx] = NULL;
+ if (!is_root_cache(s))
list_del(&s->memcg_params.list);
- }
#endif
list_move(&s->list, release);
return 0;
@@ -499,6 +489,13 @@
mutex_lock(&slab_mutex);
+ /*
+ * The memory cgroup could have been deactivated while the cache
+ * creation work was pending.
+ */
+ if (!memcg_kmem_is_active(memcg))
+ goto out_unlock;
+
idx = memcg_cache_id(memcg);
arr = rcu_dereference_protected(root_cache->memcg_params.memcg_caches,
lockdep_is_held(&slab_mutex));
@@ -548,6 +545,26 @@
put_online_cpus();
}
+void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg)
+{
+ int idx;
+ struct memcg_cache_array *arr;
+ struct kmem_cache *s;
+
+ idx = memcg_cache_id(memcg);
+
+ mutex_lock(&slab_mutex);
+ list_for_each_entry(s, &slab_caches, list) {
+ if (!is_root_cache(s))
+ continue;
+
+ arr = rcu_dereference_protected(s->memcg_params.memcg_caches,
+ lockdep_is_held(&slab_mutex));
+ arr->entries[idx] = NULL;
+ }
+ mutex_unlock(&slab_mutex);
+}
+
void memcg_destroy_kmem_caches(struct mem_cgroup *memcg)
{
LIST_HEAD(release);