blob: f570f387034dd53b446f26be5564f7c8ea6005ec [file] [log] [blame]
Vivek Goyal31e4c282009-12-03 12:59:42 -05001/*
2 * Common Block IO controller cgroup interface
3 *
4 * Based on ideas and code from CFQ, CFS and BFQ:
5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
6 *
7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8 * Paolo Valente <paolo.valente@unimore.it>
9 *
10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11 * Nauman Rafique <nauman@google.com>
Arianna Avanzinie48453c2015-06-05 23:38:42 +020012 *
13 * For policy-specific per-blkcg data:
14 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
15 * Arianna Avanzini <avanzini.arianna@gmail.com>
Vivek Goyal31e4c282009-12-03 12:59:42 -050016 */
17#include <linux/ioprio.h>
Vivek Goyal22084192009-12-03 12:59:49 -050018#include <linux/kdev_t.h>
Vivek Goyal9d6a9862009-12-04 10:36:41 -050019#include <linux/module.h>
Stephen Rothwellaccee782009-12-07 19:29:39 +110020#include <linux/err.h>
Divyesh Shah91952912010-04-01 15:01:41 -070021#include <linux/blkdev.h>
Tejun Heo52ebea72015-05-22 17:13:37 -040022#include <linux/backing-dev.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090023#include <linux/slab.h>
Gui Jianfeng34d0f172010-04-13 16:05:49 +080024#include <linux/genhd.h>
Tejun Heo72e06c22012-03-05 13:15:00 -080025#include <linux/delay.h>
Tejun Heo9a9e8a22012-03-19 15:10:56 -070026#include <linux/atomic.h>
Tejun Heo36aa9e52015-08-18 14:55:31 -070027#include <linux/ctype.h>
Tejun Heoeea8f412015-05-22 17:13:17 -040028#include <linux/blk-cgroup.h>
Tejun Heo5efd6112012-03-05 13:15:12 -080029#include "blk.h"
Vivek Goyal3e252062009-12-04 10:36:42 -050030
Divyesh Shah84c124d2010-04-09 08:31:19 +020031#define MAX_KEY_LEN 100
32
Tejun Heo838f13b2015-07-09 16:39:47 -040033/*
34 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
35 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
36 * policy [un]register operations including cgroup file additions /
37 * removals. Putting cgroup file registration outside blkcg_pol_mutex
38 * allows grabbing it from cgroup callbacks.
39 */
40static DEFINE_MUTEX(blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -070041static DEFINE_MUTEX(blkcg_pol_mutex);
Tejun Heo923adde2012-03-05 13:15:13 -080042
Arianna Avanzinie48453c2015-06-05 23:38:42 +020043struct blkcg blkcg_root;
Tejun Heo3c798392012-04-16 13:57:25 -070044EXPORT_SYMBOL_GPL(blkcg_root);
Vivek Goyal9d6a9862009-12-04 10:36:41 -050045
Tejun Heo496d5e72015-05-22 17:13:21 -040046struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
47
Tejun Heo3c798392012-04-16 13:57:25 -070048static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
Tejun Heo035d10b2012-03-05 13:15:04 -080049
Tejun Heo7876f932015-07-09 16:39:49 -040050static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
51
Tejun Heoa2b16932012-04-13 13:11:33 -070052static bool blkcg_policy_enabled(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -070053 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -070054{
55 return pol && test_bit(pol->plid, q->blkcg_pols);
56}
57
Tejun Heo03814112012-03-05 13:15:14 -080058/**
59 * blkg_free - free a blkg
60 * @blkg: blkg to free
61 *
62 * Free @blkg which may be partially allocated.
63 */
Tejun Heo3c798392012-04-16 13:57:25 -070064static void blkg_free(struct blkcg_gq *blkg)
Tejun Heo03814112012-03-05 13:15:14 -080065{
Tejun Heoe8989fa2012-03-05 13:15:20 -080066 int i;
Tejun Heo549d3aa2012-03-05 13:15:16 -080067
68 if (!blkg)
69 return;
70
Tejun Heodb613672013-05-14 13:52:31 -070071 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo001bea72015-08-18 14:55:11 -070072 if (blkg->pd[i])
73 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
Tejun Heoe8989fa2012-03-05 13:15:20 -080074
Tejun Heo994b7832015-08-18 14:55:07 -070075 if (blkg->blkcg != &blkcg_root)
76 blk_exit_rl(&blkg->rl);
Tejun Heo77ea7332015-08-18 14:55:24 -070077
78 blkg_rwstat_exit(&blkg->stat_ios);
79 blkg_rwstat_exit(&blkg->stat_bytes);
Tejun Heo549d3aa2012-03-05 13:15:16 -080080 kfree(blkg);
Tejun Heo03814112012-03-05 13:15:14 -080081}
82
83/**
84 * blkg_alloc - allocate a blkg
85 * @blkcg: block cgroup the new blkg is associated with
86 * @q: request_queue the new blkg is associated with
Tejun Heo15974992012-06-04 20:40:52 -070087 * @gfp_mask: allocation mask to use
Tejun Heo03814112012-03-05 13:15:14 -080088 *
Tejun Heoe8989fa2012-03-05 13:15:20 -080089 * Allocate a new blkg assocating @blkcg and @q.
Tejun Heo03814112012-03-05 13:15:14 -080090 */
Tejun Heo15974992012-06-04 20:40:52 -070091static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
92 gfp_t gfp_mask)
Tejun Heo03814112012-03-05 13:15:14 -080093{
Tejun Heo3c798392012-04-16 13:57:25 -070094 struct blkcg_gq *blkg;
Tejun Heoe8989fa2012-03-05 13:15:20 -080095 int i;
Tejun Heo03814112012-03-05 13:15:14 -080096
97 /* alloc and init base part */
Tejun Heo15974992012-06-04 20:40:52 -070098 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
Tejun Heo03814112012-03-05 13:15:14 -080099 if (!blkg)
100 return NULL;
101
Tejun Heo77ea7332015-08-18 14:55:24 -0700102 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
103 blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
104 goto err_free;
105
Tejun Heoc875f4d2012-03-05 13:15:22 -0800106 blkg->q = q;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800107 INIT_LIST_HEAD(&blkg->q_node);
Tejun Heo03814112012-03-05 13:15:14 -0800108 blkg->blkcg = blkcg;
Tejun Heoa5049a82014-06-19 17:42:57 -0400109 atomic_set(&blkg->refcnt, 1);
Tejun Heo03814112012-03-05 13:15:14 -0800110
Tejun Heoa0516612012-06-26 15:05:44 -0700111 /* root blkg uses @q->root_rl, init rl only for !root blkgs */
112 if (blkcg != &blkcg_root) {
113 if (blk_init_rl(&blkg->rl, q, gfp_mask))
114 goto err_free;
115 blkg->rl.blkg = blkg;
116 }
117
Tejun Heo8bd435b2012-04-13 13:11:28 -0700118 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700119 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heoe8989fa2012-03-05 13:15:20 -0800120 struct blkg_policy_data *pd;
Tejun Heo03814112012-03-05 13:15:14 -0800121
Tejun Heoa2b16932012-04-13 13:11:33 -0700122 if (!blkcg_policy_enabled(q, pol))
Tejun Heoe8989fa2012-03-05 13:15:20 -0800123 continue;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800124
Tejun Heoe8989fa2012-03-05 13:15:20 -0800125 /* alloc per-policy data and attach it to blkg */
Tejun Heo001bea72015-08-18 14:55:11 -0700126 pd = pol->pd_alloc_fn(gfp_mask, q->node);
Tejun Heoa0516612012-06-26 15:05:44 -0700127 if (!pd)
128 goto err_free;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800129
Tejun Heoe8989fa2012-03-05 13:15:20 -0800130 blkg->pd[i] = pd;
131 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -0800132 pd->plid = i;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800133 }
134
Tejun Heo03814112012-03-05 13:15:14 -0800135 return blkg;
Tejun Heoa0516612012-06-26 15:05:44 -0700136
137err_free:
138 blkg_free(blkg);
139 return NULL;
Tejun Heo03814112012-03-05 13:15:14 -0800140}
141
Tejun Heo24f29042015-08-18 14:55:17 -0700142struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
143 struct request_queue *q, bool update_hint)
Tejun Heo80fd9972012-04-13 14:50:53 -0700144{
Tejun Heo3c798392012-04-16 13:57:25 -0700145 struct blkcg_gq *blkg;
Tejun Heo80fd9972012-04-13 14:50:53 -0700146
Tejun Heoa6371202012-04-19 16:29:24 -0700147 /*
Tejun Heo86cde6b2013-01-09 08:05:10 -0800148 * Hint didn't match. Look up from the radix tree. Note that the
149 * hint can only be updated under queue_lock as otherwise @blkg
150 * could have already been removed from blkg_tree. The caller is
151 * responsible for grabbing queue_lock if @update_hint.
Tejun Heoa6371202012-04-19 16:29:24 -0700152 */
153 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800154 if (blkg && blkg->q == q) {
155 if (update_hint) {
156 lockdep_assert_held(q->queue_lock);
157 rcu_assign_pointer(blkcg->blkg_hint, blkg);
158 }
Tejun Heoa6371202012-04-19 16:29:24 -0700159 return blkg;
Tejun Heo86cde6b2013-01-09 08:05:10 -0800160 }
Tejun Heoa6371202012-04-19 16:29:24 -0700161
Tejun Heo80fd9972012-04-13 14:50:53 -0700162 return NULL;
163}
Tejun Heoae118892015-08-18 14:55:20 -0700164EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
Tejun Heo80fd9972012-04-13 14:50:53 -0700165
Tejun Heo15974992012-06-04 20:40:52 -0700166/*
167 * If @new_blkg is %NULL, this function tries to allocate a new one as
Tejun Heod93a11f2015-08-18 14:55:01 -0700168 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
Tejun Heo15974992012-06-04 20:40:52 -0700169 */
Tejun Heo86cde6b2013-01-09 08:05:10 -0800170static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
171 struct request_queue *q,
172 struct blkcg_gq *new_blkg)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400173{
Tejun Heo3c798392012-04-16 13:57:25 -0700174 struct blkcg_gq *blkg;
Tejun Heoce7acfe2015-05-22 17:13:38 -0400175 struct bdi_writeback_congested *wb_congested;
Tejun Heof427d902013-01-09 08:05:12 -0800176 int i, ret;
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400177
Tejun Heocd1604f2012-03-05 13:15:06 -0800178 WARN_ON_ONCE(!rcu_read_lock_held());
179 lockdep_assert_held(q->queue_lock);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500180
Tejun Heo7ee9c562012-03-05 13:15:11 -0800181 /* blkg holds a reference to blkcg */
Tejun Heoec903c02014-05-13 12:11:01 -0400182 if (!css_tryget_online(&blkcg->css)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700183 ret = -ENODEV;
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800184 goto err_free_blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700185 }
Tejun Heocd1604f2012-03-05 13:15:06 -0800186
Tejun Heoce7acfe2015-05-22 17:13:38 -0400187 wb_congested = wb_congested_get_create(&q->backing_dev_info,
Tejun Heo07a61342016-11-21 18:03:32 -0500188 blkcg->css.id,
189 GFP_NOWAIT | __GFP_NOWARN);
Tejun Heoce7acfe2015-05-22 17:13:38 -0400190 if (!wb_congested) {
191 ret = -ENOMEM;
192 goto err_put_css;
193 }
194
Tejun Heo496fb782012-04-19 16:29:23 -0700195 /* allocate */
Tejun Heo15974992012-06-04 20:40:52 -0700196 if (!new_blkg) {
Tejun Heo07a61342016-11-21 18:03:32 -0500197 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
Tejun Heo15974992012-06-04 20:40:52 -0700198 if (unlikely(!new_blkg)) {
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800199 ret = -ENOMEM;
Tejun Heoce7acfe2015-05-22 17:13:38 -0400200 goto err_put_congested;
Tejun Heo15974992012-06-04 20:40:52 -0700201 }
202 }
203 blkg = new_blkg;
Tejun Heoce7acfe2015-05-22 17:13:38 -0400204 blkg->wb_congested = wb_congested;
Tejun Heocd1604f2012-03-05 13:15:06 -0800205
Tejun Heodb613672013-05-14 13:52:31 -0700206 /* link parent */
Tejun Heo3c547862013-01-09 08:05:10 -0800207 if (blkcg_parent(blkcg)) {
208 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
209 if (WARN_ON_ONCE(!blkg->parent)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700210 ret = -ENODEV;
Tejun Heoce7acfe2015-05-22 17:13:38 -0400211 goto err_put_congested;
Tejun Heo3c547862013-01-09 08:05:10 -0800212 }
213 blkg_get(blkg->parent);
214 }
215
Tejun Heodb613672013-05-14 13:52:31 -0700216 /* invoke per-policy init */
217 for (i = 0; i < BLKCG_MAX_POLS; i++) {
218 struct blkcg_policy *pol = blkcg_policy[i];
219
220 if (blkg->pd[i] && pol->pd_init_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700221 pol->pd_init_fn(blkg->pd[i]);
Tejun Heodb613672013-05-14 13:52:31 -0700222 }
223
224 /* insert */
Tejun Heoa6371202012-04-19 16:29:24 -0700225 spin_lock(&blkcg->lock);
226 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
227 if (likely(!ret)) {
228 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
229 list_add(&blkg->q_node, &q->blkg_list);
Tejun Heof427d902013-01-09 08:05:12 -0800230
231 for (i = 0; i < BLKCG_MAX_POLS; i++) {
232 struct blkcg_policy *pol = blkcg_policy[i];
233
234 if (blkg->pd[i] && pol->pd_online_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700235 pol->pd_online_fn(blkg->pd[i]);
Tejun Heof427d902013-01-09 08:05:12 -0800236 }
Tejun Heoa6371202012-04-19 16:29:24 -0700237 }
Tejun Heof427d902013-01-09 08:05:12 -0800238 blkg->online = true;
Tejun Heoa6371202012-04-19 16:29:24 -0700239 spin_unlock(&blkcg->lock);
240
Tejun Heoec13b1d2015-05-22 17:13:19 -0400241 if (!ret)
Tejun Heoa6371202012-04-19 16:29:24 -0700242 return blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700243
Tejun Heo3c547862013-01-09 08:05:10 -0800244 /* @blkg failed fully initialized, use the usual release path */
245 blkg_put(blkg);
246 return ERR_PTR(ret);
247
Tejun Heoce7acfe2015-05-22 17:13:38 -0400248err_put_congested:
249 wb_congested_put(wb_congested);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800250err_put_css:
Tejun Heo496fb782012-04-19 16:29:23 -0700251 css_put(&blkcg->css);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800252err_free_blkg:
Tejun Heo15974992012-06-04 20:40:52 -0700253 blkg_free(new_blkg);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800254 return ERR_PTR(ret);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500255}
Tejun Heo3c96cb32012-04-13 13:11:34 -0700256
Tejun Heo86cde6b2013-01-09 08:05:10 -0800257/**
258 * blkg_lookup_create - lookup blkg, try to create one if not there
259 * @blkcg: blkcg of interest
260 * @q: request_queue of interest
261 *
262 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
Tejun Heo3c547862013-01-09 08:05:10 -0800263 * create one. blkg creation is performed recursively from blkcg_root such
264 * that all non-root blkg's have access to the parent blkg. This function
265 * should be called under RCU read lock and @q->queue_lock.
Tejun Heo86cde6b2013-01-09 08:05:10 -0800266 *
267 * Returns pointer to the looked up or created blkg on success, ERR_PTR()
268 * value on error. If @q is dead, returns ERR_PTR(-EINVAL). If @q is not
269 * dead and bypassing, returns ERR_PTR(-EBUSY).
270 */
Tejun Heo3c798392012-04-16 13:57:25 -0700271struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
272 struct request_queue *q)
Tejun Heo3c96cb32012-04-13 13:11:34 -0700273{
Tejun Heo86cde6b2013-01-09 08:05:10 -0800274 struct blkcg_gq *blkg;
275
276 WARN_ON_ONCE(!rcu_read_lock_held());
277 lockdep_assert_held(q->queue_lock);
278
Tejun Heo3c96cb32012-04-13 13:11:34 -0700279 /*
280 * This could be the first entry point of blkcg implementation and
281 * we shouldn't allow anything to go through for a bypassing queue.
282 */
283 if (unlikely(blk_queue_bypass(q)))
Tejun Heo20386ce2015-08-18 14:55:28 -0700284 return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800285
286 blkg = __blkg_lookup(blkcg, q, true);
287 if (blkg)
288 return blkg;
289
Tejun Heo3c547862013-01-09 08:05:10 -0800290 /*
291 * Create blkgs walking down from blkcg_root to @blkcg, so that all
292 * non-root blkgs have access to their parents.
293 */
294 while (true) {
295 struct blkcg *pos = blkcg;
296 struct blkcg *parent = blkcg_parent(blkcg);
297
298 while (parent && !__blkg_lookup(parent, q, false)) {
299 pos = parent;
300 parent = blkcg_parent(parent);
301 }
302
303 blkg = blkg_create(pos, q, NULL);
304 if (pos == blkcg || IS_ERR(blkg))
305 return blkg;
306 }
Tejun Heo3c96cb32012-04-13 13:11:34 -0700307}
Vivek Goyal31e4c282009-12-03 12:59:42 -0500308
Tejun Heo3c798392012-04-16 13:57:25 -0700309static void blkg_destroy(struct blkcg_gq *blkg)
Tejun Heo72e06c22012-03-05 13:15:00 -0800310{
Tejun Heo3c798392012-04-16 13:57:25 -0700311 struct blkcg *blkcg = blkg->blkcg;
Tejun Heo77ea7332015-08-18 14:55:24 -0700312 struct blkcg_gq *parent = blkg->parent;
Tejun Heof427d902013-01-09 08:05:12 -0800313 int i;
Tejun Heo03aa2642012-03-05 13:15:19 -0800314
Tejun Heo27e1f9d2012-06-05 13:36:44 +0200315 lockdep_assert_held(blkg->q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800316 lockdep_assert_held(&blkcg->lock);
Tejun Heo03aa2642012-03-05 13:15:19 -0800317
318 /* Something wrong if we are trying to remove same group twice */
Tejun Heoe8989fa2012-03-05 13:15:20 -0800319 WARN_ON_ONCE(list_empty(&blkg->q_node));
Tejun Heo9f13ef62012-03-05 13:15:21 -0800320 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
Tejun Heoa6371202012-04-19 16:29:24 -0700321
Tejun Heof427d902013-01-09 08:05:12 -0800322 for (i = 0; i < BLKCG_MAX_POLS; i++) {
323 struct blkcg_policy *pol = blkcg_policy[i];
324
325 if (blkg->pd[i] && pol->pd_offline_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700326 pol->pd_offline_fn(blkg->pd[i]);
Tejun Heof427d902013-01-09 08:05:12 -0800327 }
Tejun Heo77ea7332015-08-18 14:55:24 -0700328
329 if (parent) {
330 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
331 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
332 }
333
Tejun Heof427d902013-01-09 08:05:12 -0800334 blkg->online = false;
335
Tejun Heoa6371202012-04-19 16:29:24 -0700336 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
Tejun Heoe8989fa2012-03-05 13:15:20 -0800337 list_del_init(&blkg->q_node);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800338 hlist_del_init_rcu(&blkg->blkcg_node);
Tejun Heo03aa2642012-03-05 13:15:19 -0800339
Tejun Heo03aa2642012-03-05 13:15:19 -0800340 /*
Tejun Heoa6371202012-04-19 16:29:24 -0700341 * Both setting lookup hint to and clearing it from @blkg are done
342 * under queue_lock. If it's not pointing to @blkg now, it never
343 * will. Hint assignment itself can race safely.
344 */
Paul E. McKenneyec6c6762014-02-17 13:35:57 -0800345 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
Tejun Heoa6371202012-04-19 16:29:24 -0700346 rcu_assign_pointer(blkcg->blkg_hint, NULL);
347
348 /*
Tejun Heo03aa2642012-03-05 13:15:19 -0800349 * Put the reference taken at the time of creation so that when all
350 * queues are gone, group can be destroyed.
351 */
352 blkg_put(blkg);
353}
354
Tejun Heo9f13ef62012-03-05 13:15:21 -0800355/**
356 * blkg_destroy_all - destroy all blkgs associated with a request_queue
357 * @q: request_queue of interest
Tejun Heo9f13ef62012-03-05 13:15:21 -0800358 *
Tejun Heo3c96cb32012-04-13 13:11:34 -0700359 * Destroy all blkgs associated with @q.
Tejun Heo9f13ef62012-03-05 13:15:21 -0800360 */
Tejun Heo3c96cb32012-04-13 13:11:34 -0700361static void blkg_destroy_all(struct request_queue *q)
Tejun Heo03aa2642012-03-05 13:15:19 -0800362{
Tejun Heo3c798392012-04-16 13:57:25 -0700363 struct blkcg_gq *blkg, *n;
Tejun Heo72e06c22012-03-05 13:15:00 -0800364
Tejun Heo6d18b002012-04-13 13:11:35 -0700365 lockdep_assert_held(q->queue_lock);
Tejun Heo72e06c22012-03-05 13:15:00 -0800366
Tejun Heo9f13ef62012-03-05 13:15:21 -0800367 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
Tejun Heo3c798392012-04-16 13:57:25 -0700368 struct blkcg *blkcg = blkg->blkcg;
Tejun Heo72e06c22012-03-05 13:15:00 -0800369
Tejun Heo9f13ef62012-03-05 13:15:21 -0800370 spin_lock(&blkcg->lock);
371 blkg_destroy(blkg);
372 spin_unlock(&blkcg->lock);
Tejun Heo72e06c22012-03-05 13:15:00 -0800373 }
Tejun Heo6fe810b2015-09-05 15:47:36 -0400374
375 q->root_blkg = NULL;
376 q->root_rl.blkg = NULL;
Tejun Heo72e06c22012-03-05 13:15:00 -0800377}
378
Tejun Heo2a4fd072013-05-14 13:52:31 -0700379/*
380 * A group is RCU protected, but having an rcu lock does not mean that one
381 * can access all the fields of blkg and assume these are valid. For
382 * example, don't try to follow throtl_data and request queue links.
383 *
384 * Having a reference to blkg under an rcu allows accesses to only values
385 * local to groups like group stats and group rate limits.
386 */
387void __blkg_release_rcu(struct rcu_head *rcu_head)
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800388{
Tejun Heo2a4fd072013-05-14 13:52:31 -0700389 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
Tejun Heodb613672013-05-14 13:52:31 -0700390
Tejun Heo3c547862013-01-09 08:05:10 -0800391 /* release the blkcg and parent blkg refs this blkg has been holding */
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800392 css_put(&blkg->blkcg->css);
Tejun Heoa5049a82014-06-19 17:42:57 -0400393 if (blkg->parent)
Tejun Heo3c547862013-01-09 08:05:10 -0800394 blkg_put(blkg->parent);
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800395
Tejun Heoce7acfe2015-05-22 17:13:38 -0400396 wb_congested_put(blkg->wb_congested);
397
Tejun Heo2a4fd072013-05-14 13:52:31 -0700398 blkg_free(blkg);
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800399}
Tejun Heo2a4fd072013-05-14 13:52:31 -0700400EXPORT_SYMBOL_GPL(__blkg_release_rcu);
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800401
Tejun Heoa0516612012-06-26 15:05:44 -0700402/*
403 * The next function used by blk_queue_for_each_rl(). It's a bit tricky
404 * because the root blkg uses @q->root_rl instead of its own rl.
405 */
406struct request_list *__blk_queue_next_rl(struct request_list *rl,
407 struct request_queue *q)
408{
409 struct list_head *ent;
410 struct blkcg_gq *blkg;
411
412 /*
413 * Determine the current blkg list_head. The first entry is
414 * root_rl which is off @q->blkg_list and mapped to the head.
415 */
416 if (rl == &q->root_rl) {
417 ent = &q->blkg_list;
Jun'ichi Nomura65c77fd2012-10-22 10:15:37 +0900418 /* There are no more block groups, hence no request lists */
419 if (list_empty(ent))
420 return NULL;
Tejun Heoa0516612012-06-26 15:05:44 -0700421 } else {
422 blkg = container_of(rl, struct blkcg_gq, rl);
423 ent = &blkg->q_node;
424 }
425
426 /* walk to the next list_head, skip root blkcg */
427 ent = ent->next;
428 if (ent == &q->root_blkg->q_node)
429 ent = ent->next;
430 if (ent == &q->blkg_list)
431 return NULL;
432
433 blkg = container_of(ent, struct blkcg_gq, q_node);
434 return &blkg->rl;
435}
436
Tejun Heo182446d2013-08-08 20:11:24 -0400437static int blkcg_reset_stats(struct cgroup_subsys_state *css,
438 struct cftype *cftype, u64 val)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700439{
Tejun Heo182446d2013-08-08 20:11:24 -0400440 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heo3c798392012-04-16 13:57:25 -0700441 struct blkcg_gq *blkg;
Tejun Heobc0d6502012-04-13 13:11:26 -0700442 int i;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700443
Tejun Heo838f13b2015-07-09 16:39:47 -0400444 mutex_lock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700445 spin_lock_irq(&blkcg->lock);
Tejun Heo997a0262012-03-08 10:53:58 -0800446
447 /*
448 * Note that stat reset is racy - it doesn't synchronize against
449 * stat updates. This is a debug feature which shouldn't exist
450 * anyway. If you get hit by a race, retry.
451 */
Sasha Levinb67bfe02013-02-27 17:06:00 -0800452 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heo77ea7332015-08-18 14:55:24 -0700453 blkg_rwstat_reset(&blkg->stat_bytes);
454 blkg_rwstat_reset(&blkg->stat_ios);
455
Tejun Heo8bd435b2012-04-13 13:11:28 -0700456 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700457 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heo549d3aa2012-03-05 13:15:16 -0800458
Tejun Heoa9520cd2015-08-18 14:55:14 -0700459 if (blkg->pd[i] && pol->pd_reset_stats_fn)
460 pol->pd_reset_stats_fn(blkg->pd[i]);
Tejun Heobc0d6502012-04-13 13:11:26 -0700461 }
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700462 }
Vivek Goyalf0bdc8c2011-05-19 15:38:30 -0400463
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700464 spin_unlock_irq(&blkcg->lock);
Tejun Heobc0d6502012-04-13 13:11:26 -0700465 mutex_unlock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700466 return 0;
467}
468
Tejun Heodd165eb2015-08-18 14:55:33 -0700469const char *blkg_dev_name(struct blkcg_gq *blkg)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700470{
Tejun Heod3d32e62012-04-01 14:38:42 -0700471 /* some drivers (floppy) instantiate a queue w/o disk registered */
472 if (blkg->q->backing_dev_info.dev)
473 return dev_name(blkg->q->backing_dev_info.dev);
474 return NULL;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700475}
Tejun Heodd165eb2015-08-18 14:55:33 -0700476EXPORT_SYMBOL_GPL(blkg_dev_name);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700477
Tejun Heod3d32e62012-04-01 14:38:42 -0700478/**
479 * blkcg_print_blkgs - helper for printing per-blkg data
480 * @sf: seq_file to print to
481 * @blkcg: blkcg of interest
482 * @prfill: fill function to print out a blkg
483 * @pol: policy in question
484 * @data: data to be passed to @prfill
485 * @show_total: to print out sum of prfill return values or not
486 *
487 * This function invokes @prfill on each blkg of @blkcg if pd for the
488 * policy specified by @pol exists. @prfill is invoked with @sf, the
Tejun Heo810ecfa2013-01-09 08:05:13 -0800489 * policy data and @data and the matching queue lock held. If @show_total
490 * is %true, the sum of the return values from @prfill is printed with
491 * "Total" label at the end.
Tejun Heod3d32e62012-04-01 14:38:42 -0700492 *
493 * This is to be used to construct print functions for
494 * cftype->read_seq_string method.
495 */
Tejun Heo3c798392012-04-16 13:57:25 -0700496void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
Tejun Heof95a04a2012-04-16 13:57:26 -0700497 u64 (*prfill)(struct seq_file *,
498 struct blkg_policy_data *, int),
Tejun Heo3c798392012-04-16 13:57:25 -0700499 const struct blkcg_policy *pol, int data,
Tejun Heoec399342012-04-13 13:11:27 -0700500 bool show_total)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400501{
Tejun Heo3c798392012-04-16 13:57:25 -0700502 struct blkcg_gq *blkg;
Tejun Heod3d32e62012-04-01 14:38:42 -0700503 u64 total = 0;
504
Tejun Heo810ecfa2013-01-09 08:05:13 -0800505 rcu_read_lock();
Linus Torvaldsee89f812013-02-28 12:52:24 -0800506 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heo810ecfa2013-01-09 08:05:13 -0800507 spin_lock_irq(blkg->q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -0700508 if (blkcg_policy_enabled(blkg->q, pol))
Tejun Heof95a04a2012-04-16 13:57:26 -0700509 total += prfill(sf, blkg->pd[pol->plid], data);
Tejun Heo810ecfa2013-01-09 08:05:13 -0800510 spin_unlock_irq(blkg->q->queue_lock);
511 }
512 rcu_read_unlock();
Tejun Heod3d32e62012-04-01 14:38:42 -0700513
514 if (show_total)
515 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
516}
Tejun Heo829fdb52012-04-01 14:38:43 -0700517EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
Tejun Heod3d32e62012-04-01 14:38:42 -0700518
519/**
520 * __blkg_prfill_u64 - prfill helper for a single u64 value
521 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700522 * @pd: policy private data of interest
Tejun Heod3d32e62012-04-01 14:38:42 -0700523 * @v: value to print
524 *
Tejun Heof95a04a2012-04-16 13:57:26 -0700525 * Print @v to @sf for the device assocaited with @pd.
Tejun Heod3d32e62012-04-01 14:38:42 -0700526 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700527u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
Tejun Heod3d32e62012-04-01 14:38:42 -0700528{
Tejun Heof95a04a2012-04-16 13:57:26 -0700529 const char *dname = blkg_dev_name(pd->blkg);
Tejun Heod3d32e62012-04-01 14:38:42 -0700530
531 if (!dname)
532 return 0;
533
534 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
535 return v;
536}
Tejun Heo829fdb52012-04-01 14:38:43 -0700537EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
Tejun Heod3d32e62012-04-01 14:38:42 -0700538
539/**
540 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
541 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700542 * @pd: policy private data of interest
Tejun Heod3d32e62012-04-01 14:38:42 -0700543 * @rwstat: rwstat to print
544 *
Tejun Heof95a04a2012-04-16 13:57:26 -0700545 * Print @rwstat to @sf for the device assocaited with @pd.
Tejun Heod3d32e62012-04-01 14:38:42 -0700546 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700547u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
Tejun Heo829fdb52012-04-01 14:38:43 -0700548 const struct blkg_rwstat *rwstat)
Tejun Heod3d32e62012-04-01 14:38:42 -0700549{
550 static const char *rwstr[] = {
551 [BLKG_RWSTAT_READ] = "Read",
552 [BLKG_RWSTAT_WRITE] = "Write",
553 [BLKG_RWSTAT_SYNC] = "Sync",
554 [BLKG_RWSTAT_ASYNC] = "Async",
555 };
Tejun Heof95a04a2012-04-16 13:57:26 -0700556 const char *dname = blkg_dev_name(pd->blkg);
Tejun Heod3d32e62012-04-01 14:38:42 -0700557 u64 v;
558 int i;
559
560 if (!dname)
561 return 0;
562
563 for (i = 0; i < BLKG_RWSTAT_NR; i++)
564 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
Tejun Heo24bdb8e2015-08-18 14:55:22 -0700565 (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
Tejun Heod3d32e62012-04-01 14:38:42 -0700566
Tejun Heo24bdb8e2015-08-18 14:55:22 -0700567 v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
568 atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
Tejun Heod3d32e62012-04-01 14:38:42 -0700569 seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
570 return v;
571}
Tejun Heob50da392013-01-09 08:05:12 -0800572EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700573
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700574/**
575 * blkg_prfill_stat - prfill callback for blkg_stat
576 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700577 * @pd: policy private data of interest
578 * @off: offset to the blkg_stat in @pd
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700579 *
580 * prfill callback for printing a blkg_stat.
581 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700582u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
Tejun Heod3d32e62012-04-01 14:38:42 -0700583{
Tejun Heof95a04a2012-04-16 13:57:26 -0700584 return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
Tejun Heod3d32e62012-04-01 14:38:42 -0700585}
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700586EXPORT_SYMBOL_GPL(blkg_prfill_stat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700587
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700588/**
589 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
590 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700591 * @pd: policy private data of interest
592 * @off: offset to the blkg_rwstat in @pd
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700593 *
594 * prfill callback for printing a blkg_rwstat.
595 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700596u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
597 int off)
Tejun Heod3d32e62012-04-01 14:38:42 -0700598{
Tejun Heof95a04a2012-04-16 13:57:26 -0700599 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
Tejun Heod3d32e62012-04-01 14:38:42 -0700600
Tejun Heof95a04a2012-04-16 13:57:26 -0700601 return __blkg_prfill_rwstat(sf, pd, &rwstat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700602}
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700603EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700604
Tejun Heo77ea7332015-08-18 14:55:24 -0700605static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
606 struct blkg_policy_data *pd, int off)
607{
608 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);
609
610 return __blkg_prfill_rwstat(sf, pd, &rwstat);
611}
612
613/**
614 * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
615 * @sf: seq_file to print to
616 * @v: unused
617 *
618 * To be used as cftype->seq_show to print blkg->stat_bytes.
619 * cftype->private must be set to the blkcg_policy.
620 */
621int blkg_print_stat_bytes(struct seq_file *sf, void *v)
622{
623 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
624 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
625 offsetof(struct blkcg_gq, stat_bytes), true);
626 return 0;
627}
628EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);
629
630/**
631 * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
632 * @sf: seq_file to print to
633 * @v: unused
634 *
635 * To be used as cftype->seq_show to print blkg->stat_ios. cftype->private
636 * must be set to the blkcg_policy.
637 */
638int blkg_print_stat_ios(struct seq_file *sf, void *v)
639{
640 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
641 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
642 offsetof(struct blkcg_gq, stat_ios), true);
643 return 0;
644}
645EXPORT_SYMBOL_GPL(blkg_print_stat_ios);
646
647static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
648 struct blkg_policy_data *pd,
649 int off)
650{
651 struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
652 NULL, off);
653 return __blkg_prfill_rwstat(sf, pd, &rwstat);
654}
655
656/**
657 * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
658 * @sf: seq_file to print to
659 * @v: unused
660 */
661int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
662{
663 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
664 blkg_prfill_rwstat_field_recursive,
665 (void *)seq_cft(sf)->private,
666 offsetof(struct blkcg_gq, stat_bytes), true);
667 return 0;
668}
669EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);
670
671/**
672 * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
673 * @sf: seq_file to print to
674 * @v: unused
675 */
676int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
677{
678 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
679 blkg_prfill_rwstat_field_recursive,
680 (void *)seq_cft(sf)->private,
681 offsetof(struct blkcg_gq, stat_ios), true);
682 return 0;
683}
684EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
685
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700686/**
Tejun Heo16b3de62013-01-09 08:05:12 -0800687 * blkg_stat_recursive_sum - collect hierarchical blkg_stat
Tejun Heof12c74c2015-08-18 14:55:23 -0700688 * @blkg: blkg of interest
689 * @pol: blkcg_policy which contains the blkg_stat
690 * @off: offset to the blkg_stat in blkg_policy_data or @blkg
Tejun Heo16b3de62013-01-09 08:05:12 -0800691 *
Tejun Heof12c74c2015-08-18 14:55:23 -0700692 * Collect the blkg_stat specified by @blkg, @pol and @off and all its
693 * online descendants and their aux counts. The caller must be holding the
694 * queue lock for online tests.
695 *
696 * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
697 * at @off bytes into @blkg's blkg_policy_data of the policy.
Tejun Heo16b3de62013-01-09 08:05:12 -0800698 */
Tejun Heof12c74c2015-08-18 14:55:23 -0700699u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
700 struct blkcg_policy *pol, int off)
Tejun Heo16b3de62013-01-09 08:05:12 -0800701{
Tejun Heo16b3de62013-01-09 08:05:12 -0800702 struct blkcg_gq *pos_blkg;
Tejun Heo492eb212013-08-08 20:11:25 -0400703 struct cgroup_subsys_state *pos_css;
Tejun Heobd8815a2013-08-08 20:11:27 -0400704 u64 sum = 0;
Tejun Heo16b3de62013-01-09 08:05:12 -0800705
Tejun Heof12c74c2015-08-18 14:55:23 -0700706 lockdep_assert_held(blkg->q->queue_lock);
Tejun Heo16b3de62013-01-09 08:05:12 -0800707
Tejun Heo16b3de62013-01-09 08:05:12 -0800708 rcu_read_lock();
Tejun Heof12c74c2015-08-18 14:55:23 -0700709 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
710 struct blkg_stat *stat;
Tejun Heo16b3de62013-01-09 08:05:12 -0800711
Tejun Heof12c74c2015-08-18 14:55:23 -0700712 if (!pos_blkg->online)
713 continue;
714
715 if (pol)
716 stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
717 else
718 stat = (void *)blkg + off;
719
720 sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
Tejun Heo16b3de62013-01-09 08:05:12 -0800721 }
722 rcu_read_unlock();
723
724 return sum;
725}
726EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
727
728/**
729 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
Tejun Heof12c74c2015-08-18 14:55:23 -0700730 * @blkg: blkg of interest
731 * @pol: blkcg_policy which contains the blkg_rwstat
732 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
Tejun Heo16b3de62013-01-09 08:05:12 -0800733 *
Tejun Heof12c74c2015-08-18 14:55:23 -0700734 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
735 * online descendants and their aux counts. The caller must be holding the
736 * queue lock for online tests.
737 *
738 * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
739 * is at @off bytes into @blkg's blkg_policy_data of the policy.
Tejun Heo16b3de62013-01-09 08:05:12 -0800740 */
Tejun Heof12c74c2015-08-18 14:55:23 -0700741struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
742 struct blkcg_policy *pol, int off)
Tejun Heo16b3de62013-01-09 08:05:12 -0800743{
Tejun Heo16b3de62013-01-09 08:05:12 -0800744 struct blkcg_gq *pos_blkg;
Tejun Heo492eb212013-08-08 20:11:25 -0400745 struct cgroup_subsys_state *pos_css;
Tejun Heobd8815a2013-08-08 20:11:27 -0400746 struct blkg_rwstat sum = { };
Tejun Heo16b3de62013-01-09 08:05:12 -0800747 int i;
748
Tejun Heof12c74c2015-08-18 14:55:23 -0700749 lockdep_assert_held(blkg->q->queue_lock);
Tejun Heo16b3de62013-01-09 08:05:12 -0800750
Tejun Heo16b3de62013-01-09 08:05:12 -0800751 rcu_read_lock();
Tejun Heof12c74c2015-08-18 14:55:23 -0700752 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
Tejun Heo3a7faea2015-08-18 14:55:26 -0700753 struct blkg_rwstat *rwstat;
Tejun Heo16b3de62013-01-09 08:05:12 -0800754
755 if (!pos_blkg->online)
756 continue;
757
Tejun Heof12c74c2015-08-18 14:55:23 -0700758 if (pol)
759 rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
760 else
761 rwstat = (void *)pos_blkg + off;
762
Tejun Heo16b3de62013-01-09 08:05:12 -0800763 for (i = 0; i < BLKG_RWSTAT_NR; i++)
Tejun Heo3a7faea2015-08-18 14:55:26 -0700764 atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
765 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
766 &sum.aux_cnt[i]);
Tejun Heo16b3de62013-01-09 08:05:12 -0800767 }
768 rcu_read_unlock();
769
770 return sum;
771}
772EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
773
774/**
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700775 * blkg_conf_prep - parse and prepare for per-blkg config update
776 * @blkcg: target block cgroup
Tejun Heoda8b0662012-04-13 13:11:29 -0700777 * @pol: target policy
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700778 * @input: input string
779 * @ctx: blkg_conf_ctx to be filled
780 *
781 * Parse per-blkg config update from @input and initialize @ctx with the
Tejun Heo36aa9e52015-08-18 14:55:31 -0700782 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
783 * part of @input following MAJ:MIN. This function returns with RCU read
784 * lock and queue lock held and must be paired with blkg_conf_finish().
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700785 */
Tejun Heo3c798392012-04-16 13:57:25 -0700786int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
Tejun Heo36aa9e52015-08-18 14:55:31 -0700787 char *input, struct blkg_conf_ctx *ctx)
Tejun Heoda8b0662012-04-13 13:11:29 -0700788 __acquires(rcu) __acquires(disk->queue->queue_lock)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800789{
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700790 struct gendisk *disk;
Tejun Heo3c798392012-04-16 13:57:25 -0700791 struct blkcg_gq *blkg;
Roman Pen39a169b2016-02-09 12:33:35 -0700792 struct module *owner;
Tejun Heo726fa692012-04-01 14:38:43 -0700793 unsigned int major, minor;
Tejun Heo36aa9e52015-08-18 14:55:31 -0700794 int key_len, part, ret;
795 char *body;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800796
Tejun Heo36aa9e52015-08-18 14:55:31 -0700797 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
Tejun Heo726fa692012-04-01 14:38:43 -0700798 return -EINVAL;
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700799
Tejun Heo36aa9e52015-08-18 14:55:31 -0700800 body = input + key_len;
801 if (!isspace(*body))
802 return -EINVAL;
803 body = skip_spaces(body);
804
Tejun Heo726fa692012-04-01 14:38:43 -0700805 disk = get_gendisk(MKDEV(major, minor), &part);
Tejun Heo5f6c2d22015-07-22 18:05:53 -0400806 if (!disk)
Tejun Heo20386ce2015-08-18 14:55:28 -0700807 return -ENODEV;
Tejun Heo5f6c2d22015-07-22 18:05:53 -0400808 if (part) {
Roman Pen39a169b2016-02-09 12:33:35 -0700809 owner = disk->fops->owner;
Tejun Heo5f6c2d22015-07-22 18:05:53 -0400810 put_disk(disk);
Roman Pen39a169b2016-02-09 12:33:35 -0700811 module_put(owner);
Tejun Heo20386ce2015-08-18 14:55:28 -0700812 return -ENODEV;
Tejun Heo5f6c2d22015-07-22 18:05:53 -0400813 }
Tejun Heoe56da7e2012-03-05 13:15:07 -0800814
815 rcu_read_lock();
Tejun Heo4bfd4822012-03-05 13:15:08 -0800816 spin_lock_irq(disk->queue->queue_lock);
Tejun Heoda8b0662012-04-13 13:11:29 -0700817
Tejun Heoa2b16932012-04-13 13:11:33 -0700818 if (blkcg_policy_enabled(disk->queue, pol))
Tejun Heo3c96cb32012-04-13 13:11:34 -0700819 blkg = blkg_lookup_create(blkcg, disk->queue);
Tejun Heoa2b16932012-04-13 13:11:33 -0700820 else
Tejun Heo20386ce2015-08-18 14:55:28 -0700821 blkg = ERR_PTR(-EOPNOTSUPP);
Tejun Heoe56da7e2012-03-05 13:15:07 -0800822
Tejun Heo4bfd4822012-03-05 13:15:08 -0800823 if (IS_ERR(blkg)) {
824 ret = PTR_ERR(blkg);
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700825 rcu_read_unlock();
Tejun Heoda8b0662012-04-13 13:11:29 -0700826 spin_unlock_irq(disk->queue->queue_lock);
Roman Pen39a169b2016-02-09 12:33:35 -0700827 owner = disk->fops->owner;
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700828 put_disk(disk);
Roman Pen39a169b2016-02-09 12:33:35 -0700829 module_put(owner);
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700830 /*
831 * If queue was bypassing, we should retry. Do so after a
832 * short msleep(). It isn't strictly necessary but queue
833 * can be bypassing for some time and it's always nice to
834 * avoid busy looping.
835 */
836 if (ret == -EBUSY) {
837 msleep(10);
838 ret = restart_syscall();
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400839 }
Tejun Heo726fa692012-04-01 14:38:43 -0700840 return ret;
Vivek Goyal062a6442010-09-15 17:06:33 -0400841 }
Tejun Heoe56da7e2012-03-05 13:15:07 -0800842
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700843 ctx->disk = disk;
844 ctx->blkg = blkg;
Tejun Heo36aa9e52015-08-18 14:55:31 -0700845 ctx->body = body;
Tejun Heo726fa692012-04-01 14:38:43 -0700846 return 0;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800847}
Tejun Heo829fdb52012-04-01 14:38:43 -0700848EXPORT_SYMBOL_GPL(blkg_conf_prep);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800849
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700850/**
851 * blkg_conf_finish - finish up per-blkg config update
852 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
853 *
854 * Finish up after per-blkg config update. This function must be paired
855 * with blkg_conf_prep().
856 */
Tejun Heo829fdb52012-04-01 14:38:43 -0700857void blkg_conf_finish(struct blkg_conf_ctx *ctx)
Tejun Heoda8b0662012-04-13 13:11:29 -0700858 __releases(ctx->disk->queue->queue_lock) __releases(rcu)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800859{
Roman Pen39a169b2016-02-09 12:33:35 -0700860 struct module *owner;
861
Tejun Heoda8b0662012-04-13 13:11:29 -0700862 spin_unlock_irq(ctx->disk->queue->queue_lock);
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700863 rcu_read_unlock();
Roman Pen39a169b2016-02-09 12:33:35 -0700864 owner = ctx->disk->fops->owner;
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700865 put_disk(ctx->disk);
Roman Pen39a169b2016-02-09 12:33:35 -0700866 module_put(owner);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800867}
Tejun Heo829fdb52012-04-01 14:38:43 -0700868EXPORT_SYMBOL_GPL(blkg_conf_finish);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800869
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700870static int blkcg_print_stat(struct seq_file *sf, void *v)
871{
872 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
873 struct blkcg_gq *blkg;
874
875 rcu_read_lock();
876
877 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
878 const char *dname;
879 struct blkg_rwstat rwstat;
880 u64 rbytes, wbytes, rios, wios;
881
882 dname = blkg_dev_name(blkg);
883 if (!dname)
884 continue;
885
886 spin_lock_irq(blkg->q->queue_lock);
887
888 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
889 offsetof(struct blkcg_gq, stat_bytes));
890 rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
891 wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
892
893 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
894 offsetof(struct blkcg_gq, stat_ios));
895 rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
896 wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
897
898 spin_unlock_irq(blkg->q->queue_lock);
899
900 if (rbytes || wbytes || rios || wios)
901 seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n",
902 dname, rbytes, wbytes, rios, wios);
903 }
904
905 rcu_read_unlock();
906 return 0;
907}
908
Bart Van Asschee1f3b942016-06-14 17:04:32 +0200909static struct cftype blkcg_files[] = {
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700910 {
911 .name = "stat",
Tejun Heoca0752c2015-10-22 09:48:37 +0900912 .flags = CFTYPE_NOT_ON_ROOT,
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700913 .seq_show = blkcg_print_stat,
914 },
915 { } /* terminate */
916};
917
Bart Van Asschee1f3b942016-06-14 17:04:32 +0200918static struct cftype blkcg_legacy_files[] = {
Vivek Goyal31e4c282009-12-03 12:59:42 -0500919 {
Divyesh Shah84c124d2010-04-09 08:31:19 +0200920 .name = "reset_stats",
Tejun Heo3c798392012-04-16 13:57:25 -0700921 .write_u64 = blkcg_reset_stats,
Vivek Goyal22084192009-12-03 12:59:49 -0500922 },
Tejun Heo4baf6e32012-04-01 12:09:55 -0700923 { } /* terminate */
Vivek Goyal31e4c282009-12-03 12:59:42 -0500924};
925
Tejun Heo9f13ef62012-03-05 13:15:21 -0800926/**
Tejun Heo92fb9742012-11-19 08:13:38 -0800927 * blkcg_css_offline - cgroup css_offline callback
Tejun Heoeb954192013-08-08 20:11:23 -0400928 * @css: css of interest
Tejun Heo9f13ef62012-03-05 13:15:21 -0800929 *
Tejun Heoeb954192013-08-08 20:11:23 -0400930 * This function is called when @css is about to go away and responsible
931 * for shooting down all blkgs associated with @css. blkgs should be
Tejun Heo9f13ef62012-03-05 13:15:21 -0800932 * removed while holding both q and blkcg locks. As blkcg lock is nested
933 * inside q lock, this function performs reverse double lock dancing.
934 *
935 * This is the blkcg counterpart of ioc_release_fn().
936 */
Tejun Heoeb954192013-08-08 20:11:23 -0400937static void blkcg_css_offline(struct cgroup_subsys_state *css)
Vivek Goyal31e4c282009-12-03 12:59:42 -0500938{
Tejun Heoeb954192013-08-08 20:11:23 -0400939 struct blkcg *blkcg = css_to_blkcg(css);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500940
Tejun Heo9f13ef62012-03-05 13:15:21 -0800941 spin_lock_irq(&blkcg->lock);
Tejun Heo7ee9c562012-03-05 13:15:11 -0800942
Tejun Heo9f13ef62012-03-05 13:15:21 -0800943 while (!hlist_empty(&blkcg->blkg_list)) {
Tejun Heo3c798392012-04-16 13:57:25 -0700944 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
945 struct blkcg_gq, blkcg_node);
Tejun Heoc875f4d2012-03-05 13:15:22 -0800946 struct request_queue *q = blkg->q;
Vivek Goyalb1c35762009-12-03 12:59:47 -0500947
Tejun Heo9f13ef62012-03-05 13:15:21 -0800948 if (spin_trylock(q->queue_lock)) {
949 blkg_destroy(blkg);
950 spin_unlock(q->queue_lock);
951 } else {
952 spin_unlock_irq(&blkcg->lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800953 cpu_relax();
Dan Carpentera5567932012-03-29 20:57:08 +0200954 spin_lock_irq(&blkcg->lock);
Jens Axboe0f3942a2010-05-03 14:28:55 +0200955 }
Tejun Heo9f13ef62012-03-05 13:15:21 -0800956 }
Jens Axboe0f3942a2010-05-03 14:28:55 +0200957
Tejun Heo9f13ef62012-03-05 13:15:21 -0800958 spin_unlock_irq(&blkcg->lock);
Tejun Heo52ebea72015-05-22 17:13:37 -0400959
960 wb_blkcg_offline(blkcg);
Tejun Heo7ee9c562012-03-05 13:15:11 -0800961}
962
Tejun Heoeb954192013-08-08 20:11:23 -0400963static void blkcg_css_free(struct cgroup_subsys_state *css)
Tejun Heo7ee9c562012-03-05 13:15:11 -0800964{
Tejun Heoeb954192013-08-08 20:11:23 -0400965 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heobc915e62015-08-18 14:55:08 -0700966 int i;
Tejun Heo7ee9c562012-03-05 13:15:11 -0800967
Tejun Heo7876f932015-07-09 16:39:49 -0400968 mutex_lock(&blkcg_pol_mutex);
Tejun Heoe4a9bde2015-08-18 14:55:16 -0700969
Tejun Heo7876f932015-07-09 16:39:49 -0400970 list_del(&blkcg->all_blkcgs_node);
Tejun Heo7876f932015-07-09 16:39:49 -0400971
Tejun Heobc915e62015-08-18 14:55:08 -0700972 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heoe4a9bde2015-08-18 14:55:16 -0700973 if (blkcg->cpd[i])
974 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
975
976 mutex_unlock(&blkcg_pol_mutex);
977
Tejun Heobc915e62015-08-18 14:55:08 -0700978 kfree(blkcg);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500979}
980
Tejun Heoeb954192013-08-08 20:11:23 -0400981static struct cgroup_subsys_state *
982blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
Vivek Goyal31e4c282009-12-03 12:59:42 -0500983{
Tejun Heo3c798392012-04-16 13:57:25 -0700984 struct blkcg *blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +0200985 struct cgroup_subsys_state *ret;
986 int i;
Vivek Goyal31e4c282009-12-03 12:59:42 -0500987
Tejun Heo7876f932015-07-09 16:39:49 -0400988 mutex_lock(&blkcg_pol_mutex);
989
Tejun Heoeb954192013-08-08 20:11:23 -0400990 if (!parent_css) {
Tejun Heo3c798392012-04-16 13:57:25 -0700991 blkcg = &blkcg_root;
Tejun Heobc915e62015-08-18 14:55:08 -0700992 } else {
993 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
994 if (!blkcg) {
995 ret = ERR_PTR(-ENOMEM);
996 goto free_blkcg;
997 }
Arianna Avanzinie48453c2015-06-05 23:38:42 +0200998 }
Vivek Goyal31e4c282009-12-03 12:59:42 -0500999
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001000 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1001 struct blkcg_policy *pol = blkcg_policy[i];
1002 struct blkcg_policy_data *cpd;
1003
1004 /*
1005 * If the policy hasn't been attached yet, wait for it
1006 * to be attached before doing anything else. Otherwise,
1007 * check if the policy requires any specific per-cgroup
1008 * data: if it does, allocate and initialize it.
1009 */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001010 if (!pol || !pol->cpd_alloc_fn)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001011 continue;
1012
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001013 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001014 if (!cpd) {
1015 ret = ERR_PTR(-ENOMEM);
1016 goto free_pd_blkcg;
1017 }
Tejun Heo81437642015-08-18 14:55:15 -07001018 blkcg->cpd[i] = cpd;
1019 cpd->blkcg = blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001020 cpd->plid = i;
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001021 if (pol->cpd_init_fn)
1022 pol->cpd_init_fn(cpd);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001023 }
1024
Vivek Goyal31e4c282009-12-03 12:59:42 -05001025 spin_lock_init(&blkcg->lock);
Tejun Heo07a61342016-11-21 18:03:32 -05001026 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001027 INIT_HLIST_HEAD(&blkcg->blkg_list);
Tejun Heo52ebea72015-05-22 17:13:37 -04001028#ifdef CONFIG_CGROUP_WRITEBACK
1029 INIT_LIST_HEAD(&blkcg->cgwb_list);
1030#endif
Tejun Heo7876f932015-07-09 16:39:49 -04001031 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1032
1033 mutex_unlock(&blkcg_pol_mutex);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001034 return &blkcg->css;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001035
1036free_pd_blkcg:
1037 for (i--; i >= 0; i--)
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001038 if (blkcg->cpd[i])
1039 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001040free_blkcg:
1041 kfree(blkcg);
Tejun Heo7876f932015-07-09 16:39:49 -04001042 mutex_unlock(&blkcg_pol_mutex);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001043 return ret;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001044}
1045
Tejun Heo5efd6112012-03-05 13:15:12 -08001046/**
1047 * blkcg_init_queue - initialize blkcg part of request queue
1048 * @q: request_queue to initialize
1049 *
1050 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
1051 * part of new request_queue @q.
1052 *
1053 * RETURNS:
1054 * 0 on success, -errno on failure.
1055 */
1056int blkcg_init_queue(struct request_queue *q)
1057{
Tejun Heoec13b1d2015-05-22 17:13:19 -04001058 struct blkcg_gq *new_blkg, *blkg;
1059 bool preloaded;
1060 int ret;
Tejun Heo5efd6112012-03-05 13:15:12 -08001061
Tejun Heoec13b1d2015-05-22 17:13:19 -04001062 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1063 if (!new_blkg)
1064 return -ENOMEM;
1065
1066 preloaded = !radix_tree_preload(GFP_KERNEL);
1067
1068 /*
1069 * Make sure the root blkg exists and count the existing blkgs. As
1070 * @q is bypassing at this point, blkg_lookup_create() can't be
1071 * used. Open code insertion.
1072 */
1073 rcu_read_lock();
1074 spin_lock_irq(q->queue_lock);
1075 blkg = blkg_create(&blkcg_root, q, new_blkg);
1076 spin_unlock_irq(q->queue_lock);
1077 rcu_read_unlock();
1078
1079 if (preloaded)
1080 radix_tree_preload_end();
1081
Hou Tao7bd2d0c2017-02-03 17:19:07 +08001082 if (IS_ERR(blkg))
Tejun Heoec13b1d2015-05-22 17:13:19 -04001083 return PTR_ERR(blkg);
Tejun Heoec13b1d2015-05-22 17:13:19 -04001084
1085 q->root_blkg = blkg;
1086 q->root_rl.blkg = blkg;
1087
1088 ret = blk_throtl_init(q);
1089 if (ret) {
1090 spin_lock_irq(q->queue_lock);
1091 blkg_destroy_all(q);
1092 spin_unlock_irq(q->queue_lock);
1093 }
1094 return ret;
Tejun Heo5efd6112012-03-05 13:15:12 -08001095}
1096
1097/**
1098 * blkcg_drain_queue - drain blkcg part of request_queue
1099 * @q: request_queue to drain
1100 *
1101 * Called from blk_drain_queue(). Responsible for draining blkcg part.
1102 */
1103void blkcg_drain_queue(struct request_queue *q)
1104{
1105 lockdep_assert_held(q->queue_lock);
1106
Tejun Heo0b462c82014-07-05 18:43:21 -04001107 /*
1108 * @q could be exiting and already have destroyed all blkgs as
1109 * indicated by NULL root_blkg. If so, don't confuse policies.
1110 */
1111 if (!q->root_blkg)
1112 return;
1113
Tejun Heo5efd6112012-03-05 13:15:12 -08001114 blk_throtl_drain(q);
1115}
1116
1117/**
1118 * blkcg_exit_queue - exit and release blkcg part of request_queue
1119 * @q: request_queue being released
1120 *
1121 * Called from blk_release_queue(). Responsible for exiting blkcg part.
1122 */
1123void blkcg_exit_queue(struct request_queue *q)
1124{
Tejun Heo6d18b002012-04-13 13:11:35 -07001125 spin_lock_irq(q->queue_lock);
Tejun Heo3c96cb32012-04-13 13:11:34 -07001126 blkg_destroy_all(q);
Tejun Heo6d18b002012-04-13 13:11:35 -07001127 spin_unlock_irq(q->queue_lock);
1128
Tejun Heo5efd6112012-03-05 13:15:12 -08001129 blk_throtl_exit(q);
1130}
1131
Vivek Goyal31e4c282009-12-03 12:59:42 -05001132/*
1133 * We cannot support shared io contexts, as we have no mean to support
1134 * two tasks with the same ioc in two different groups without major rework
1135 * of the main cic data structures. For now we allow a task to change
1136 * its cgroup only if it's the only owner of its ioc.
1137 */
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05001138static int blkcg_can_attach(struct cgroup_taskset *tset)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001139{
Tejun Heobb9d97b2011-12-12 18:12:21 -08001140 struct task_struct *task;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05001141 struct cgroup_subsys_state *dst_css;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001142 struct io_context *ioc;
1143 int ret = 0;
1144
1145 /* task_lock() is needed to avoid races with exit_io_context() */
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05001146 cgroup_taskset_for_each(task, dst_css, tset) {
Tejun Heobb9d97b2011-12-12 18:12:21 -08001147 task_lock(task);
1148 ioc = task->io_context;
1149 if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1150 ret = -EINVAL;
1151 task_unlock(task);
1152 if (ret)
1153 break;
1154 }
Vivek Goyal31e4c282009-12-03 12:59:42 -05001155 return ret;
1156}
1157
Tejun Heo69d7fde2015-08-18 14:55:36 -07001158static void blkcg_bind(struct cgroup_subsys_state *root_css)
1159{
1160 int i;
1161
1162 mutex_lock(&blkcg_pol_mutex);
1163
1164 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1165 struct blkcg_policy *pol = blkcg_policy[i];
1166 struct blkcg *blkcg;
1167
1168 if (!pol || !pol->cpd_bind_fn)
1169 continue;
1170
1171 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1172 if (blkcg->cpd[pol->plid])
1173 pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1174 }
1175 mutex_unlock(&blkcg_pol_mutex);
1176}
1177
Tejun Heoc165b3e2015-08-18 14:55:29 -07001178struct cgroup_subsys io_cgrp_subsys = {
Tejun Heo92fb9742012-11-19 08:13:38 -08001179 .css_alloc = blkcg_css_alloc,
1180 .css_offline = blkcg_css_offline,
1181 .css_free = blkcg_css_free,
Tejun Heo3c798392012-04-16 13:57:25 -07001182 .can_attach = blkcg_can_attach,
Tejun Heo69d7fde2015-08-18 14:55:36 -07001183 .bind = blkcg_bind,
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001184 .dfl_cftypes = blkcg_files,
Tejun Heo880f50e2015-08-18 14:55:30 -07001185 .legacy_cftypes = blkcg_legacy_files,
Tejun Heoc165b3e2015-08-18 14:55:29 -07001186 .legacy_name = "blkio",
Tejun Heo1ced9532014-07-08 18:02:57 -04001187#ifdef CONFIG_MEMCG
1188 /*
1189 * This ensures that, if available, memcg is automatically enabled
1190 * together on the default hierarchy so that the owner cgroup can
1191 * be retrieved from writeback pages.
1192 */
1193 .depends_on = 1 << memory_cgrp_id,
1194#endif
Tejun Heo676f7c82012-04-01 12:09:55 -07001195};
Tejun Heoc165b3e2015-08-18 14:55:29 -07001196EXPORT_SYMBOL_GPL(io_cgrp_subsys);
Tejun Heo676f7c82012-04-01 12:09:55 -07001197
Tejun Heo8bd435b2012-04-13 13:11:28 -07001198/**
Tejun Heoa2b16932012-04-13 13:11:33 -07001199 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1200 * @q: request_queue of interest
1201 * @pol: blkcg policy to activate
1202 *
1203 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1204 * bypass mode to populate its blkgs with policy_data for @pol.
1205 *
1206 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1207 * from IO path. Update of each blkg is protected by both queue and blkcg
1208 * locks so that holding either lock and testing blkcg_policy_enabled() is
1209 * always enough for dereferencing policy data.
1210 *
1211 * The caller is responsible for synchronizing [de]activations and policy
1212 * [un]registerations. Returns 0 on success, -errno on failure.
1213 */
1214int blkcg_activate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001215 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001216{
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001217 struct blkg_policy_data *pd_prealloc = NULL;
Tejun Heoec13b1d2015-05-22 17:13:19 -04001218 struct blkcg_gq *blkg;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001219 int ret;
Tejun Heoa2b16932012-04-13 13:11:33 -07001220
1221 if (blkcg_policy_enabled(q, pol))
1222 return 0;
1223
1224 blk_queue_bypass_start(q);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001225pd_prealloc:
1226 if (!pd_prealloc) {
Tejun Heo001bea72015-08-18 14:55:11 -07001227 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001228 if (!pd_prealloc) {
Tejun Heoa2b16932012-04-13 13:11:33 -07001229 ret = -ENOMEM;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001230 goto out_bypass_end;
Tejun Heoa2b16932012-04-13 13:11:33 -07001231 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001232 }
1233
Tejun Heoa2b16932012-04-13 13:11:33 -07001234 spin_lock_irq(q->queue_lock);
1235
1236 list_for_each_entry(blkg, &q->blkg_list, q_node) {
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001237 struct blkg_policy_data *pd;
Tejun Heoa2b16932012-04-13 13:11:33 -07001238
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001239 if (blkg->pd[pol->plid])
1240 continue;
1241
Tejun Heo07a61342016-11-21 18:03:32 -05001242 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q->node);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001243 if (!pd)
1244 swap(pd, pd_prealloc);
1245 if (!pd) {
1246 spin_unlock_irq(q->queue_lock);
1247 goto pd_prealloc;
1248 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001249
1250 blkg->pd[pol->plid] = pd;
1251 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -08001252 pd->plid = pol->plid;
Tejun Heo3e418712015-08-18 14:55:10 -07001253 if (pol->pd_init_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -07001254 pol->pd_init_fn(pd);
Tejun Heoa2b16932012-04-13 13:11:33 -07001255 }
1256
1257 __set_bit(pol->plid, q->blkcg_pols);
1258 ret = 0;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001259
Tejun Heoa2b16932012-04-13 13:11:33 -07001260 spin_unlock_irq(q->queue_lock);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001261out_bypass_end:
Tejun Heoa2b16932012-04-13 13:11:33 -07001262 blk_queue_bypass_end(q);
Tejun Heo001bea72015-08-18 14:55:11 -07001263 if (pd_prealloc)
1264 pol->pd_free_fn(pd_prealloc);
Tejun Heoa2b16932012-04-13 13:11:33 -07001265 return ret;
1266}
1267EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1268
1269/**
1270 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1271 * @q: request_queue of interest
1272 * @pol: blkcg policy to deactivate
1273 *
1274 * Deactivate @pol on @q. Follows the same synchronization rules as
1275 * blkcg_activate_policy().
1276 */
1277void blkcg_deactivate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001278 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001279{
Tejun Heo3c798392012-04-16 13:57:25 -07001280 struct blkcg_gq *blkg;
Tejun Heoa2b16932012-04-13 13:11:33 -07001281
1282 if (!blkcg_policy_enabled(q, pol))
1283 return;
1284
1285 blk_queue_bypass_start(q);
1286 spin_lock_irq(q->queue_lock);
1287
1288 __clear_bit(pol->plid, q->blkcg_pols);
1289
1290 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1291 /* grab blkcg lock too while removing @pd from @blkg */
1292 spin_lock(&blkg->blkcg->lock);
1293
Tejun Heo001bea72015-08-18 14:55:11 -07001294 if (blkg->pd[pol->plid]) {
Tejun Heoa9520cd2015-08-18 14:55:14 -07001295 if (pol->pd_offline_fn)
1296 pol->pd_offline_fn(blkg->pd[pol->plid]);
Tejun Heo001bea72015-08-18 14:55:11 -07001297 pol->pd_free_fn(blkg->pd[pol->plid]);
1298 blkg->pd[pol->plid] = NULL;
1299 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001300
1301 spin_unlock(&blkg->blkcg->lock);
1302 }
1303
1304 spin_unlock_irq(q->queue_lock);
1305 blk_queue_bypass_end(q);
1306}
1307EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1308
1309/**
Tejun Heo3c798392012-04-16 13:57:25 -07001310 * blkcg_policy_register - register a blkcg policy
1311 * @pol: blkcg policy to register
Tejun Heo8bd435b2012-04-13 13:11:28 -07001312 *
Tejun Heo3c798392012-04-16 13:57:25 -07001313 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1314 * successful registration. Returns 0 on success and -errno on failure.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001315 */
Jens Axboed5bf0292014-06-22 16:31:56 -06001316int blkcg_policy_register(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001317{
Tejun Heo06b285b2015-07-09 16:39:50 -04001318 struct blkcg *blkcg;
Tejun Heo8bd435b2012-04-13 13:11:28 -07001319 int i, ret;
Tejun Heoe8989fa2012-03-05 13:15:20 -08001320
Tejun Heo838f13b2015-07-09 16:39:47 -04001321 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001322 mutex_lock(&blkcg_pol_mutex);
1323
Tejun Heo8bd435b2012-04-13 13:11:28 -07001324 /* find an empty slot */
1325 ret = -ENOSPC;
1326 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo3c798392012-04-16 13:57:25 -07001327 if (!blkcg_policy[i])
Tejun Heo8bd435b2012-04-13 13:11:28 -07001328 break;
1329 if (i >= BLKCG_MAX_POLS)
Tejun Heo838f13b2015-07-09 16:39:47 -04001330 goto err_unlock;
Tejun Heo035d10b2012-03-05 13:15:04 -08001331
Tejun Heo06b285b2015-07-09 16:39:50 -04001332 /* register @pol */
Tejun Heo3c798392012-04-16 13:57:25 -07001333 pol->plid = i;
Tejun Heo06b285b2015-07-09 16:39:50 -04001334 blkcg_policy[pol->plid] = pol;
1335
1336 /* allocate and install cpd's */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001337 if (pol->cpd_alloc_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001338 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1339 struct blkcg_policy_data *cpd;
1340
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001341 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Bart Van Asschebbb427e2016-09-29 08:33:30 -07001342 if (!cpd)
Tejun Heo06b285b2015-07-09 16:39:50 -04001343 goto err_free_cpds;
Tejun Heo06b285b2015-07-09 16:39:50 -04001344
Tejun Heo81437642015-08-18 14:55:15 -07001345 blkcg->cpd[pol->plid] = cpd;
1346 cpd->blkcg = blkcg;
Tejun Heo06b285b2015-07-09 16:39:50 -04001347 cpd->plid = pol->plid;
Tejun Heo81437642015-08-18 14:55:15 -07001348 pol->cpd_init_fn(cpd);
Tejun Heo06b285b2015-07-09 16:39:50 -04001349 }
1350 }
1351
Tejun Heo838f13b2015-07-09 16:39:47 -04001352 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001353
Tejun Heo8bd435b2012-04-13 13:11:28 -07001354 /* everything is in place, add intf files for the new policy */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001355 if (pol->dfl_cftypes)
1356 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1357 pol->dfl_cftypes));
Tejun Heo880f50e2015-08-18 14:55:30 -07001358 if (pol->legacy_cftypes)
Tejun Heoc165b3e2015-08-18 14:55:29 -07001359 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
Tejun Heo880f50e2015-08-18 14:55:30 -07001360 pol->legacy_cftypes));
Tejun Heo838f13b2015-07-09 16:39:47 -04001361 mutex_unlock(&blkcg_pol_register_mutex);
1362 return 0;
1363
Tejun Heo06b285b2015-07-09 16:39:50 -04001364err_free_cpds:
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001365 if (pol->cpd_alloc_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001366 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001367 if (blkcg->cpd[pol->plid]) {
1368 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1369 blkcg->cpd[pol->plid] = NULL;
1370 }
Tejun Heo06b285b2015-07-09 16:39:50 -04001371 }
1372 }
1373 blkcg_policy[pol->plid] = NULL;
Tejun Heo838f13b2015-07-09 16:39:47 -04001374err_unlock:
Tejun Heobc0d6502012-04-13 13:11:26 -07001375 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001376 mutex_unlock(&blkcg_pol_register_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001377 return ret;
Vivek Goyal3e252062009-12-04 10:36:42 -05001378}
Tejun Heo3c798392012-04-16 13:57:25 -07001379EXPORT_SYMBOL_GPL(blkcg_policy_register);
Vivek Goyal3e252062009-12-04 10:36:42 -05001380
Tejun Heo8bd435b2012-04-13 13:11:28 -07001381/**
Tejun Heo3c798392012-04-16 13:57:25 -07001382 * blkcg_policy_unregister - unregister a blkcg policy
1383 * @pol: blkcg policy to unregister
Tejun Heo8bd435b2012-04-13 13:11:28 -07001384 *
Tejun Heo3c798392012-04-16 13:57:25 -07001385 * Undo blkcg_policy_register(@pol). Might sleep.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001386 */
Tejun Heo3c798392012-04-16 13:57:25 -07001387void blkcg_policy_unregister(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001388{
Tejun Heo06b285b2015-07-09 16:39:50 -04001389 struct blkcg *blkcg;
1390
Tejun Heo838f13b2015-07-09 16:39:47 -04001391 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001392
Tejun Heo3c798392012-04-16 13:57:25 -07001393 if (WARN_ON(blkcg_policy[pol->plid] != pol))
Tejun Heo8bd435b2012-04-13 13:11:28 -07001394 goto out_unlock;
1395
1396 /* kill the intf files first */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001397 if (pol->dfl_cftypes)
1398 cgroup_rm_cftypes(pol->dfl_cftypes);
Tejun Heo880f50e2015-08-18 14:55:30 -07001399 if (pol->legacy_cftypes)
1400 cgroup_rm_cftypes(pol->legacy_cftypes);
Tejun Heo44ea53d2012-04-01 14:38:43 -07001401
Tejun Heo06b285b2015-07-09 16:39:50 -04001402 /* remove cpds and unregister */
Tejun Heo838f13b2015-07-09 16:39:47 -04001403 mutex_lock(&blkcg_pol_mutex);
Tejun Heo06b285b2015-07-09 16:39:50 -04001404
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001405 if (pol->cpd_alloc_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001406 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001407 if (blkcg->cpd[pol->plid]) {
1408 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1409 blkcg->cpd[pol->plid] = NULL;
1410 }
Tejun Heo06b285b2015-07-09 16:39:50 -04001411 }
1412 }
Tejun Heo3c798392012-04-16 13:57:25 -07001413 blkcg_policy[pol->plid] = NULL;
Tejun Heo06b285b2015-07-09 16:39:50 -04001414
Tejun Heobc0d6502012-04-13 13:11:26 -07001415 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001416out_unlock:
1417 mutex_unlock(&blkcg_pol_register_mutex);
Vivek Goyal3e252062009-12-04 10:36:42 -05001418}
Tejun Heo3c798392012-04-16 13:57:25 -07001419EXPORT_SYMBOL_GPL(blkcg_policy_unregister);