blob: c52356d90fe3854f05faebda241707a1b112a543 [file] [log] [blame]
Jens Axboecf43e6b2016-11-07 21:32:37 -07001/*
2 * Block stat tracking code
3 *
4 * Copyright (C) 2016 Jens Axboe
5 */
6#include <linux/kernel.h>
Omar Sandoval34dbad52017-03-21 08:56:08 -07007#include <linux/rculist.h>
Jens Axboecf43e6b2016-11-07 21:32:37 -07008#include <linux/blk-mq.h>
9
10#include "blk-stat.h"
11#include "blk-mq.h"
Shaohua Lib9147dd2017-03-27 15:19:42 -070012#include "blk.h"
Jens Axboecf43e6b2016-11-07 21:32:37 -070013
Omar Sandoval48752532017-03-21 08:56:07 -070014#define BLK_RQ_STAT_BATCH 64
15
Omar Sandoval34dbad52017-03-21 08:56:08 -070016struct blk_queue_stats {
17 struct list_head callbacks;
18 spinlock_t lock;
Shaohua Lib9147dd2017-03-27 15:19:42 -070019 bool enable_accounting;
Omar Sandoval34dbad52017-03-21 08:56:08 -070020};
21
Omar Sandoval34dbad52017-03-21 08:56:08 -070022static void blk_stat_init(struct blk_rq_stat *stat)
23{
24 stat->min = -1ULL;
25 stat->max = stat->nr_samples = stat->mean = 0;
26 stat->batch = stat->nr_batch = 0;
27}
28
Jens Axboecf43e6b2016-11-07 21:32:37 -070029static void blk_stat_flush_batch(struct blk_rq_stat *stat)
30{
31 const s32 nr_batch = READ_ONCE(stat->nr_batch);
Shaohua Li209200e2016-12-02 17:13:09 -080032 const s32 nr_samples = READ_ONCE(stat->nr_samples);
Jens Axboecf43e6b2016-11-07 21:32:37 -070033
34 if (!nr_batch)
35 return;
36 if (!nr_samples)
37 stat->mean = div64_s64(stat->batch, nr_batch);
38 else {
39 stat->mean = div64_s64((stat->mean * nr_samples) +
40 stat->batch,
41 nr_batch + nr_samples);
42 }
43
44 stat->nr_samples += nr_batch;
45 stat->nr_batch = stat->batch = 0;
46}
47
48static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
49{
Omar Sandoval7d8d0012017-03-16 09:46:14 -060050 blk_stat_flush_batch(src);
51
Jens Axboecf43e6b2016-11-07 21:32:37 -070052 if (!src->nr_samples)
53 return;
54
Jens Axboecf43e6b2016-11-07 21:32:37 -070055 dst->min = min(dst->min, src->min);
56 dst->max = max(dst->max, src->max);
57
58 if (!dst->nr_samples)
59 dst->mean = src->mean;
60 else {
61 dst->mean = div64_s64((src->mean * src->nr_samples) +
62 (dst->mean * dst->nr_samples),
63 dst->nr_samples + src->nr_samples);
64 }
65 dst->nr_samples += src->nr_samples;
66}
67
Omar Sandoval34dbad52017-03-21 08:56:08 -070068static void __blk_stat_add(struct blk_rq_stat *stat, u64 value)
Jens Axboecf43e6b2016-11-07 21:32:37 -070069{
Omar Sandoval34dbad52017-03-21 08:56:08 -070070 stat->min = min(stat->min, value);
71 stat->max = max(stat->max, value);
Jens Axboecf43e6b2016-11-07 21:32:37 -070072
73 if (stat->batch + value < stat->batch ||
74 stat->nr_batch + 1 == BLK_RQ_STAT_BATCH)
75 blk_stat_flush_batch(stat);
76
77 stat->batch += value;
78 stat->nr_batch++;
79}
80
Omar Sandoval34dbad52017-03-21 08:56:08 -070081void blk_stat_add(struct request *rq)
Jens Axboecf43e6b2016-11-07 21:32:37 -070082{
Omar Sandoval34dbad52017-03-21 08:56:08 -070083 struct request_queue *q = rq->q;
84 struct blk_stat_callback *cb;
85 struct blk_rq_stat *stat;
86 int bucket;
87 s64 now, value;
Jens Axboecf43e6b2016-11-07 21:32:37 -070088
Omar Sandoval34dbad52017-03-21 08:56:08 -070089 now = __blk_stat_time(ktime_to_ns(ktime_get()));
90 if (now < blk_stat_time(&rq->issue_stat))
91 return;
92
93 value = now - blk_stat_time(&rq->issue_stat);
94
Shaohua Lib9147dd2017-03-27 15:19:42 -070095 blk_throtl_stat_add(rq, value);
96
Omar Sandoval34dbad52017-03-21 08:56:08 -070097 rcu_read_lock();
98 list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
Jens Axboed3738122017-05-09 11:39:56 -060099 if (!blk_stat_is_active(cb))
100 continue;
101
102 bucket = cb->bucket_fn(rq);
103 if (bucket < 0)
104 continue;
105
106 stat = &get_cpu_ptr(cb->cpu_stat)[bucket];
107 __blk_stat_add(stat, value);
108 put_cpu_ptr(cb->cpu_stat);
Jens Axboecf43e6b2016-11-07 21:32:37 -0700109 }
Omar Sandoval34dbad52017-03-21 08:56:08 -0700110 rcu_read_unlock();
Jens Axboecf43e6b2016-11-07 21:32:37 -0700111}
112
Omar Sandoval34dbad52017-03-21 08:56:08 -0700113static void blk_stat_timer_fn(unsigned long data)
Jens Axboecf43e6b2016-11-07 21:32:37 -0700114{
Omar Sandoval34dbad52017-03-21 08:56:08 -0700115 struct blk_stat_callback *cb = (void *)data;
116 unsigned int bucket;
117 int cpu;
Jens Axboecf43e6b2016-11-07 21:32:37 -0700118
Omar Sandoval34dbad52017-03-21 08:56:08 -0700119 for (bucket = 0; bucket < cb->buckets; bucket++)
120 blk_stat_init(&cb->stat[bucket]);
121
122 for_each_online_cpu(cpu) {
123 struct blk_rq_stat *cpu_stat;
124
125 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
126 for (bucket = 0; bucket < cb->buckets; bucket++) {
127 blk_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
128 blk_stat_init(&cpu_stat[bucket]);
129 }
Jens Axboecf43e6b2016-11-07 21:32:37 -0700130 }
131
Omar Sandoval34dbad52017-03-21 08:56:08 -0700132 cb->timer_fn(cb);
133}
134
135struct blk_stat_callback *
136blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
Stephen Batesa37244e2017-04-20 15:29:16 -0600137 int (*bucket_fn)(const struct request *),
Omar Sandoval34dbad52017-03-21 08:56:08 -0700138 unsigned int buckets, void *data)
139{
140 struct blk_stat_callback *cb;
141
142 cb = kmalloc(sizeof(*cb), GFP_KERNEL);
143 if (!cb)
144 return NULL;
145
146 cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
147 GFP_KERNEL);
148 if (!cb->stat) {
149 kfree(cb);
150 return NULL;
151 }
152 cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
153 __alignof__(struct blk_rq_stat));
154 if (!cb->cpu_stat) {
155 kfree(cb->stat);
156 kfree(cb);
157 return NULL;
158 }
159
160 cb->timer_fn = timer_fn;
161 cb->bucket_fn = bucket_fn;
162 cb->data = data;
163 cb->buckets = buckets;
164 setup_timer(&cb->timer, blk_stat_timer_fn, (unsigned long)cb);
165
166 return cb;
167}
168EXPORT_SYMBOL_GPL(blk_stat_alloc_callback);
169
170void blk_stat_add_callback(struct request_queue *q,
171 struct blk_stat_callback *cb)
172{
173 unsigned int bucket;
174 int cpu;
175
176 for_each_possible_cpu(cpu) {
177 struct blk_rq_stat *cpu_stat;
178
179 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
180 for (bucket = 0; bucket < cb->buckets; bucket++)
181 blk_stat_init(&cpu_stat[bucket]);
182 }
183
184 spin_lock(&q->stats->lock);
185 list_add_tail_rcu(&cb->list, &q->stats->callbacks);
186 set_bit(QUEUE_FLAG_STATS, &q->queue_flags);
187 spin_unlock(&q->stats->lock);
188}
189EXPORT_SYMBOL_GPL(blk_stat_add_callback);
190
191void blk_stat_remove_callback(struct request_queue *q,
192 struct blk_stat_callback *cb)
193{
194 spin_lock(&q->stats->lock);
195 list_del_rcu(&cb->list);
Shaohua Lib9147dd2017-03-27 15:19:42 -0700196 if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting)
Omar Sandoval34dbad52017-03-21 08:56:08 -0700197 clear_bit(QUEUE_FLAG_STATS, &q->queue_flags);
198 spin_unlock(&q->stats->lock);
199
200 del_timer_sync(&cb->timer);
201}
202EXPORT_SYMBOL_GPL(blk_stat_remove_callback);
203
204static void blk_stat_free_callback_rcu(struct rcu_head *head)
205{
206 struct blk_stat_callback *cb;
207
208 cb = container_of(head, struct blk_stat_callback, rcu);
209 free_percpu(cb->cpu_stat);
210 kfree(cb->stat);
211 kfree(cb);
212}
213
214void blk_stat_free_callback(struct blk_stat_callback *cb)
215{
Jens Axboea83b5762017-03-21 17:20:01 -0600216 if (cb)
217 call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
Omar Sandoval34dbad52017-03-21 08:56:08 -0700218}
219EXPORT_SYMBOL_GPL(blk_stat_free_callback);
220
Shaohua Lib9147dd2017-03-27 15:19:42 -0700221void blk_stat_enable_accounting(struct request_queue *q)
222{
223 spin_lock(&q->stats->lock);
224 q->stats->enable_accounting = true;
225 set_bit(QUEUE_FLAG_STATS, &q->queue_flags);
226 spin_unlock(&q->stats->lock);
227}
228
Omar Sandoval34dbad52017-03-21 08:56:08 -0700229struct blk_queue_stats *blk_alloc_queue_stats(void)
230{
231 struct blk_queue_stats *stats;
232
233 stats = kmalloc(sizeof(*stats), GFP_KERNEL);
234 if (!stats)
235 return NULL;
236
237 INIT_LIST_HEAD(&stats->callbacks);
238 spin_lock_init(&stats->lock);
Shaohua Lib9147dd2017-03-27 15:19:42 -0700239 stats->enable_accounting = false;
Omar Sandoval34dbad52017-03-21 08:56:08 -0700240
241 return stats;
242}
243
244void blk_free_queue_stats(struct blk_queue_stats *stats)
245{
246 if (!stats)
247 return;
248
249 WARN_ON(!list_empty(&stats->callbacks));
250
251 kfree(stats);
Jens Axboecf43e6b2016-11-07 21:32:37 -0700252}