blob: 2dd36347252ab1eacd76fd0fc7814341dc58f4f8 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Jens Axboecf43e6b2016-11-07 21:32:37 -07002#ifndef BLK_STAT_H
3#define BLK_STAT_H
4
Omar Sandoval34dbad52017-03-21 08:56:08 -07005#include <linux/kernel.h>
6#include <linux/blkdev.h>
7#include <linux/ktime.h>
8#include <linux/rcupdate.h>
9#include <linux/timer.h>
Jens Axboecf43e6b2016-11-07 21:32:37 -070010
11/*
Shaohua Li88eeca42017-03-27 15:19:41 -070012 * from upper:
13 * 3 bits: reserved for other usage
14 * 12 bits: size
15 * 49 bits: time
Jens Axboecf43e6b2016-11-07 21:32:37 -070016 */
17#define BLK_STAT_RES_BITS 3
Shaohua Li88eeca42017-03-27 15:19:41 -070018#define BLK_STAT_SIZE_BITS 12
19#define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS)
20#define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS)
21#define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1)
22#define BLK_STAT_SIZE_MASK \
23 (((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT)
24#define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1))
Jens Axboecf43e6b2016-11-07 21:32:37 -070025
Omar Sandoval34dbad52017-03-21 08:56:08 -070026/**
27 * struct blk_stat_callback - Block statistics callback.
28 *
29 * A &struct blk_stat_callback is associated with a &struct request_queue. While
30 * @timer is active, that queue's request completion latencies are sorted into
31 * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the
32 * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked.
33 */
34struct blk_stat_callback {
35 /*
36 * @list: RCU list of callbacks for a &struct request_queue.
37 */
38 struct list_head list;
39
40 /**
41 * @timer: Timer for the next callback invocation.
42 */
43 struct timer_list timer;
44
45 /**
46 * @cpu_stat: Per-cpu statistics buckets.
47 */
48 struct blk_rq_stat __percpu *cpu_stat;
49
50 /**
51 * @bucket_fn: Given a request, returns which statistics bucket it
Stephen Batesa37244e2017-04-20 15:29:16 -060052 * should be accounted under. Return -1 for no bucket for this
53 * request.
Omar Sandoval34dbad52017-03-21 08:56:08 -070054 */
Stephen Batesa37244e2017-04-20 15:29:16 -060055 int (*bucket_fn)(const struct request *);
Omar Sandoval34dbad52017-03-21 08:56:08 -070056
57 /**
58 * @buckets: Number of statistics buckets.
59 */
60 unsigned int buckets;
61
62 /**
63 * @stat: Array of statistics buckets.
64 */
65 struct blk_rq_stat *stat;
66
67 /**
68 * @fn: Callback function.
69 */
70 void (*timer_fn)(struct blk_stat_callback *);
71
72 /**
73 * @data: Private pointer for the user.
74 */
75 void *data;
76
77 struct rcu_head rcu;
78};
79
80struct blk_queue_stats *blk_alloc_queue_stats(void);
81void blk_free_queue_stats(struct blk_queue_stats *);
82
83void blk_stat_add(struct request *);
84
Jens Axboecf43e6b2016-11-07 21:32:37 -070085static inline u64 __blk_stat_time(u64 time)
86{
87 return time & BLK_STAT_TIME_MASK;
88}
89
90static inline u64 blk_stat_time(struct blk_issue_stat *stat)
91{
Shaohua Li88eeca42017-03-27 15:19:41 -070092 return __blk_stat_time(stat->stat);
93}
94
95static inline sector_t blk_capped_size(sector_t size)
96{
97 return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1);
98}
99
100static inline sector_t blk_stat_size(struct blk_issue_stat *stat)
101{
102 return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT;
103}
104
105static inline void blk_stat_set_issue(struct blk_issue_stat *stat,
106 sector_t size)
107{
108 stat->stat = (stat->stat & BLK_STAT_RES_MASK) |
109 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) |
110 (((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT);
Jens Axboecf43e6b2016-11-07 21:32:37 -0700111}
112
Shaohua Lib9147dd2017-03-27 15:19:42 -0700113/* record time/size info in request but not add a callback */
114void blk_stat_enable_accounting(struct request_queue *q);
115
Omar Sandoval34dbad52017-03-21 08:56:08 -0700116/**
117 * blk_stat_alloc_callback() - Allocate a block statistics callback.
118 * @timer_fn: Timer callback function.
119 * @bucket_fn: Bucket callback function.
120 * @buckets: Number of statistics buckets.
121 * @data: Value for the @data field of the &struct blk_stat_callback.
122 *
123 * See &struct blk_stat_callback for details on the callback functions.
124 *
125 * Return: &struct blk_stat_callback on success or NULL on ENOMEM.
126 */
127struct blk_stat_callback *
128blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
Stephen Batesa37244e2017-04-20 15:29:16 -0600129 int (*bucket_fn)(const struct request *),
Omar Sandoval34dbad52017-03-21 08:56:08 -0700130 unsigned int buckets, void *data);
131
132/**
133 * blk_stat_add_callback() - Add a block statistics callback to be run on a
134 * request queue.
135 * @q: The request queue.
136 * @cb: The callback.
137 *
138 * Note that a single &struct blk_stat_callback can only be added to a single
139 * &struct request_queue.
140 */
141void blk_stat_add_callback(struct request_queue *q,
142 struct blk_stat_callback *cb);
143
144/**
145 * blk_stat_remove_callback() - Remove a block statistics callback from a
146 * request queue.
147 * @q: The request queue.
148 * @cb: The callback.
149 *
150 * When this returns, the callback is not running on any CPUs and will not be
151 * called again unless readded.
152 */
153void blk_stat_remove_callback(struct request_queue *q,
154 struct blk_stat_callback *cb);
155
156/**
157 * blk_stat_free_callback() - Free a block statistics callback.
158 * @cb: The callback.
159 *
160 * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must
161 * not be associated with a request queue. I.e., if it was previously added with
162 * blk_stat_add_callback(), it must also have been removed since then with
163 * blk_stat_remove_callback().
164 */
165void blk_stat_free_callback(struct blk_stat_callback *cb);
166
167/**
168 * blk_stat_is_active() - Check if a block statistics callback is currently
169 * gathering statistics.
170 * @cb: The callback.
171 */
172static inline bool blk_stat_is_active(struct blk_stat_callback *cb)
173{
174 return timer_pending(&cb->timer);
175}
176
177/**
178 * blk_stat_activate_nsecs() - Gather block statistics during a time window in
179 * nanoseconds.
180 * @cb: The callback.
181 * @nsecs: Number of nanoseconds to gather statistics for.
182 *
183 * The timer callback will be called when the window expires.
184 */
185static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb,
186 u64 nsecs)
187{
188 mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs));
189}
190
191/**
192 * blk_stat_activate_msecs() - Gather block statistics during a time window in
193 * milliseconds.
194 * @cb: The callback.
195 * @msecs: Number of milliseconds to gather statistics for.
196 *
197 * The timer callback will be called when the window expires.
198 */
199static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb,
200 unsigned int msecs)
201{
202 mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs));
203}
204
Jens Axboecf43e6b2016-11-07 21:32:37 -0700205#endif