blob: a232c98fbf4d85675501cf82e5b5ed666ee5f30c [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Jens Axboee34cbd32016-11-09 12:36:15 -07002#ifndef WB_THROTTLE_H
3#define WB_THROTTLE_H
4
5#include <linux/kernel.h>
6#include <linux/atomic.h>
7#include <linux/wait.h>
8#include <linux/timer.h>
9#include <linux/ktime.h>
10
11#include "blk-stat.h"
12
13enum wbt_flags {
14 WBT_TRACKED = 1, /* write, tracked for throttling */
15 WBT_READ = 2, /* read */
16 WBT_KSWAPD = 4, /* write, from kswapd */
17
18 WBT_NR_BITS = 3, /* number of bits */
19};
20
21enum {
22 WBT_NUM_RWQ = 2,
23};
24
Jens Axboed62118b2016-11-28 09:40:34 -070025/*
26 * Enable states. Either off, or on by default (done at init time),
27 * or on through manual setup in sysfs.
28 */
29enum {
30 WBT_STATE_ON_DEFAULT = 1,
31 WBT_STATE_ON_MANUAL = 2,
32};
33
Jens Axboee34cbd32016-11-09 12:36:15 -070034static inline void wbt_clear_state(struct blk_issue_stat *stat)
35{
Shaohua Li88eeca42017-03-27 15:19:41 -070036 stat->stat &= ~BLK_STAT_RES_MASK;
Jens Axboee34cbd32016-11-09 12:36:15 -070037}
38
39static inline enum wbt_flags wbt_stat_to_mask(struct blk_issue_stat *stat)
40{
Shaohua Li88eeca42017-03-27 15:19:41 -070041 return (stat->stat & BLK_STAT_RES_MASK) >> BLK_STAT_RES_SHIFT;
Jens Axboee34cbd32016-11-09 12:36:15 -070042}
43
44static inline void wbt_track(struct blk_issue_stat *stat, enum wbt_flags wb_acct)
45{
Shaohua Li88eeca42017-03-27 15:19:41 -070046 stat->stat |= ((u64) wb_acct) << BLK_STAT_RES_SHIFT;
Jens Axboee34cbd32016-11-09 12:36:15 -070047}
48
49static inline bool wbt_is_tracked(struct blk_issue_stat *stat)
50{
Shaohua Li88eeca42017-03-27 15:19:41 -070051 return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_TRACKED;
Jens Axboee34cbd32016-11-09 12:36:15 -070052}
53
54static inline bool wbt_is_read(struct blk_issue_stat *stat)
55{
Shaohua Li88eeca42017-03-27 15:19:41 -070056 return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_READ;
Jens Axboee34cbd32016-11-09 12:36:15 -070057}
58
Jens Axboee34cbd32016-11-09 12:36:15 -070059struct rq_wait {
60 wait_queue_head_t wait;
61 atomic_t inflight;
62};
63
64struct rq_wb {
65 /*
66 * Settings that govern how we throttle
67 */
68 unsigned int wb_background; /* background writeback */
69 unsigned int wb_normal; /* normal writeback */
70 unsigned int wb_max; /* max throughput writeback */
71 int scale_step;
72 bool scaled_max;
73
Jens Axboed62118b2016-11-28 09:40:34 -070074 short enable_state; /* WBT_STATE_* */
75
Jens Axboee34cbd32016-11-09 12:36:15 -070076 /*
77 * Number of consecutive periods where we don't have enough
78 * information to make a firm scale up/down decision.
79 */
80 unsigned int unknown_cnt;
81
82 u64 win_nsec; /* default window size */
83 u64 cur_win_nsec; /* current window size */
84
Omar Sandoval34dbad52017-03-21 08:56:08 -070085 struct blk_stat_callback *cb;
Jens Axboee34cbd32016-11-09 12:36:15 -070086
87 s64 sync_issue;
88 void *sync_cookie;
89
90 unsigned int wc;
91 unsigned int queue_depth;
92
93 unsigned long last_issue; /* last non-throttled issue */
94 unsigned long last_comp; /* last non-throttled comp */
95 unsigned long min_lat_nsec;
Jens Axboed8a0cbf2016-11-10 21:52:53 -070096 struct request_queue *queue;
Jens Axboee34cbd32016-11-09 12:36:15 -070097 struct rq_wait rq_wait[WBT_NUM_RWQ];
Jens Axboee34cbd32016-11-09 12:36:15 -070098};
99
100static inline unsigned int wbt_inflight(struct rq_wb *rwb)
101{
102 unsigned int i, ret = 0;
103
104 for (i = 0; i < WBT_NUM_RWQ; i++)
105 ret += atomic_read(&rwb->rq_wait[i].inflight);
106
107 return ret;
108}
109
Jens Axboee34cbd32016-11-09 12:36:15 -0700110#ifdef CONFIG_BLK_WBT
111
112void __wbt_done(struct rq_wb *, enum wbt_flags);
113void wbt_done(struct rq_wb *, struct blk_issue_stat *);
114enum wbt_flags wbt_wait(struct rq_wb *, struct bio *, spinlock_t *);
Jens Axboe8054b892016-11-10 21:50:51 -0700115int wbt_init(struct request_queue *);
Jens Axboee34cbd32016-11-09 12:36:15 -0700116void wbt_exit(struct request_queue *);
117void wbt_update_limits(struct rq_wb *);
118void wbt_requeue(struct rq_wb *, struct blk_issue_stat *);
119void wbt_issue(struct rq_wb *, struct blk_issue_stat *);
Jens Axboefa224ee2016-11-28 09:25:50 -0700120void wbt_disable_default(struct request_queue *);
Jan Kara8330cdb2017-04-19 11:33:27 +0200121void wbt_enable_default(struct request_queue *);
Jens Axboee34cbd32016-11-09 12:36:15 -0700122
123void wbt_set_queue_depth(struct rq_wb *, unsigned int);
124void wbt_set_write_cache(struct rq_wb *, bool);
125
Jens Axboe80e091d2016-11-28 09:22:47 -0700126u64 wbt_default_latency_nsec(struct request_queue *);
127
Jens Axboee34cbd32016-11-09 12:36:15 -0700128#else
129
130static inline void __wbt_done(struct rq_wb *rwb, enum wbt_flags flags)
131{
132}
133static inline void wbt_done(struct rq_wb *rwb, struct blk_issue_stat *stat)
134{
135}
136static inline enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio,
137 spinlock_t *lock)
138{
139 return 0;
140}
Jens Axboe8054b892016-11-10 21:50:51 -0700141static inline int wbt_init(struct request_queue *q)
Jens Axboee34cbd32016-11-09 12:36:15 -0700142{
143 return -EINVAL;
144}
145static inline void wbt_exit(struct request_queue *q)
146{
147}
148static inline void wbt_update_limits(struct rq_wb *rwb)
149{
150}
151static inline void wbt_requeue(struct rq_wb *rwb, struct blk_issue_stat *stat)
152{
153}
154static inline void wbt_issue(struct rq_wb *rwb, struct blk_issue_stat *stat)
155{
156}
Jens Axboefa224ee2016-11-28 09:25:50 -0700157static inline void wbt_disable_default(struct request_queue *q)
Jens Axboee34cbd32016-11-09 12:36:15 -0700158{
159}
Jan Kara8330cdb2017-04-19 11:33:27 +0200160static inline void wbt_enable_default(struct request_queue *q)
161{
162}
Jens Axboee34cbd32016-11-09 12:36:15 -0700163static inline void wbt_set_queue_depth(struct rq_wb *rwb, unsigned int depth)
164{
165}
166static inline void wbt_set_write_cache(struct rq_wb *rwb, bool wc)
167{
168}
Jens Axboe80e091d2016-11-28 09:22:47 -0700169static inline u64 wbt_default_latency_nsec(struct request_queue *q)
170{
171 return 0;
172}
Jens Axboee34cbd32016-11-09 12:36:15 -0700173
174#endif /* CONFIG_BLK_WBT */
175
176#endif