blob: 1e2f46a69d50196f71f1fb7ae97e7732c2a8a059 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * net/core/gen_stats.c
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Thomas Graf <tgraf@suug.ch>
10 * Jamal Hadi Salim
11 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
12 *
13 * See Documentation/networking/gen_stats.txt
14 */
15
16#include <linux/types.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/interrupt.h>
20#include <linux/socket.h>
21#include <linux/rtnetlink.h>
22#include <linux/gen_stats.h>
Patrick McHardy1e904742008-01-22 22:11:17 -080023#include <net/netlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <net/gen_stats.h>
25
26
27static inline int
28gnet_stats_copy(struct gnet_dump *d, int type, void *buf, int size)
29{
David S. Miller14ad6642012-04-01 20:47:35 -040030 if (nla_put(d->skb, type, size, buf))
31 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -070032 return 0;
33
Patrick McHardy1e904742008-01-22 22:11:17 -080034nla_put_failure:
Ignacy Gawędzki1c4cff02015-02-13 14:47:05 -080035 kfree(d->xstats);
36 d->xstats = NULL;
37 d->xstats_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 spin_unlock_bh(d->lock);
39 return -1;
40}
41
42/**
43 * gnet_stats_start_copy_compat - start dumping procedure in compatibility mode
44 * @skb: socket buffer to put statistics TLVs into
45 * @type: TLV type for top level statistic TLV
46 * @tc_stats_type: TLV type for backward compatibility struct tc_stats TLV
47 * @xstats_type: TLV type for backward compatibility xstats TLV
48 * @lock: statistics lock
49 * @d: dumping handle
50 *
51 * Initializes the dumping handle, grabs the statistic lock and appends
52 * an empty TLV header to the socket buffer for use a container for all
53 * other statistic TLVS.
54 *
55 * The dumping handle is marked to be in backward compatibility mode telling
56 * all gnet_stats_copy_XXX() functions to fill a local copy of struct tc_stats.
57 *
58 * Returns 0 on success or -1 if the room in the socket buffer was not sufficient.
59 */
60int
61gnet_stats_start_copy_compat(struct sk_buff *skb, int type, int tc_stats_type,
62 int xstats_type, spinlock_t *lock, struct gnet_dump *d)
Eric Dumazet9a429c42008-01-01 21:58:02 -080063 __acquires(lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -070064{
65 memset(d, 0, sizeof(*d));
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +090066
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 spin_lock_bh(lock);
68 d->lock = lock;
69 if (type)
Patrick McHardy1e904742008-01-22 22:11:17 -080070 d->tail = (struct nlattr *)skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -070071 d->skb = skb;
72 d->compat_tc_stats = tc_stats_type;
73 d->compat_xstats = xstats_type;
74
75 if (d->tail)
76 return gnet_stats_copy(d, type, NULL, 0);
77
78 return 0;
79}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +000080EXPORT_SYMBOL(gnet_stats_start_copy_compat);
Linus Torvalds1da177e2005-04-16 15:20:36 -070081
82/**
83 * gnet_stats_start_copy_compat - start dumping procedure in compatibility mode
84 * @skb: socket buffer to put statistics TLVs into
85 * @type: TLV type for top level statistic TLV
86 * @lock: statistics lock
87 * @d: dumping handle
88 *
89 * Initializes the dumping handle, grabs the statistic lock and appends
90 * an empty TLV header to the socket buffer for use a container for all
91 * other statistic TLVS.
92 *
93 * Returns 0 on success or -1 if the room in the socket buffer was not sufficient.
94 */
95int
96gnet_stats_start_copy(struct sk_buff *skb, int type, spinlock_t *lock,
97 struct gnet_dump *d)
98{
99 return gnet_stats_start_copy_compat(skb, type, 0, 0, lock, d);
100}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +0000101EXPORT_SYMBOL(gnet_stats_start_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
John Fastabend22e0f8b2014-09-28 11:52:56 -0700103static void
104__gnet_stats_copy_basic_cpu(struct gnet_stats_basic_packed *bstats,
105 struct gnet_stats_basic_cpu __percpu *cpu)
106{
107 int i;
108
109 for_each_possible_cpu(i) {
110 struct gnet_stats_basic_cpu *bcpu = per_cpu_ptr(cpu, i);
111 unsigned int start;
WANG Cong02c0fc12014-10-06 17:01:33 -0700112 u64 bytes;
113 u32 packets;
John Fastabend22e0f8b2014-09-28 11:52:56 -0700114
115 do {
116 start = u64_stats_fetch_begin_irq(&bcpu->syncp);
117 bytes = bcpu->bstats.bytes;
118 packets = bcpu->bstats.packets;
119 } while (u64_stats_fetch_retry_irq(&bcpu->syncp, start));
120
WANG Cong02c0fc12014-10-06 17:01:33 -0700121 bstats->bytes += bytes;
122 bstats->packets += packets;
John Fastabend22e0f8b2014-09-28 11:52:56 -0700123 }
124}
125
126void
127__gnet_stats_copy_basic(struct gnet_stats_basic_packed *bstats,
128 struct gnet_stats_basic_cpu __percpu *cpu,
129 struct gnet_stats_basic_packed *b)
130{
131 if (cpu) {
132 __gnet_stats_copy_basic_cpu(bstats, cpu);
133 } else {
134 bstats->bytes = b->bytes;
135 bstats->packets = b->packets;
136 }
137}
138EXPORT_SYMBOL(__gnet_stats_copy_basic);
139
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140/**
141 * gnet_stats_copy_basic - copy basic statistics into statistic TLV
142 * @d: dumping handle
143 * @b: basic statistics
144 *
145 * Appends the basic statistics to the top level TLV created by
146 * gnet_stats_start_copy().
147 *
148 * Returns 0 on success or -1 with the statistic lock released
149 * if the room in the socket buffer was not sufficient.
150 */
151int
John Fastabend22e0f8b2014-09-28 11:52:56 -0700152gnet_stats_copy_basic(struct gnet_dump *d,
153 struct gnet_stats_basic_cpu __percpu *cpu,
154 struct gnet_stats_basic_packed *b)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155{
John Fastabend22e0f8b2014-09-28 11:52:56 -0700156 struct gnet_stats_basic_packed bstats = {0};
157
158 __gnet_stats_copy_basic(&bstats, cpu, b);
159
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 if (d->compat_tc_stats) {
John Fastabend22e0f8b2014-09-28 11:52:56 -0700161 d->tc_stats.bytes = bstats.bytes;
162 d->tc_stats.packets = bstats.packets;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 }
164
Eric Dumazetc1a8f1f2009-08-16 09:36:49 +0000165 if (d->tail) {
166 struct gnet_stats_basic sb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167
Eric Dumazetc1a8f1f2009-08-16 09:36:49 +0000168 memset(&sb, 0, sizeof(sb));
John Fastabend22e0f8b2014-09-28 11:52:56 -0700169 sb.bytes = bstats.bytes;
170 sb.packets = bstats.packets;
Eric Dumazetc1a8f1f2009-08-16 09:36:49 +0000171 return gnet_stats_copy(d, TCA_STATS_BASIC, &sb, sizeof(sb));
172 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 return 0;
174}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +0000175EXPORT_SYMBOL(gnet_stats_copy_basic);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176
177/**
178 * gnet_stats_copy_rate_est - copy rate estimator statistics into statistics TLV
179 * @d: dumping handle
Eric Dumazetd250a5f2009-10-02 10:32:18 +0000180 * @b: basic statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 * @r: rate estimator statistics
182 *
183 * Appends the rate estimator statistics to the top level TLV created by
184 * gnet_stats_start_copy().
185 *
186 * Returns 0 on success or -1 with the statistic lock released
187 * if the room in the socket buffer was not sufficient.
188 */
189int
Eric Dumazetd250a5f2009-10-02 10:32:18 +0000190gnet_stats_copy_rate_est(struct gnet_dump *d,
191 const struct gnet_stats_basic_packed *b,
Eric Dumazet45203a32013-06-06 08:43:22 -0700192 struct gnet_stats_rate_est64 *r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193{
Eric Dumazet45203a32013-06-06 08:43:22 -0700194 struct gnet_stats_rate_est est;
195 int res;
196
Eric Dumazetd250a5f2009-10-02 10:32:18 +0000197 if (b && !gen_estimator_active(b, r))
198 return 0;
199
Eric Dumazet45203a32013-06-06 08:43:22 -0700200 est.bps = min_t(u64, UINT_MAX, r->bps);
201 /* we have some time before reaching 2^32 packets per second */
202 est.pps = r->pps;
203
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 if (d->compat_tc_stats) {
Eric Dumazet45203a32013-06-06 08:43:22 -0700205 d->tc_stats.bps = est.bps;
206 d->tc_stats.pps = est.pps;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 }
208
Eric Dumazet45203a32013-06-06 08:43:22 -0700209 if (d->tail) {
210 res = gnet_stats_copy(d, TCA_STATS_RATE_EST, &est, sizeof(est));
211 if (res < 0 || est.bps == r->bps)
212 return res;
213 /* emit 64bit stats only if needed */
214 return gnet_stats_copy(d, TCA_STATS_RATE_EST64, r, sizeof(*r));
215 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217 return 0;
218}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +0000219EXPORT_SYMBOL(gnet_stats_copy_rate_est);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
John Fastabendb0ab6f92014-09-28 11:54:24 -0700221static void
222__gnet_stats_copy_queue_cpu(struct gnet_stats_queue *qstats,
223 const struct gnet_stats_queue __percpu *q)
224{
225 int i;
226
227 for_each_possible_cpu(i) {
228 const struct gnet_stats_queue *qcpu = per_cpu_ptr(q, i);
229
230 qstats->qlen = 0;
231 qstats->backlog += qcpu->backlog;
232 qstats->drops += qcpu->drops;
233 qstats->requeues += qcpu->requeues;
234 qstats->overlimits += qcpu->overlimits;
235 }
236}
237
238static void __gnet_stats_copy_queue(struct gnet_stats_queue *qstats,
239 const struct gnet_stats_queue __percpu *cpu,
240 const struct gnet_stats_queue *q,
241 __u32 qlen)
242{
243 if (cpu) {
244 __gnet_stats_copy_queue_cpu(qstats, cpu);
245 } else {
246 qstats->qlen = q->qlen;
247 qstats->backlog = q->backlog;
248 qstats->drops = q->drops;
249 qstats->requeues = q->requeues;
250 qstats->overlimits = q->overlimits;
251 }
252
253 qstats->qlen = qlen;
254}
255
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256/**
257 * gnet_stats_copy_queue - copy queue statistics into statistics TLV
258 * @d: dumping handle
John Fastabendb0ab6f92014-09-28 11:54:24 -0700259 * @cpu_q: per cpu queue statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 * @q: queue statistics
John Fastabend64015852014-09-28 11:53:57 -0700261 * @qlen: queue length statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 *
263 * Appends the queue statistics to the top level TLV created by
John Fastabendb0ab6f92014-09-28 11:54:24 -0700264 * gnet_stats_start_copy(). Using per cpu queue statistics if
265 * they are available.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 *
267 * Returns 0 on success or -1 with the statistic lock released
268 * if the room in the socket buffer was not sufficient.
269 */
270int
John Fastabend64015852014-09-28 11:53:57 -0700271gnet_stats_copy_queue(struct gnet_dump *d,
John Fastabendb0ab6f92014-09-28 11:54:24 -0700272 struct gnet_stats_queue __percpu *cpu_q,
John Fastabend64015852014-09-28 11:53:57 -0700273 struct gnet_stats_queue *q, __u32 qlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274{
John Fastabendb0ab6f92014-09-28 11:54:24 -0700275 struct gnet_stats_queue qstats = {0};
276
277 __gnet_stats_copy_queue(&qstats, cpu_q, q, qlen);
John Fastabend64015852014-09-28 11:53:57 -0700278
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 if (d->compat_tc_stats) {
John Fastabendb0ab6f92014-09-28 11:54:24 -0700280 d->tc_stats.drops = qstats.drops;
281 d->tc_stats.qlen = qstats.qlen;
282 d->tc_stats.backlog = qstats.backlog;
283 d->tc_stats.overlimits = qstats.overlimits;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 }
285
286 if (d->tail)
John Fastabendb0ab6f92014-09-28 11:54:24 -0700287 return gnet_stats_copy(d, TCA_STATS_QUEUE,
288 &qstats, sizeof(qstats));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289
290 return 0;
291}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +0000292EXPORT_SYMBOL(gnet_stats_copy_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293
294/**
295 * gnet_stats_copy_app - copy application specific statistics into statistics TLV
296 * @d: dumping handle
297 * @st: application specific statistics data
298 * @len: length of data
299 *
Masanari Iidae793c0f2014-09-04 23:44:36 +0900300 * Appends the application specific statistics to the top level TLV created by
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 * gnet_stats_start_copy() and remembers the data for XSTATS if the dumping
302 * handle is in backward compatibility mode.
303 *
304 * Returns 0 on success or -1 with the statistic lock released
305 * if the room in the socket buffer was not sufficient.
306 */
307int
308gnet_stats_copy_app(struct gnet_dump *d, void *st, int len)
309{
310 if (d->compat_xstats) {
Ignacy Gawędzki1c4cff02015-02-13 14:47:05 -0800311 d->xstats = kmemdup(st, len, GFP_ATOMIC);
312 if (!d->xstats)
313 goto err_out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 d->xstats_len = len;
315 }
316
317 if (d->tail)
318 return gnet_stats_copy(d, TCA_STATS_APP, st, len);
319
320 return 0;
Ignacy Gawędzki1c4cff02015-02-13 14:47:05 -0800321
322err_out:
323 d->xstats_len = 0;
324 spin_unlock_bh(d->lock);
325 return -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +0000327EXPORT_SYMBOL(gnet_stats_copy_app);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328
329/**
330 * gnet_stats_finish_copy - finish dumping procedure
331 * @d: dumping handle
332 *
333 * Corrects the length of the top level TLV to include all TLVs added
334 * by gnet_stats_copy_XXX() calls. Adds the backward compatibility TLVs
335 * if gnet_stats_start_copy_compat() was used and releases the statistics
336 * lock.
337 *
338 * Returns 0 on success or -1 with the statistic lock released
339 * if the room in the socket buffer was not sufficient.
340 */
341int
342gnet_stats_finish_copy(struct gnet_dump *d)
343{
344 if (d->tail)
Patrick McHardy1e904742008-01-22 22:11:17 -0800345 d->tail->nla_len = skb_tail_pointer(d->skb) - (u8 *)d->tail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
347 if (d->compat_tc_stats)
348 if (gnet_stats_copy(d, d->compat_tc_stats, &d->tc_stats,
349 sizeof(d->tc_stats)) < 0)
350 return -1;
351
352 if (d->compat_xstats && d->xstats) {
353 if (gnet_stats_copy(d, d->compat_xstats, d->xstats,
354 d->xstats_len) < 0)
355 return -1;
356 }
357
Ignacy Gawędzki1c4cff02015-02-13 14:47:05 -0800358 kfree(d->xstats);
359 d->xstats = NULL;
360 d->xstats_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 spin_unlock_bh(d->lock);
362 return 0;
363}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364EXPORT_SYMBOL(gnet_stats_finish_copy);