blob: b5c2cf2aa6d4bc00e5c4bac2b3326db3d785c8be [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * net/sched/sch_api.c Packet scheduler API.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10 *
11 * Fixes:
12 *
13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
16 */
17
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/module.h>
19#include <linux/types.h>
20#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/string.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/errno.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/skbuff.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/init.h>
25#include <linux/proc_fs.h>
26#include <linux/seq_file.h>
27#include <linux/kmod.h>
28#include <linux/list.h>
Patrick McHardy41794772007-03-16 01:19:15 -070029#include <linux/hrtimer.h>
Jarek Poplawski25bfcd52008-08-18 20:53:34 -070030#include <linux/lockdep.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090031#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020033#include <net/net_namespace.h>
Denis V. Lunevb8542722007-12-01 00:21:31 +110034#include <net/sock.h>
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -070035#include <net/netlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <net/pkt_sched.h>
37
Tom Goff7316ae82010-03-19 15:40:13 +000038static int qdisc_notify(struct net *net, struct sk_buff *oskb,
39 struct nlmsghdr *n, u32 clid,
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 struct Qdisc *old, struct Qdisc *new);
Tom Goff7316ae82010-03-19 15:40:13 +000041static int tclass_notify(struct net *net, struct sk_buff *oskb,
42 struct nlmsghdr *n, struct Qdisc *q,
43 unsigned long cl, int event);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45/*
46
47 Short review.
48 -------------
49
50 This file consists of two interrelated parts:
51
52 1. queueing disciplines manager frontend.
53 2. traffic classes manager frontend.
54
55 Generally, queueing discipline ("qdisc") is a black box,
56 which is able to enqueue packets and to dequeue them (when
57 device is ready to send something) in order and at times
58 determined by algorithm hidden in it.
59
60 qdisc's are divided to two categories:
61 - "queues", which have no internal structure visible from outside.
62 - "schedulers", which split all the packets to "traffic classes",
63 using "packet classifiers" (look at cls_api.c)
64
65 In turn, classes may have child qdiscs (as rule, queues)
66 attached to them etc. etc. etc.
67
68 The goal of the routines in this file is to translate
69 information supplied by user in the form of handles
70 to more intelligible for kernel form, to make some sanity
71 checks and part of work, which is common to all qdiscs
72 and to provide rtnetlink notifications.
73
74 All real intelligent work is done inside qdisc modules.
75
76
77
78 Every discipline has two major routines: enqueue and dequeue.
79
80 ---dequeue
81
82 dequeue usually returns a skb to send. It is allowed to return NULL,
83 but it does not mean that queue is empty, it just means that
84 discipline does not want to send anything this time.
85 Queue is really empty if q->q.qlen == 0.
86 For complicated disciplines with multiple queues q->q is not
87 real packet queue, but however q->q.qlen must be valid.
88
89 ---enqueue
90
91 enqueue returns 0, if packet was enqueued successfully.
92 If packet (this one or another one) was dropped, it returns
93 not zero error code.
94 NET_XMIT_DROP - this packet dropped
95 Expected action: do not backoff, but wait until queue will clear.
96 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
97 Expected action: backoff or ignore
98 NET_XMIT_POLICED - dropped by police.
99 Expected action: backoff or error to real-time apps.
100
101 Auxiliary routines:
102
Jarek Poplawski99c0db22008-10-31 00:45:27 -0700103 ---peek
104
105 like dequeue but without removing a packet from the queue
106
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 ---reset
108
109 returns qdisc to initial state: purge all buffers, clear all
110 timers, counters (except for statistics) etc.
111
112 ---init
113
114 initializes newly created qdisc.
115
116 ---destroy
117
118 destroys resources allocated by init and during lifetime of qdisc.
119
120 ---change
121
122 changes qdisc parameters.
123 */
124
125/* Protects list of registered TC modules. It is pure SMP lock. */
126static DEFINE_RWLOCK(qdisc_mod_lock);
127
128
129/************************************************
130 * Queueing disciplines manipulation. *
131 ************************************************/
132
133
134/* The list of all installed queueing disciplines. */
135
136static struct Qdisc_ops *qdisc_base;
137
Zhi Yong Wu21eb2182014-01-01 04:34:51 +0800138/* Register/unregister queueing discipline */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
140int register_qdisc(struct Qdisc_ops *qops)
141{
142 struct Qdisc_ops *q, **qp;
143 int rc = -EEXIST;
144
145 write_lock(&qdisc_mod_lock);
146 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
147 if (!strcmp(qops->id, q->id))
148 goto out;
149
150 if (qops->enqueue == NULL)
151 qops->enqueue = noop_qdisc_ops.enqueue;
Jarek Poplawski99c0db22008-10-31 00:45:27 -0700152 if (qops->peek == NULL) {
Jarek Poplawski68fd26b2010-08-09 12:18:48 +0000153 if (qops->dequeue == NULL)
Jarek Poplawski99c0db22008-10-31 00:45:27 -0700154 qops->peek = noop_qdisc_ops.peek;
Jarek Poplawski68fd26b2010-08-09 12:18:48 +0000155 else
156 goto out_einval;
Jarek Poplawski99c0db22008-10-31 00:45:27 -0700157 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 if (qops->dequeue == NULL)
159 qops->dequeue = noop_qdisc_ops.dequeue;
160
Jarek Poplawski68fd26b2010-08-09 12:18:48 +0000161 if (qops->cl_ops) {
162 const struct Qdisc_class_ops *cops = qops->cl_ops;
163
Jarek Poplawski3e9e5a52010-08-10 22:31:20 +0000164 if (!(cops->get && cops->put && cops->walk && cops->leaf))
Jarek Poplawski68fd26b2010-08-09 12:18:48 +0000165 goto out_einval;
166
167 if (cops->tcf_chain && !(cops->bind_tcf && cops->unbind_tcf))
168 goto out_einval;
169 }
170
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 qops->next = NULL;
172 *qp = qops;
173 rc = 0;
174out:
175 write_unlock(&qdisc_mod_lock);
176 return rc;
Jarek Poplawski68fd26b2010-08-09 12:18:48 +0000177
178out_einval:
179 rc = -EINVAL;
180 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800182EXPORT_SYMBOL(register_qdisc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183
184int unregister_qdisc(struct Qdisc_ops *qops)
185{
186 struct Qdisc_ops *q, **qp;
187 int err = -ENOENT;
188
189 write_lock(&qdisc_mod_lock);
Eric Dumazetcc7ec452011-01-19 19:26:56 +0000190 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 if (q == qops)
192 break;
193 if (q) {
194 *qp = q->next;
195 q->next = NULL;
196 err = 0;
197 }
198 write_unlock(&qdisc_mod_lock);
199 return err;
200}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800201EXPORT_SYMBOL(unregister_qdisc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202
stephen hemminger6da7c8f2013-08-27 16:19:08 -0700203/* Get default qdisc if not otherwise specified */
204void qdisc_get_default(char *name, size_t len)
205{
206 read_lock(&qdisc_mod_lock);
207 strlcpy(name, default_qdisc_ops->id, len);
208 read_unlock(&qdisc_mod_lock);
209}
210
211static struct Qdisc_ops *qdisc_lookup_default(const char *name)
212{
213 struct Qdisc_ops *q = NULL;
214
215 for (q = qdisc_base; q; q = q->next) {
216 if (!strcmp(name, q->id)) {
217 if (!try_module_get(q->owner))
218 q = NULL;
219 break;
220 }
221 }
222
223 return q;
224}
225
226/* Set new default qdisc to use */
227int qdisc_set_default(const char *name)
228{
229 const struct Qdisc_ops *ops;
230
231 if (!capable(CAP_NET_ADMIN))
232 return -EPERM;
233
234 write_lock(&qdisc_mod_lock);
235 ops = qdisc_lookup_default(name);
236 if (!ops) {
237 /* Not found, drop lock and try to load module */
238 write_unlock(&qdisc_mod_lock);
239 request_module("sch_%s", name);
240 write_lock(&qdisc_mod_lock);
241
242 ops = qdisc_lookup_default(name);
243 }
244
245 if (ops) {
246 /* Set new default */
247 module_put(default_qdisc_ops->owner);
248 default_qdisc_ops = ops;
249 }
250 write_unlock(&qdisc_mod_lock);
251
252 return ops ? 0 : -ENOENT;
253}
254
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255/* We know handle. Find qdisc among all qdisc's attached to device
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800256 * (root qdisc, all its children, children of children etc.)
257 * Note: caller either uses rtnl or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 */
259
Hannes Eder6113b742008-11-28 03:06:46 -0800260static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
David S. Miller8123b422008-08-08 23:23:39 -0700261{
262 struct Qdisc *q;
263
264 if (!(root->flags & TCQ_F_BUILTIN) &&
265 root->handle == handle)
266 return root;
267
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800268 list_for_each_entry_rcu(q, &root->list, list) {
David S. Miller8123b422008-08-08 23:23:39 -0700269 if (q->handle == handle)
270 return q;
271 }
272 return NULL;
273}
274
Eric Dumazet95dc1922013-12-05 11:12:02 -0800275void qdisc_list_add(struct Qdisc *q)
Jarek Poplawskif6e0b232008-08-22 03:24:05 -0700276{
Eric Dumazet37314362014-03-08 08:01:19 -0800277 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
278 struct Qdisc *root = qdisc_dev(q)->qdisc;
Eric Dumazete57a7842013-12-12 15:41:56 -0800279
Eric Dumazet37314362014-03-08 08:01:19 -0800280 WARN_ON_ONCE(root == &noop_qdisc);
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800281 ASSERT_RTNL();
282 list_add_tail_rcu(&q->list, &root->list);
Eric Dumazet37314362014-03-08 08:01:19 -0800283 }
Jarek Poplawskif6e0b232008-08-22 03:24:05 -0700284}
Eric Dumazet95dc1922013-12-05 11:12:02 -0800285EXPORT_SYMBOL(qdisc_list_add);
Jarek Poplawskif6e0b232008-08-22 03:24:05 -0700286
287void qdisc_list_del(struct Qdisc *q)
288{
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800289 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
290 ASSERT_RTNL();
291 list_del_rcu(&q->list);
292 }
Jarek Poplawskif6e0b232008-08-22 03:24:05 -0700293}
294EXPORT_SYMBOL(qdisc_list_del);
295
David S. Milleread81cc2008-07-17 00:50:32 -0700296struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
Patrick McHardy43effa12006-11-29 17:35:48 -0800297{
Jarek Poplawskif6e0b232008-08-22 03:24:05 -0700298 struct Qdisc *q;
299
Patrick McHardyaf356af2009-09-04 06:41:18 +0000300 q = qdisc_match_from_root(dev->qdisc, handle);
301 if (q)
302 goto out;
Jarek Poplawskif6e0b232008-08-22 03:24:05 -0700303
Eric Dumazet24824a02010-10-02 06:11:55 +0000304 if (dev_ingress_queue(dev))
305 q = qdisc_match_from_root(
306 dev_ingress_queue(dev)->qdisc_sleeping,
307 handle);
Jarek Poplawskif6486d42008-11-25 13:56:06 -0800308out:
Jarek Poplawskif6e0b232008-08-22 03:24:05 -0700309 return q;
Patrick McHardy43effa12006-11-29 17:35:48 -0800310}
311
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
313{
314 unsigned long cl;
315 struct Qdisc *leaf;
Eric Dumazet20fea082007-11-14 01:44:41 -0800316 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317
318 if (cops == NULL)
319 return NULL;
320 cl = cops->get(p, classid);
321
322 if (cl == 0)
323 return NULL;
324 leaf = cops->leaf(p, cl);
325 cops->put(p, cl);
326 return leaf;
327}
328
329/* Find queueing discipline by name */
330
Patrick McHardy1e904742008-01-22 22:11:17 -0800331static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332{
333 struct Qdisc_ops *q = NULL;
334
335 if (kind) {
336 read_lock(&qdisc_mod_lock);
337 for (q = qdisc_base; q; q = q->next) {
Patrick McHardy1e904742008-01-22 22:11:17 -0800338 if (nla_strcmp(kind, q->id) == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 if (!try_module_get(q->owner))
340 q = NULL;
341 break;
342 }
343 }
344 read_unlock(&qdisc_mod_lock);
345 }
346 return q;
347}
348
Jesper Dangaard Brouer8a8e3d82013-08-14 23:47:11 +0200349/* The linklayer setting were not transferred from iproute2, in older
350 * versions, and the rate tables lookup systems have been dropped in
351 * the kernel. To keep backward compatible with older iproute2 tc
352 * utils, we detect the linklayer setting by detecting if the rate
353 * table were modified.
354 *
355 * For linklayer ATM table entries, the rate table will be aligned to
356 * 48 bytes, thus some table entries will contain the same value. The
357 * mpu (min packet unit) is also encoded into the old rate table, thus
358 * starting from the mpu, we find low and high table entries for
359 * mapping this cell. If these entries contain the same value, when
360 * the rate tables have been modified for linklayer ATM.
361 *
362 * This is done by rounding mpu to the nearest 48 bytes cell/entry,
363 * and then roundup to the next cell, calc the table entry one below,
364 * and compare.
365 */
366static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
367{
368 int low = roundup(r->mpu, 48);
369 int high = roundup(low+1, 48);
370 int cell_low = low >> r->cell_log;
371 int cell_high = (high >> r->cell_log) - 1;
372
373 /* rtab is too inaccurate at rates > 100Mbit/s */
374 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
375 pr_debug("TC linklayer: Giving up ATM detection\n");
376 return TC_LINKLAYER_ETHERNET;
377 }
378
379 if ((cell_high > cell_low) && (cell_high < 256)
380 && (rtab[cell_low] == rtab[cell_high])) {
381 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
382 cell_low, cell_high, rtab[cell_high]);
383 return TC_LINKLAYER_ATM;
384 }
385 return TC_LINKLAYER_ETHERNET;
386}
387
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388static struct qdisc_rate_table *qdisc_rtab_list;
389
Patrick McHardy1e904742008-01-22 22:11:17 -0800390struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391{
392 struct qdisc_rate_table *rtab;
393
Eric Dumazet40edeff2013-06-02 11:15:55 +0000394 if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
395 nla_len(tab) != TC_RTAB_SIZE)
396 return NULL;
397
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
Eric Dumazet40edeff2013-06-02 11:15:55 +0000399 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
400 !memcmp(&rtab->data, nla_data(tab), 1024)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 rtab->refcnt++;
402 return rtab;
403 }
404 }
405
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
407 if (rtab) {
408 rtab->rate = *r;
409 rtab->refcnt = 1;
Patrick McHardy1e904742008-01-22 22:11:17 -0800410 memcpy(rtab->data, nla_data(tab), 1024);
Jesper Dangaard Brouer8a8e3d82013-08-14 23:47:11 +0200411 if (r->linklayer == TC_LINKLAYER_UNAWARE)
412 r->linklayer = __detect_linklayer(r, rtab->data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 rtab->next = qdisc_rtab_list;
414 qdisc_rtab_list = rtab;
415 }
416 return rtab;
417}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800418EXPORT_SYMBOL(qdisc_get_rtab);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
420void qdisc_put_rtab(struct qdisc_rate_table *tab)
421{
422 struct qdisc_rate_table *rtab, **rtabp;
423
424 if (!tab || --tab->refcnt)
425 return;
426
Eric Dumazetcc7ec452011-01-19 19:26:56 +0000427 for (rtabp = &qdisc_rtab_list;
428 (rtab = *rtabp) != NULL;
429 rtabp = &rtab->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 if (rtab == tab) {
431 *rtabp = rtab->next;
432 kfree(rtab);
433 return;
434 }
435 }
436}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800437EXPORT_SYMBOL(qdisc_put_rtab);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700439static LIST_HEAD(qdisc_stab_list);
440static DEFINE_SPINLOCK(qdisc_stab_lock);
441
442static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
443 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
444 [TCA_STAB_DATA] = { .type = NLA_BINARY },
445};
446
447static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt)
448{
449 struct nlattr *tb[TCA_STAB_MAX + 1];
450 struct qdisc_size_table *stab;
451 struct tc_sizespec *s;
452 unsigned int tsize = 0;
453 u16 *tab = NULL;
454 int err;
455
456 err = nla_parse_nested(tb, TCA_STAB_MAX, opt, stab_policy);
457 if (err < 0)
458 return ERR_PTR(err);
459 if (!tb[TCA_STAB_BASE])
460 return ERR_PTR(-EINVAL);
461
462 s = nla_data(tb[TCA_STAB_BASE]);
463
464 if (s->tsize > 0) {
465 if (!tb[TCA_STAB_DATA])
466 return ERR_PTR(-EINVAL);
467 tab = nla_data(tb[TCA_STAB_DATA]);
468 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
469 }
470
Dan Carpenter00093fa2010-08-14 11:09:49 +0000471 if (tsize != s->tsize || (!tab && tsize > 0))
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700472 return ERR_PTR(-EINVAL);
473
David S. Millerf3b96052008-08-18 22:33:05 -0700474 spin_lock(&qdisc_stab_lock);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700475
476 list_for_each_entry(stab, &qdisc_stab_list, list) {
477 if (memcmp(&stab->szopts, s, sizeof(*s)))
478 continue;
479 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
480 continue;
481 stab->refcnt++;
David S. Millerf3b96052008-08-18 22:33:05 -0700482 spin_unlock(&qdisc_stab_lock);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700483 return stab;
484 }
485
David S. Millerf3b96052008-08-18 22:33:05 -0700486 spin_unlock(&qdisc_stab_lock);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700487
488 stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
489 if (!stab)
490 return ERR_PTR(-ENOMEM);
491
492 stab->refcnt = 1;
493 stab->szopts = *s;
494 if (tsize > 0)
495 memcpy(stab->data, tab, tsize * sizeof(u16));
496
David S. Millerf3b96052008-08-18 22:33:05 -0700497 spin_lock(&qdisc_stab_lock);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700498 list_add_tail(&stab->list, &qdisc_stab_list);
David S. Millerf3b96052008-08-18 22:33:05 -0700499 spin_unlock(&qdisc_stab_lock);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700500
501 return stab;
502}
503
Eric Dumazeta2da5702011-01-20 03:48:19 +0000504static void stab_kfree_rcu(struct rcu_head *head)
505{
506 kfree(container_of(head, struct qdisc_size_table, rcu));
507}
508
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700509void qdisc_put_stab(struct qdisc_size_table *tab)
510{
511 if (!tab)
512 return;
513
David S. Millerf3b96052008-08-18 22:33:05 -0700514 spin_lock(&qdisc_stab_lock);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700515
516 if (--tab->refcnt == 0) {
517 list_del(&tab->list);
Eric Dumazeta2da5702011-01-20 03:48:19 +0000518 call_rcu_bh(&tab->rcu, stab_kfree_rcu);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700519 }
520
David S. Millerf3b96052008-08-18 22:33:05 -0700521 spin_unlock(&qdisc_stab_lock);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700522}
523EXPORT_SYMBOL(qdisc_put_stab);
524
525static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
526{
527 struct nlattr *nest;
528
529 nest = nla_nest_start(skb, TCA_STAB);
Patrick McHardy3aa46142008-11-20 04:07:14 -0800530 if (nest == NULL)
531 goto nla_put_failure;
David S. Miller1b34ec42012-03-29 05:11:39 -0400532 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
533 goto nla_put_failure;
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700534 nla_nest_end(skb, nest);
535
536 return skb->len;
537
538nla_put_failure:
539 return -1;
540}
541
Eric Dumazeta2da5702011-01-20 03:48:19 +0000542void __qdisc_calculate_pkt_len(struct sk_buff *skb, const struct qdisc_size_table *stab)
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700543{
544 int pkt_len, slot;
545
546 pkt_len = skb->len + stab->szopts.overhead;
547 if (unlikely(!stab->szopts.tsize))
548 goto out;
549
550 slot = pkt_len + stab->szopts.cell_align;
551 if (unlikely(slot < 0))
552 slot = 0;
553
554 slot >>= stab->szopts.cell_log;
555 if (likely(slot < stab->szopts.tsize))
556 pkt_len = stab->data[slot];
557 else
558 pkt_len = stab->data[stab->szopts.tsize - 1] *
559 (slot / stab->szopts.tsize) +
560 stab->data[slot % stab->szopts.tsize];
561
562 pkt_len <<= stab->szopts.size_log;
563out:
564 if (unlikely(pkt_len < 1))
565 pkt_len = 1;
566 qdisc_skb_cb(skb)->pkt_len = pkt_len;
567}
Eric Dumazeta2da5702011-01-20 03:48:19 +0000568EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700569
Florian Westphal6e765a02014-06-11 20:35:18 +0200570void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
Jarek Poplawskib00355d2009-02-01 01:12:42 -0800571{
572 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
Eric Dumazetcc7ec452011-01-19 19:26:56 +0000573 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
574 txt, qdisc->ops->id, qdisc->handle >> 16);
Jarek Poplawskib00355d2009-02-01 01:12:42 -0800575 qdisc->flags |= TCQ_F_WARN_NONWC;
576 }
577}
578EXPORT_SYMBOL(qdisc_warn_nonwc);
579
Patrick McHardy41794772007-03-16 01:19:15 -0700580static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
581{
582 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
David S. Miller2fbd3da2009-09-01 17:59:25 -0700583 timer);
Patrick McHardy41794772007-03-16 01:19:15 -0700584
John Fastabend1e203c12014-10-02 22:43:09 -0700585 rcu_read_lock();
Eric Dumazetfd245a42011-01-20 05:27:16 +0000586 qdisc_unthrottled(wd->qdisc);
David S. Miller8608db02008-08-18 20:51:18 -0700587 __netif_schedule(qdisc_root(wd->qdisc));
John Fastabend1e203c12014-10-02 22:43:09 -0700588 rcu_read_unlock();
Stephen Hemminger19365022007-03-22 12:18:35 -0700589
Patrick McHardy41794772007-03-16 01:19:15 -0700590 return HRTIMER_NORESTART;
591}
592
593void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
594{
Eric Dumazet4a8e3202014-09-20 18:01:30 -0700595 hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
David S. Miller2fbd3da2009-09-01 17:59:25 -0700596 wd->timer.function = qdisc_watchdog;
Patrick McHardy41794772007-03-16 01:19:15 -0700597 wd->qdisc = qdisc;
598}
599EXPORT_SYMBOL(qdisc_watchdog_init);
600
Eric Dumazetf2600cf2014-10-04 10:11:31 -0700601void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires, bool throttle)
Patrick McHardy41794772007-03-16 01:19:15 -0700602{
Jarek Poplawski2540e052008-08-21 05:11:14 -0700603 if (test_bit(__QDISC_STATE_DEACTIVATED,
604 &qdisc_root_sleeping(wd->qdisc)->state))
605 return;
606
Eric Dumazetf2600cf2014-10-04 10:11:31 -0700607 if (throttle)
608 qdisc_throttled(wd->qdisc);
Eric Dumazet46baac32012-10-20 00:40:51 +0000609
610 hrtimer_start(&wd->timer,
Jiri Pirko34c5d292013-02-12 00:12:04 +0000611 ns_to_ktime(expires),
Eric Dumazet4a8e3202014-09-20 18:01:30 -0700612 HRTIMER_MODE_ABS_PINNED);
Patrick McHardy41794772007-03-16 01:19:15 -0700613}
Jiri Pirko34c5d292013-02-12 00:12:04 +0000614EXPORT_SYMBOL(qdisc_watchdog_schedule_ns);
Patrick McHardy41794772007-03-16 01:19:15 -0700615
616void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
617{
David S. Miller2fbd3da2009-09-01 17:59:25 -0700618 hrtimer_cancel(&wd->timer);
Eric Dumazetfd245a42011-01-20 05:27:16 +0000619 qdisc_unthrottled(wd->qdisc);
Patrick McHardy41794772007-03-16 01:19:15 -0700620}
621EXPORT_SYMBOL(qdisc_watchdog_cancel);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622
Adrian Bunka94f7792008-07-22 14:20:11 -0700623static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700624{
625 unsigned int size = n * sizeof(struct hlist_head), i;
626 struct hlist_head *h;
627
628 if (size <= PAGE_SIZE)
629 h = kmalloc(size, GFP_KERNEL);
630 else
631 h = (struct hlist_head *)
632 __get_free_pages(GFP_KERNEL, get_order(size));
633
634 if (h != NULL) {
635 for (i = 0; i < n; i++)
636 INIT_HLIST_HEAD(&h[i]);
637 }
638 return h;
639}
640
641static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n)
642{
643 unsigned int size = n * sizeof(struct hlist_head);
644
645 if (size <= PAGE_SIZE)
646 kfree(h);
647 else
648 free_pages((unsigned long)h, get_order(size));
649}
650
651void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
652{
653 struct Qdisc_class_common *cl;
Sasha Levinb67bfe02013-02-27 17:06:00 -0800654 struct hlist_node *next;
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700655 struct hlist_head *nhash, *ohash;
656 unsigned int nsize, nmask, osize;
657 unsigned int i, h;
658
659 /* Rehash when load factor exceeds 0.75 */
660 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
661 return;
662 nsize = clhash->hashsize * 2;
663 nmask = nsize - 1;
664 nhash = qdisc_class_hash_alloc(nsize);
665 if (nhash == NULL)
666 return;
667
668 ohash = clhash->hash;
669 osize = clhash->hashsize;
670
671 sch_tree_lock(sch);
672 for (i = 0; i < osize; i++) {
Sasha Levinb67bfe02013-02-27 17:06:00 -0800673 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700674 h = qdisc_class_hash(cl->classid, nmask);
675 hlist_add_head(&cl->hnode, &nhash[h]);
676 }
677 }
678 clhash->hash = nhash;
679 clhash->hashsize = nsize;
680 clhash->hashmask = nmask;
681 sch_tree_unlock(sch);
682
683 qdisc_class_hash_free(ohash, osize);
684}
685EXPORT_SYMBOL(qdisc_class_hash_grow);
686
687int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
688{
689 unsigned int size = 4;
690
691 clhash->hash = qdisc_class_hash_alloc(size);
692 if (clhash->hash == NULL)
693 return -ENOMEM;
694 clhash->hashsize = size;
695 clhash->hashmask = size - 1;
696 clhash->hashelems = 0;
697 return 0;
698}
699EXPORT_SYMBOL(qdisc_class_hash_init);
700
701void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
702{
703 qdisc_class_hash_free(clhash->hash, clhash->hashsize);
704}
705EXPORT_SYMBOL(qdisc_class_hash_destroy);
706
707void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
708 struct Qdisc_class_common *cl)
709{
710 unsigned int h;
711
712 INIT_HLIST_NODE(&cl->hnode);
713 h = qdisc_class_hash(cl->classid, clhash->hashmask);
714 hlist_add_head(&cl->hnode, &clhash->hash[h]);
715 clhash->hashelems++;
716}
717EXPORT_SYMBOL(qdisc_class_hash_insert);
718
719void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
720 struct Qdisc_class_common *cl)
721{
722 hlist_del(&cl->hnode);
723 clhash->hashelems--;
724}
725EXPORT_SYMBOL(qdisc_class_hash_remove);
726
Eric Dumazetfa0f5aa2012-01-03 00:00:11 +0000727/* Allocate an unique handle from space managed by kernel
728 * Possible range is [8000-FFFF]:0000 (0x8000 values)
729 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730static u32 qdisc_alloc_handle(struct net_device *dev)
731{
Eric Dumazetfa0f5aa2012-01-03 00:00:11 +0000732 int i = 0x8000;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
734
735 do {
736 autohandle += TC_H_MAKE(0x10000U, 0);
737 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
738 autohandle = TC_H_MAKE(0x80000000U, 0);
Eric Dumazetfa0f5aa2012-01-03 00:00:11 +0000739 if (!qdisc_lookup(dev, autohandle))
740 return autohandle;
741 cond_resched();
742 } while (--i > 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743
Eric Dumazetfa0f5aa2012-01-03 00:00:11 +0000744 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745}
746
Patrick McHardy43effa12006-11-29 17:35:48 -0800747void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n)
748{
Eric Dumazet20fea082007-11-14 01:44:41 -0800749 const struct Qdisc_class_ops *cops;
Patrick McHardy43effa12006-11-29 17:35:48 -0800750 unsigned long cl;
751 u32 parentid;
Eric Dumazet2c8c8e62013-10-07 08:32:32 -0700752 int drops;
Patrick McHardy43effa12006-11-29 17:35:48 -0800753
754 if (n == 0)
755 return;
Eric Dumazet2c8c8e62013-10-07 08:32:32 -0700756 drops = max_t(int, n, 0);
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800757 rcu_read_lock();
Patrick McHardy43effa12006-11-29 17:35:48 -0800758 while ((parentid = sch->parent)) {
Jarek Poplawski066a3b52008-04-14 15:10:42 -0700759 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800760 break;
Jarek Poplawski066a3b52008-04-14 15:10:42 -0700761
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800762 if (sch->flags & TCQ_F_NOPARENT)
763 break;
764 /* TODO: perform the search on a per txq basis */
David S. Miller5ce2d482008-07-08 17:06:30 -0700765 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
Patrick McHardyffc8fef2007-07-30 17:11:50 -0700766 if (sch == NULL) {
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800767 WARN_ON_ONCE(parentid != TC_H_ROOT);
768 break;
Patrick McHardyffc8fef2007-07-30 17:11:50 -0700769 }
Patrick McHardy43effa12006-11-29 17:35:48 -0800770 cops = sch->ops->cl_ops;
771 if (cops->qlen_notify) {
772 cl = cops->get(sch, parentid);
773 cops->qlen_notify(sch, cl);
774 cops->put(sch, cl);
775 }
776 sch->q.qlen -= n;
John Fastabend25331d62014-09-28 11:53:29 -0700777 __qdisc_qstats_drop(sch, drops);
Patrick McHardy43effa12006-11-29 17:35:48 -0800778 }
Eric Dumazet4eaf3b82015-12-01 20:08:51 -0800779 rcu_read_unlock();
Patrick McHardy43effa12006-11-29 17:35:48 -0800780}
781EXPORT_SYMBOL(qdisc_tree_decrease_qlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782
Tom Goff7316ae82010-03-19 15:40:13 +0000783static void notify_and_destroy(struct net *net, struct sk_buff *skb,
784 struct nlmsghdr *n, u32 clid,
David S. Miller99194cf2008-07-17 04:54:10 -0700785 struct Qdisc *old, struct Qdisc *new)
786{
787 if (new || old)
Tom Goff7316ae82010-03-19 15:40:13 +0000788 qdisc_notify(net, skb, n, clid, old, new);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789
David S. Miller4d8863a2008-08-18 21:03:15 -0700790 if (old)
David S. Miller99194cf2008-07-17 04:54:10 -0700791 qdisc_destroy(old);
David S. Miller99194cf2008-07-17 04:54:10 -0700792}
793
794/* Graft qdisc "new" to class "classid" of qdisc "parent" or
795 * to device "dev".
796 *
797 * When appropriate send a netlink notification using 'skb'
798 * and "n".
799 *
800 * On success, destroy old qdisc.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801 */
802
803static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
David S. Miller99194cf2008-07-17 04:54:10 -0700804 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
805 struct Qdisc *new, struct Qdisc *old)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806{
David S. Miller99194cf2008-07-17 04:54:10 -0700807 struct Qdisc *q = old;
Tom Goff7316ae82010-03-19 15:40:13 +0000808 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +0900811 if (parent == NULL) {
David S. Miller99194cf2008-07-17 04:54:10 -0700812 unsigned int i, num_q, ingress;
813
814 ingress = 0;
815 num_q = dev->num_tx_queues;
David S. Miller8d50b532008-07-30 02:37:46 -0700816 if ((q && q->flags & TCQ_F_INGRESS) ||
817 (new && new->flags & TCQ_F_INGRESS)) {
David S. Miller99194cf2008-07-17 04:54:10 -0700818 num_q = 1;
819 ingress = 1;
Eric Dumazet24824a02010-10-02 06:11:55 +0000820 if (!dev_ingress_queue(dev))
821 return -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 }
David S. Miller99194cf2008-07-17 04:54:10 -0700823
824 if (dev->flags & IFF_UP)
825 dev_deactivate(dev);
826
WANG Cong86e363d2015-05-26 16:08:48 -0700827 if (new && new->ops->attach)
828 goto skip;
David S. Miller6ec1c692009-09-06 01:58:51 -0700829
David S. Miller99194cf2008-07-17 04:54:10 -0700830 for (i = 0; i < num_q; i++) {
Eric Dumazet24824a02010-10-02 06:11:55 +0000831 struct netdev_queue *dev_queue = dev_ingress_queue(dev);
David S. Miller99194cf2008-07-17 04:54:10 -0700832
833 if (!ingress)
834 dev_queue = netdev_get_tx_queue(dev, i);
835
David S. Miller8d50b532008-07-30 02:37:46 -0700836 old = dev_graft_qdisc(dev_queue, new);
837 if (new && i > 0)
838 atomic_inc(&new->refcnt);
839
Jarek Poplawski036d6a62009-09-13 22:35:44 +0000840 if (!ingress)
841 qdisc_destroy(old);
David S. Miller99194cf2008-07-17 04:54:10 -0700842 }
843
WANG Cong86e363d2015-05-26 16:08:48 -0700844skip:
Jarek Poplawski036d6a62009-09-13 22:35:44 +0000845 if (!ingress) {
Tom Goff7316ae82010-03-19 15:40:13 +0000846 notify_and_destroy(net, skb, n, classid,
847 dev->qdisc, new);
Jarek Poplawski036d6a62009-09-13 22:35:44 +0000848 if (new && !new->ops->attach)
849 atomic_inc(&new->refcnt);
850 dev->qdisc = new ? : &noop_qdisc;
WANG Cong86e363d2015-05-26 16:08:48 -0700851
852 if (new && new->ops->attach)
853 new->ops->attach(new);
Jarek Poplawski036d6a62009-09-13 22:35:44 +0000854 } else {
Tom Goff7316ae82010-03-19 15:40:13 +0000855 notify_and_destroy(net, skb, n, classid, old, new);
Jarek Poplawski036d6a62009-09-13 22:35:44 +0000856 }
Patrick McHardyaf356af2009-09-04 06:41:18 +0000857
David S. Miller99194cf2008-07-17 04:54:10 -0700858 if (dev->flags & IFF_UP)
859 dev_activate(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 } else {
Eric Dumazet20fea082007-11-14 01:44:41 -0800861 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862
Patrick McHardyc9f1d032009-09-04 06:41:13 +0000863 err = -EOPNOTSUPP;
864 if (cops && cops->graft) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865 unsigned long cl = cops->get(parent, classid);
866 if (cl) {
David S. Miller99194cf2008-07-17 04:54:10 -0700867 err = cops->graft(parent, cl, new, &old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 cops->put(parent, cl);
Patrick McHardyc9f1d032009-09-04 06:41:13 +0000869 } else
870 err = -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 }
David S. Miller99194cf2008-07-17 04:54:10 -0700872 if (!err)
Tom Goff7316ae82010-03-19 15:40:13 +0000873 notify_and_destroy(net, skb, n, classid, old, new);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 }
875 return err;
876}
877
Jarek Poplawski25bfcd52008-08-18 20:53:34 -0700878/* lockdep annotation is needed for ingress; egress gets it only for name */
879static struct lock_class_key qdisc_tx_lock;
880static struct lock_class_key qdisc_rx_lock;
881
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882/*
883 Allocate and initialize new qdisc.
884
885 Parameters are passed via opt.
886 */
887
888static struct Qdisc *
David S. Millerbb949fb2008-07-08 16:55:56 -0700889qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
Patrick McHardy23bcf632009-09-09 18:11:23 -0700890 struct Qdisc *p, u32 parent, u32 handle,
891 struct nlattr **tca, int *errp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892{
893 int err;
Patrick McHardy1e904742008-01-22 22:11:17 -0800894 struct nlattr *kind = tca[TCA_KIND];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 struct Qdisc *sch;
896 struct Qdisc_ops *ops;
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700897 struct qdisc_size_table *stab;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898
899 ops = qdisc_lookup_ops(kind);
Johannes Berg95a5afc2008-10-16 15:24:51 -0700900#ifdef CONFIG_MODULES
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 if (ops == NULL && kind != NULL) {
902 char name[IFNAMSIZ];
Patrick McHardy1e904742008-01-22 22:11:17 -0800903 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 /* We dropped the RTNL semaphore in order to
905 * perform the module load. So, even if we
906 * succeeded in loading the module we have to
907 * tell the caller to replay the request. We
908 * indicate this using -EAGAIN.
909 * We replay the request because the device may
910 * go away in the mean time.
911 */
912 rtnl_unlock();
913 request_module("sch_%s", name);
914 rtnl_lock();
915 ops = qdisc_lookup_ops(kind);
916 if (ops != NULL) {
917 /* We will try again qdisc_lookup_ops,
918 * so don't keep a reference.
919 */
920 module_put(ops->owner);
921 err = -EAGAIN;
922 goto err_out;
923 }
924 }
925 }
926#endif
927
Jamal Hadi Salimb9e2cc02006-08-03 16:36:51 -0700928 err = -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 if (ops == NULL)
930 goto err_out;
931
David S. Miller5ce2d482008-07-08 17:06:30 -0700932 sch = qdisc_alloc(dev_queue, ops);
Thomas Graf3d54b822005-07-05 14:15:09 -0700933 if (IS_ERR(sch)) {
934 err = PTR_ERR(sch);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935 goto err_out2;
Thomas Graf3d54b822005-07-05 14:15:09 -0700936 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937
Patrick McHardyffc8fef2007-07-30 17:11:50 -0700938 sch->parent = parent;
939
Thomas Graf3d54b822005-07-05 14:15:09 -0700940 if (handle == TC_H_INGRESS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 sch->flags |= TCQ_F_INGRESS;
Thomas Graf3d54b822005-07-05 14:15:09 -0700942 handle = TC_H_MAKE(TC_H_INGRESS, 0);
Jarek Poplawski25bfcd52008-08-18 20:53:34 -0700943 lockdep_set_class(qdisc_lock(sch), &qdisc_rx_lock);
Patrick McHardyfd44de72007-04-16 17:07:08 -0700944 } else {
Patrick McHardyfd44de72007-04-16 17:07:08 -0700945 if (handle == 0) {
946 handle = qdisc_alloc_handle(dev);
947 err = -ENOMEM;
948 if (handle == 0)
949 goto err_out3;
950 }
Jarek Poplawski25bfcd52008-08-18 20:53:34 -0700951 lockdep_set_class(qdisc_lock(sch), &qdisc_tx_lock);
Eric Dumazet1abbe132012-12-11 15:54:33 +0000952 if (!netif_is_multiqueue(dev))
Eric Dumazet225734d2015-12-15 09:43:12 -0800953 sch->flags |= TCQ_F_ONETXQUEUE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 }
955
Thomas Graf3d54b822005-07-05 14:15:09 -0700956 sch->handle = handle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957
Patrick McHardy1e904742008-01-22 22:11:17 -0800958 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
John Fastabend22e0f8b2014-09-28 11:52:56 -0700959 if (qdisc_is_percpu_stats(sch)) {
960 sch->cpu_bstats =
Sabrina Dubroca7c1c97d2014-10-21 11:23:30 +0200961 netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
John Fastabend22e0f8b2014-09-28 11:52:56 -0700962 if (!sch->cpu_bstats)
963 goto err_out4;
John Fastabendb0ab6f92014-09-28 11:54:24 -0700964
965 sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
966 if (!sch->cpu_qstats)
967 goto err_out4;
John Fastabend22e0f8b2014-09-28 11:52:56 -0700968 }
969
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700970 if (tca[TCA_STAB]) {
971 stab = qdisc_get_stab(tca[TCA_STAB]);
972 if (IS_ERR(stab)) {
973 err = PTR_ERR(stab);
Jarek Poplawski7c64b9f2009-09-15 23:42:05 -0700974 goto err_out4;
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700975 }
Eric Dumazeta2da5702011-01-20 03:48:19 +0000976 rcu_assign_pointer(sch->stab, stab);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -0700977 }
Patrick McHardy1e904742008-01-22 22:11:17 -0800978 if (tca[TCA_RATE]) {
Jarek Poplawskif6f9b932008-08-27 02:25:17 -0700979 spinlock_t *root_lock;
980
Patrick McHardy23bcf632009-09-09 18:11:23 -0700981 err = -EOPNOTSUPP;
982 if (sch->flags & TCQ_F_MQROOT)
983 goto err_out4;
984
Jarek Poplawskif6f9b932008-08-27 02:25:17 -0700985 if ((sch->parent != TC_H_ROOT) &&
Patrick McHardy23bcf632009-09-09 18:11:23 -0700986 !(sch->flags & TCQ_F_INGRESS) &&
987 (!p || !(p->flags & TCQ_F_MQROOT)))
Jarek Poplawskif6f9b932008-08-27 02:25:17 -0700988 root_lock = qdisc_root_sleeping_lock(sch);
989 else
990 root_lock = qdisc_lock(sch);
991
John Fastabend22e0f8b2014-09-28 11:52:56 -0700992 err = gen_new_estimator(&sch->bstats,
993 sch->cpu_bstats,
994 &sch->rate_est,
995 root_lock,
996 tca[TCA_RATE]);
Patrick McHardy23bcf632009-09-09 18:11:23 -0700997 if (err)
998 goto err_out4;
Thomas Graf023e09a2005-07-05 14:15:53 -0700999 }
Jarek Poplawskif6e0b232008-08-22 03:24:05 -07001000
1001 qdisc_list_add(sch);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003 return sch;
1004 }
1005err_out3:
1006 dev_put(dev);
Thomas Graf3d54b822005-07-05 14:15:09 -07001007 kfree((char *) sch - sch->padded);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008err_out2:
1009 module_put(ops->owner);
1010err_out:
1011 *errp = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 return NULL;
Patrick McHardy23bcf632009-09-09 18:11:23 -07001013
1014err_out4:
John Fastabend22e0f8b2014-09-28 11:52:56 -07001015 free_percpu(sch->cpu_bstats);
John Fastabendb0ab6f92014-09-28 11:54:24 -07001016 free_percpu(sch->cpu_qstats);
Patrick McHardy23bcf632009-09-09 18:11:23 -07001017 /*
1018 * Any broken qdiscs that would require a ops->reset() here?
1019 * The qdisc was never in action so it shouldn't be necessary.
1020 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00001021 qdisc_put_stab(rtnl_dereference(sch->stab));
Patrick McHardy23bcf632009-09-09 18:11:23 -07001022 if (ops->destroy)
1023 ops->destroy(sch);
1024 goto err_out3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025}
1026
Patrick McHardy1e904742008-01-22 22:11:17 -08001027static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028{
Eric Dumazeta2da5702011-01-20 03:48:19 +00001029 struct qdisc_size_table *ostab, *stab = NULL;
Jussi Kivilinna175f9c12008-07-20 00:08:47 -07001030 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Jussi Kivilinna175f9c12008-07-20 00:08:47 -07001032 if (tca[TCA_OPTIONS]) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 if (sch->ops->change == NULL)
1034 return -EINVAL;
Patrick McHardy1e904742008-01-22 22:11:17 -08001035 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 if (err)
1037 return err;
1038 }
Jussi Kivilinna175f9c12008-07-20 00:08:47 -07001039
1040 if (tca[TCA_STAB]) {
1041 stab = qdisc_get_stab(tca[TCA_STAB]);
1042 if (IS_ERR(stab))
1043 return PTR_ERR(stab);
1044 }
1045
Eric Dumazeta2da5702011-01-20 03:48:19 +00001046 ostab = rtnl_dereference(sch->stab);
1047 rcu_assign_pointer(sch->stab, stab);
1048 qdisc_put_stab(ostab);
Jussi Kivilinna175f9c12008-07-20 00:08:47 -07001049
Patrick McHardy23bcf632009-09-09 18:11:23 -07001050 if (tca[TCA_RATE]) {
Stephen Hemminger71bcb092008-11-25 21:13:31 -08001051 /* NB: ignores errors from replace_estimator
1052 because change can't be undone. */
Patrick McHardy23bcf632009-09-09 18:11:23 -07001053 if (sch->flags & TCQ_F_MQROOT)
1054 goto out;
John Fastabend22e0f8b2014-09-28 11:52:56 -07001055 gen_replace_estimator(&sch->bstats,
1056 sch->cpu_bstats,
1057 &sch->rate_est,
1058 qdisc_root_sleeping_lock(sch),
1059 tca[TCA_RATE]);
Patrick McHardy23bcf632009-09-09 18:11:23 -07001060 }
1061out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 return 0;
1063}
1064
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001065struct check_loop_arg {
1066 struct qdisc_walker w;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 struct Qdisc *p;
1068 int depth;
1069};
1070
1071static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w);
1072
1073static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1074{
1075 struct check_loop_arg arg;
1076
1077 if (q->ops->cl_ops == NULL)
1078 return 0;
1079
1080 arg.w.stop = arg.w.skip = arg.w.count = 0;
1081 arg.w.fn = check_loop_fn;
1082 arg.depth = depth;
1083 arg.p = p;
1084 q->ops->cl_ops->walk(q, &arg.w);
1085 return arg.w.stop ? -ELOOP : 0;
1086}
1087
1088static int
1089check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1090{
1091 struct Qdisc *leaf;
Eric Dumazet20fea082007-11-14 01:44:41 -08001092 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 struct check_loop_arg *arg = (struct check_loop_arg *)w;
1094
1095 leaf = cops->leaf(q, cl);
1096 if (leaf) {
1097 if (leaf == arg->p || arg->depth > 7)
1098 return -ELOOP;
1099 return check_loop(leaf, arg->p, arg->depth + 1);
1100 }
1101 return 0;
1102}
1103
1104/*
1105 * Delete/get qdisc.
1106 */
1107
Thomas Graf661d2962013-03-21 07:45:29 +00001108static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001110 struct net *net = sock_net(skb->sk);
David S. Miller02ef22c2012-06-26 21:50:05 -07001111 struct tcmsg *tcm = nlmsg_data(n);
Patrick McHardy1e904742008-01-22 22:11:17 -08001112 struct nlattr *tca[TCA_MAX + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 struct net_device *dev;
Hong zhi guode179c82013-03-25 17:36:33 +00001114 u32 clid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 struct Qdisc *q = NULL;
1116 struct Qdisc *p = NULL;
1117 int err;
1118
Stéphane Graber4e8bbb82014-04-30 11:25:43 -04001119 if ((n->nlmsg_type != RTM_GETQDISC) &&
David S. Miller5f013c9b2014-05-12 13:19:14 -04001120 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
Eric W. Biedermandfc47ef2012-11-16 03:03:00 +00001121 return -EPERM;
1122
Patrick McHardy1e904742008-01-22 22:11:17 -08001123 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1124 if (err < 0)
1125 return err;
1126
Hong zhi guode179c82013-03-25 17:36:33 +00001127 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1128 if (!dev)
1129 return -ENODEV;
1130
1131 clid = tcm->tcm_parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132 if (clid) {
1133 if (clid != TC_H_ROOT) {
1134 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001135 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1136 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 return -ENOENT;
1138 q = qdisc_leaf(p, clid);
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001139 } else if (dev_ingress_queue(dev)) {
1140 q = dev_ingress_queue(dev)->qdisc_sleeping;
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001141 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142 } else {
Patrick McHardyaf356af2009-09-04 06:41:18 +00001143 q = dev->qdisc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144 }
1145 if (!q)
1146 return -ENOENT;
1147
1148 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
1149 return -EINVAL;
1150 } else {
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001151 q = qdisc_lookup(dev, tcm->tcm_handle);
1152 if (!q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153 return -ENOENT;
1154 }
1155
Patrick McHardy1e904742008-01-22 22:11:17 -08001156 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157 return -EINVAL;
1158
1159 if (n->nlmsg_type == RTM_DELQDISC) {
1160 if (!clid)
1161 return -EINVAL;
1162 if (q->handle == 0)
1163 return -ENOENT;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001164 err = qdisc_graft(dev, p, skb, n, clid, NULL, q);
1165 if (err != 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001167 } else {
Tom Goff7316ae82010-03-19 15:40:13 +00001168 qdisc_notify(net, skb, n, clid, NULL, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001169 }
1170 return 0;
1171}
1172
1173/*
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001174 * Create/change qdisc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 */
1176
Thomas Graf661d2962013-03-21 07:45:29 +00001177static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001179 struct net *net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001180 struct tcmsg *tcm;
Patrick McHardy1e904742008-01-22 22:11:17 -08001181 struct nlattr *tca[TCA_MAX + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 struct net_device *dev;
1183 u32 clid;
1184 struct Qdisc *q, *p;
1185 int err;
1186
David S. Miller5f013c9b2014-05-12 13:19:14 -04001187 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
Eric W. Biedermandfc47ef2012-11-16 03:03:00 +00001188 return -EPERM;
1189
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190replay:
1191 /* Reinit, just in case something touches this. */
Hong zhi guode179c82013-03-25 17:36:33 +00001192 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1193 if (err < 0)
1194 return err;
1195
David S. Miller02ef22c2012-06-26 21:50:05 -07001196 tcm = nlmsg_data(n);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 clid = tcm->tcm_parent;
1198 q = p = NULL;
1199
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001200 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1201 if (!dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202 return -ENODEV;
1203
Patrick McHardy1e904742008-01-22 22:11:17 -08001204
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205 if (clid) {
1206 if (clid != TC_H_ROOT) {
1207 if (clid != TC_H_INGRESS) {
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001208 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1209 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210 return -ENOENT;
1211 q = qdisc_leaf(p, clid);
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001212 } else if (dev_ingress_queue_create(dev)) {
1213 q = dev_ingress_queue(dev)->qdisc_sleeping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 }
1215 } else {
Patrick McHardyaf356af2009-09-04 06:41:18 +00001216 q = dev->qdisc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217 }
1218
1219 /* It may be default qdisc, ignore it */
1220 if (q && q->handle == 0)
1221 q = NULL;
1222
1223 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1224 if (tcm->tcm_handle) {
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001225 if (q && !(n->nlmsg_flags & NLM_F_REPLACE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 return -EEXIST;
1227 if (TC_H_MIN(tcm->tcm_handle))
1228 return -EINVAL;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001229 q = qdisc_lookup(dev, tcm->tcm_handle);
1230 if (!q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 goto create_n_graft;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001232 if (n->nlmsg_flags & NLM_F_EXCL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 return -EEXIST;
Patrick McHardy1e904742008-01-22 22:11:17 -08001234 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235 return -EINVAL;
1236 if (q == p ||
1237 (p && check_loop(q, p, 0)))
1238 return -ELOOP;
1239 atomic_inc(&q->refcnt);
1240 goto graft;
1241 } else {
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001242 if (!q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 goto create_n_graft;
1244
1245 /* This magic test requires explanation.
1246 *
1247 * We know, that some child q is already
1248 * attached to this parent and have choice:
1249 * either to change it or to create/graft new one.
1250 *
1251 * 1. We are allowed to create/graft only
1252 * if CREATE and REPLACE flags are set.
1253 *
1254 * 2. If EXCL is set, requestor wanted to say,
1255 * that qdisc tcm_handle is not expected
1256 * to exist, so that we choose create/graft too.
1257 *
1258 * 3. The last case is when no flags are set.
1259 * Alas, it is sort of hole in API, we
1260 * cannot decide what to do unambiguously.
1261 * For now we select create/graft, if
1262 * user gave KIND, which does not match existing.
1263 */
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001264 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1265 (n->nlmsg_flags & NLM_F_REPLACE) &&
1266 ((n->nlmsg_flags & NLM_F_EXCL) ||
Patrick McHardy1e904742008-01-22 22:11:17 -08001267 (tca[TCA_KIND] &&
1268 nla_strcmp(tca[TCA_KIND], q->ops->id))))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 goto create_n_graft;
1270 }
1271 }
1272 } else {
1273 if (!tcm->tcm_handle)
1274 return -EINVAL;
1275 q = qdisc_lookup(dev, tcm->tcm_handle);
1276 }
1277
1278 /* Change qdisc parameters */
1279 if (q == NULL)
1280 return -ENOENT;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001281 if (n->nlmsg_flags & NLM_F_EXCL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 return -EEXIST;
Patrick McHardy1e904742008-01-22 22:11:17 -08001283 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284 return -EINVAL;
1285 err = qdisc_change(q, tca);
1286 if (err == 0)
Tom Goff7316ae82010-03-19 15:40:13 +00001287 qdisc_notify(net, skb, n, clid, NULL, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 return err;
1289
1290create_n_graft:
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001291 if (!(n->nlmsg_flags & NLM_F_CREATE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 return -ENOENT;
Eric Dumazet24824a02010-10-02 06:11:55 +00001293 if (clid == TC_H_INGRESS) {
1294 if (dev_ingress_queue(dev))
1295 q = qdisc_create(dev, dev_ingress_queue(dev), p,
1296 tcm->tcm_parent, tcm->tcm_parent,
1297 tca, &err);
1298 else
1299 err = -ENOENT;
1300 } else {
Jarek Poplawski926e61b2009-09-15 02:53:07 -07001301 struct netdev_queue *dev_queue;
David S. Miller6ec1c692009-09-06 01:58:51 -07001302
1303 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
Jarek Poplawski926e61b2009-09-15 02:53:07 -07001304 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1305 else if (p)
1306 dev_queue = p->dev_queue;
1307 else
1308 dev_queue = netdev_get_tx_queue(dev, 0);
David S. Miller6ec1c692009-09-06 01:58:51 -07001309
Jarek Poplawski926e61b2009-09-15 02:53:07 -07001310 q = qdisc_create(dev, dev_queue, p,
David S. Millerbb949fb2008-07-08 16:55:56 -07001311 tcm->tcm_parent, tcm->tcm_handle,
Patrick McHardyffc8fef2007-07-30 17:11:50 -07001312 tca, &err);
David S. Miller6ec1c692009-09-06 01:58:51 -07001313 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 if (q == NULL) {
1315 if (err == -EAGAIN)
1316 goto replay;
1317 return err;
1318 }
1319
1320graft:
Ilpo Järvinene5befbd2008-08-18 22:30:01 -07001321 err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
1322 if (err) {
1323 if (q)
1324 qdisc_destroy(q);
1325 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326 }
Ilpo Järvinene5befbd2008-08-18 22:30:01 -07001327
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328 return 0;
1329}
1330
1331static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
Eric W. Biederman15e47302012-09-07 20:12:54 +00001332 u32 portid, u32 seq, u16 flags, int event)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333{
John Fastabend22e0f8b2014-09-28 11:52:56 -07001334 struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
John Fastabendb0ab6f92014-09-28 11:54:24 -07001335 struct gnet_stats_queue __percpu *cpu_qstats = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 struct tcmsg *tcm;
1337 struct nlmsghdr *nlh;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001338 unsigned char *b = skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339 struct gnet_dump d;
Eric Dumazeta2da5702011-01-20 03:48:19 +00001340 struct qdisc_size_table *stab;
John Fastabend64015852014-09-28 11:53:57 -07001341 __u32 qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342
Eric Dumazetfba373d2014-03-10 17:11:43 -07001343 cond_resched();
Eric W. Biederman15e47302012-09-07 20:12:54 +00001344 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
David S. Miller02ef22c2012-06-26 21:50:05 -07001345 if (!nlh)
1346 goto out_nlmsg_trim;
1347 tcm = nlmsg_data(nlh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 tcm->tcm_family = AF_UNSPEC;
Patrick McHardy9ef1d4c2005-06-28 12:55:30 -07001349 tcm->tcm__pad1 = 0;
1350 tcm->tcm__pad2 = 0;
David S. Miller5ce2d482008-07-08 17:06:30 -07001351 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 tcm->tcm_parent = clid;
1353 tcm->tcm_handle = q->handle;
1354 tcm->tcm_info = atomic_read(&q->refcnt);
David S. Miller1b34ec42012-03-29 05:11:39 -04001355 if (nla_put_string(skb, TCA_KIND, q->ops->id))
1356 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 if (q->ops->dump && q->ops->dump(q, skb) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001358 goto nla_put_failure;
John Fastabend64015852014-09-28 11:53:57 -07001359 qlen = q->q.qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360
Eric Dumazeta2da5702011-01-20 03:48:19 +00001361 stab = rtnl_dereference(q->stab);
1362 if (stab && qdisc_dump_stab(skb, stab) < 0)
Jussi Kivilinna175f9c12008-07-20 00:08:47 -07001363 goto nla_put_failure;
1364
Jarek Poplawski102396a2008-08-29 14:21:52 -07001365 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1366 qdisc_root_sleeping_lock(q), &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001367 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368
1369 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001370 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371
John Fastabendb0ab6f92014-09-28 11:54:24 -07001372 if (qdisc_is_percpu_stats(q)) {
John Fastabend22e0f8b2014-09-28 11:52:56 -07001373 cpu_bstats = q->cpu_bstats;
John Fastabendb0ab6f92014-09-28 11:54:24 -07001374 cpu_qstats = q->cpu_qstats;
1375 }
John Fastabend22e0f8b2014-09-28 11:52:56 -07001376
1377 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats) < 0 ||
Eric Dumazetd250a5f2009-10-02 10:32:18 +00001378 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
John Fastabendb0ab6f92014-09-28 11:54:24 -07001379 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001380 goto nla_put_failure;
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001381
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 if (gnet_stats_finish_copy(&d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001383 goto nla_put_failure;
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001384
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001385 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386 return skb->len;
1387
David S. Miller02ef22c2012-06-26 21:50:05 -07001388out_nlmsg_trim:
Patrick McHardy1e904742008-01-22 22:11:17 -08001389nla_put_failure:
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -07001390 nlmsg_trim(skb, b);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391 return -1;
1392}
1393
Eric Dumazet53b0f082010-05-22 20:37:44 +00001394static bool tc_qdisc_dump_ignore(struct Qdisc *q)
1395{
1396 return (q->flags & TCQ_F_BUILTIN) ? true : false;
1397}
1398
Tom Goff7316ae82010-03-19 15:40:13 +00001399static int qdisc_notify(struct net *net, struct sk_buff *oskb,
1400 struct nlmsghdr *n, u32 clid,
1401 struct Qdisc *old, struct Qdisc *new)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402{
1403 struct sk_buff *skb;
Eric W. Biederman15e47302012-09-07 20:12:54 +00001404 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405
1406 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1407 if (!skb)
1408 return -ENOBUFS;
1409
Eric Dumazet53b0f082010-05-22 20:37:44 +00001410 if (old && !tc_qdisc_dump_ignore(old)) {
Eric W. Biederman15e47302012-09-07 20:12:54 +00001411 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001412 0, RTM_DELQDISC) < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413 goto err_out;
1414 }
Eric Dumazet53b0f082010-05-22 20:37:44 +00001415 if (new && !tc_qdisc_dump_ignore(new)) {
Eric W. Biederman15e47302012-09-07 20:12:54 +00001416 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001417 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 goto err_out;
1419 }
1420
1421 if (skb->len)
Eric W. Biederman15e47302012-09-07 20:12:54 +00001422 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001423 n->nlmsg_flags & NLM_F_ECHO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424
1425err_out:
1426 kfree_skb(skb);
1427 return -EINVAL;
1428}
1429
David S. Miller30723672008-07-18 22:50:15 -07001430static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1431 struct netlink_callback *cb,
1432 int *q_idx_p, int s_q_idx)
1433{
1434 int ret = 0, q_idx = *q_idx_p;
1435 struct Qdisc *q;
1436
1437 if (!root)
1438 return 0;
1439
1440 q = root;
1441 if (q_idx < s_q_idx) {
1442 q_idx++;
1443 } else {
1444 if (!tc_qdisc_dump_ignore(q) &&
Eric W. Biederman15e47302012-09-07 20:12:54 +00001445 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
David S. Miller30723672008-07-18 22:50:15 -07001446 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1447 goto done;
1448 q_idx++;
1449 }
1450 list_for_each_entry(q, &root->list, list) {
1451 if (q_idx < s_q_idx) {
1452 q_idx++;
1453 continue;
1454 }
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001455 if (!tc_qdisc_dump_ignore(q) &&
Eric W. Biederman15e47302012-09-07 20:12:54 +00001456 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
David S. Miller30723672008-07-18 22:50:15 -07001457 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1458 goto done;
1459 q_idx++;
1460 }
1461
1462out:
1463 *q_idx_p = q_idx;
1464 return ret;
1465done:
1466 ret = -1;
1467 goto out;
1468}
1469
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1471{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001472 struct net *net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473 int idx, q_idx;
1474 int s_idx, s_q_idx;
1475 struct net_device *dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476
1477 s_idx = cb->args[0];
1478 s_q_idx = q_idx = cb->args[1];
stephen hemmingerf1e90162009-11-10 07:54:49 +00001479
Pavel Emelianov7562f872007-05-03 15:13:45 -07001480 idx = 0;
Eric Dumazet15dc36e2014-03-10 17:11:42 -07001481 ASSERT_RTNL();
1482 for_each_netdev(net, dev) {
David S. Miller30723672008-07-18 22:50:15 -07001483 struct netdev_queue *dev_queue;
1484
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485 if (idx < s_idx)
Pavel Emelianov7562f872007-05-03 15:13:45 -07001486 goto cont;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 if (idx > s_idx)
1488 s_q_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 q_idx = 0;
David S. Miller30723672008-07-18 22:50:15 -07001490
Patrick McHardyaf356af2009-09-04 06:41:18 +00001491 if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx) < 0)
David S. Miller30723672008-07-18 22:50:15 -07001492 goto done;
1493
Eric Dumazet24824a02010-10-02 06:11:55 +00001494 dev_queue = dev_ingress_queue(dev);
1495 if (dev_queue &&
1496 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1497 &q_idx, s_q_idx) < 0)
David S. Miller30723672008-07-18 22:50:15 -07001498 goto done;
1499
Pavel Emelianov7562f872007-05-03 15:13:45 -07001500cont:
1501 idx++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 }
1503
1504done:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001505 cb->args[0] = idx;
1506 cb->args[1] = q_idx;
1507
1508 return skb->len;
1509}
1510
1511
1512
1513/************************************************
1514 * Traffic classes manipulation. *
1515 ************************************************/
1516
1517
1518
Thomas Graf661d2962013-03-21 07:45:29 +00001519static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001521 struct net *net = sock_net(skb->sk);
David S. Miller02ef22c2012-06-26 21:50:05 -07001522 struct tcmsg *tcm = nlmsg_data(n);
Patrick McHardy1e904742008-01-22 22:11:17 -08001523 struct nlattr *tca[TCA_MAX + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524 struct net_device *dev;
1525 struct Qdisc *q = NULL;
Eric Dumazet20fea082007-11-14 01:44:41 -08001526 const struct Qdisc_class_ops *cops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527 unsigned long cl = 0;
1528 unsigned long new_cl;
Hong zhi guode179c82013-03-25 17:36:33 +00001529 u32 portid;
1530 u32 clid;
1531 u32 qid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 int err;
1533
Stéphane Graber4e8bbb82014-04-30 11:25:43 -04001534 if ((n->nlmsg_type != RTM_GETTCLASS) &&
David S. Miller5f013c9b2014-05-12 13:19:14 -04001535 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
Eric W. Biedermandfc47ef2012-11-16 03:03:00 +00001536 return -EPERM;
1537
Patrick McHardy1e904742008-01-22 22:11:17 -08001538 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1539 if (err < 0)
1540 return err;
1541
Hong zhi guode179c82013-03-25 17:36:33 +00001542 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1543 if (!dev)
1544 return -ENODEV;
1545
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 /*
1547 parent == TC_H_UNSPEC - unspecified parent.
1548 parent == TC_H_ROOT - class is root, which has no parent.
1549 parent == X:0 - parent is root class.
1550 parent == X:Y - parent is a node in hierarchy.
1551 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
1552
1553 handle == 0:0 - generate handle from kernel pool.
1554 handle == 0:Y - class is X:Y, where X:0 is qdisc.
1555 handle == X:Y - clear.
1556 handle == X:0 - root class.
1557 */
1558
1559 /* Step 1. Determine qdisc handle X:0 */
1560
Hong zhi guode179c82013-03-25 17:36:33 +00001561 portid = tcm->tcm_parent;
1562 clid = tcm->tcm_handle;
1563 qid = TC_H_MAJ(clid);
1564
Eric W. Biederman15e47302012-09-07 20:12:54 +00001565 if (portid != TC_H_ROOT) {
1566 u32 qid1 = TC_H_MAJ(portid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567
1568 if (qid && qid1) {
1569 /* If both majors are known, they must be identical. */
1570 if (qid != qid1)
1571 return -EINVAL;
1572 } else if (qid1) {
1573 qid = qid1;
1574 } else if (qid == 0)
Patrick McHardyaf356af2009-09-04 06:41:18 +00001575 qid = dev->qdisc->handle;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576
1577 /* Now qid is genuine qdisc handle consistent
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001578 * both with parent and child.
1579 *
Eric W. Biederman15e47302012-09-07 20:12:54 +00001580 * TC_H_MAJ(portid) still may be unspecified, complete it now.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581 */
Eric W. Biederman15e47302012-09-07 20:12:54 +00001582 if (portid)
1583 portid = TC_H_MAKE(qid, portid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584 } else {
1585 if (qid == 0)
Patrick McHardyaf356af2009-09-04 06:41:18 +00001586 qid = dev->qdisc->handle;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587 }
1588
1589 /* OK. Locate qdisc */
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001590 q = qdisc_lookup(dev, qid);
1591 if (!q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592 return -ENOENT;
1593
1594 /* An check that it supports classes */
1595 cops = q->ops->cl_ops;
1596 if (cops == NULL)
1597 return -EINVAL;
1598
1599 /* Now try to get class */
1600 if (clid == 0) {
Eric W. Biederman15e47302012-09-07 20:12:54 +00001601 if (portid == TC_H_ROOT)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602 clid = qid;
1603 } else
1604 clid = TC_H_MAKE(qid, clid);
1605
1606 if (clid)
1607 cl = cops->get(q, clid);
1608
1609 if (cl == 0) {
1610 err = -ENOENT;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001611 if (n->nlmsg_type != RTM_NEWTCLASS ||
1612 !(n->nlmsg_flags & NLM_F_CREATE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613 goto out;
1614 } else {
1615 switch (n->nlmsg_type) {
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001616 case RTM_NEWTCLASS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 err = -EEXIST;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001618 if (n->nlmsg_flags & NLM_F_EXCL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619 goto out;
1620 break;
1621 case RTM_DELTCLASS:
Patrick McHardyde6d5cd2009-09-04 06:41:16 +00001622 err = -EOPNOTSUPP;
1623 if (cops->delete)
1624 err = cops->delete(q, cl);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625 if (err == 0)
Tom Goff7316ae82010-03-19 15:40:13 +00001626 tclass_notify(net, skb, n, q, cl, RTM_DELTCLASS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627 goto out;
1628 case RTM_GETTCLASS:
Tom Goff7316ae82010-03-19 15:40:13 +00001629 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 goto out;
1631 default:
1632 err = -EINVAL;
1633 goto out;
1634 }
1635 }
1636
1637 new_cl = cl;
Patrick McHardyde6d5cd2009-09-04 06:41:16 +00001638 err = -EOPNOTSUPP;
1639 if (cops->change)
Eric W. Biederman15e47302012-09-07 20:12:54 +00001640 err = cops->change(q, clid, portid, tca, &new_cl);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641 if (err == 0)
Tom Goff7316ae82010-03-19 15:40:13 +00001642 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643
1644out:
1645 if (cl)
1646 cops->put(q, cl);
1647
1648 return err;
1649}
1650
1651
1652static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1653 unsigned long cl,
Eric W. Biederman15e47302012-09-07 20:12:54 +00001654 u32 portid, u32 seq, u16 flags, int event)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655{
1656 struct tcmsg *tcm;
1657 struct nlmsghdr *nlh;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001658 unsigned char *b = skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659 struct gnet_dump d;
Eric Dumazet20fea082007-11-14 01:44:41 -08001660 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661
Eric Dumazetfba373d2014-03-10 17:11:43 -07001662 cond_resched();
Eric W. Biederman15e47302012-09-07 20:12:54 +00001663 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
David S. Miller02ef22c2012-06-26 21:50:05 -07001664 if (!nlh)
1665 goto out_nlmsg_trim;
1666 tcm = nlmsg_data(nlh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 tcm->tcm_family = AF_UNSPEC;
Eric Dumazet16ebb5e2009-09-02 02:40:09 +00001668 tcm->tcm__pad1 = 0;
1669 tcm->tcm__pad2 = 0;
David S. Miller5ce2d482008-07-08 17:06:30 -07001670 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671 tcm->tcm_parent = q->handle;
1672 tcm->tcm_handle = q->handle;
1673 tcm->tcm_info = 0;
David S. Miller1b34ec42012-03-29 05:11:39 -04001674 if (nla_put_string(skb, TCA_KIND, q->ops->id))
1675 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001677 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001678
Jarek Poplawski102396a2008-08-29 14:21:52 -07001679 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1680 qdisc_root_sleeping_lock(q), &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001681 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682
1683 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001684 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685
1686 if (gnet_stats_finish_copy(&d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001687 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001689 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690 return skb->len;
1691
David S. Miller02ef22c2012-06-26 21:50:05 -07001692out_nlmsg_trim:
Patrick McHardy1e904742008-01-22 22:11:17 -08001693nla_put_failure:
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -07001694 nlmsg_trim(skb, b);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 return -1;
1696}
1697
Tom Goff7316ae82010-03-19 15:40:13 +00001698static int tclass_notify(struct net *net, struct sk_buff *oskb,
1699 struct nlmsghdr *n, struct Qdisc *q,
1700 unsigned long cl, int event)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701{
1702 struct sk_buff *skb;
Eric W. Biederman15e47302012-09-07 20:12:54 +00001703 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704
1705 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1706 if (!skb)
1707 return -ENOBUFS;
1708
Eric W. Biederman15e47302012-09-07 20:12:54 +00001709 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001710 kfree_skb(skb);
1711 return -EINVAL;
1712 }
1713
Eric W. Biederman15e47302012-09-07 20:12:54 +00001714 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001715 n->nlmsg_flags & NLM_F_ECHO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716}
1717
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001718struct qdisc_dump_args {
1719 struct qdisc_walker w;
1720 struct sk_buff *skb;
1721 struct netlink_callback *cb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001722};
1723
1724static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg)
1725{
1726 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1727
Eric W. Biederman15e47302012-09-07 20:12:54 +00001728 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS);
1730}
1731
David S. Miller30723672008-07-18 22:50:15 -07001732static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
1733 struct tcmsg *tcm, struct netlink_callback *cb,
1734 int *t_p, int s_t)
1735{
1736 struct qdisc_dump_args arg;
1737
1738 if (tc_qdisc_dump_ignore(q) ||
1739 *t_p < s_t || !q->ops->cl_ops ||
1740 (tcm->tcm_parent &&
1741 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1742 (*t_p)++;
1743 return 0;
1744 }
1745 if (*t_p > s_t)
1746 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1747 arg.w.fn = qdisc_class_dump;
1748 arg.skb = skb;
1749 arg.cb = cb;
1750 arg.w.stop = 0;
1751 arg.w.skip = cb->args[1];
1752 arg.w.count = 0;
1753 q->ops->cl_ops->walk(q, &arg.w);
1754 cb->args[1] = arg.w.count;
1755 if (arg.w.stop)
1756 return -1;
1757 (*t_p)++;
1758 return 0;
1759}
1760
1761static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
1762 struct tcmsg *tcm, struct netlink_callback *cb,
1763 int *t_p, int s_t)
1764{
1765 struct Qdisc *q;
1766
1767 if (!root)
1768 return 0;
1769
1770 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
1771 return -1;
1772
1773 list_for_each_entry(q, &root->list, list) {
1774 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
1775 return -1;
1776 }
1777
1778 return 0;
1779}
1780
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1782{
David S. Miller02ef22c2012-06-26 21:50:05 -07001783 struct tcmsg *tcm = nlmsg_data(cb->nlh);
David S. Miller30723672008-07-18 22:50:15 -07001784 struct net *net = sock_net(skb->sk);
1785 struct netdev_queue *dev_queue;
1786 struct net_device *dev;
1787 int t, s_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001788
Hong zhi guo573ce262013-03-27 06:47:04 +00001789 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790 return 0;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001791 dev = dev_get_by_index(net, tcm->tcm_ifindex);
1792 if (!dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793 return 0;
1794
1795 s_t = cb->args[0];
1796 t = 0;
1797
Patrick McHardyaf356af2009-09-04 06:41:18 +00001798 if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t) < 0)
David S. Miller30723672008-07-18 22:50:15 -07001799 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800
Eric Dumazet24824a02010-10-02 06:11:55 +00001801 dev_queue = dev_ingress_queue(dev);
1802 if (dev_queue &&
1803 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
1804 &t, s_t) < 0)
David S. Miller30723672008-07-18 22:50:15 -07001805 goto done;
1806
1807done:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001808 cb->args[0] = t;
1809
1810 dev_put(dev);
1811 return skb->len;
1812}
1813
1814/* Main classifier routine: scans classifier chain attached
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001815 * to this qdisc, (optionally) tests for protocol and asks
1816 * specific classifiers.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001817 */
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001818int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1819 struct tcf_result *res, bool compat_mode)
Patrick McHardy73ca4912007-07-15 00:02:31 -07001820{
Jiri Pirkod8b96052015-01-13 17:13:43 +01001821 __be16 protocol = tc_skb_protocol(skb);
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001822#ifdef CONFIG_NET_CLS_ACT
1823 const struct tcf_proto *old_tp = tp;
1824 int limit = 0;
Patrick McHardy73ca4912007-07-15 00:02:31 -07001825
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001826reclassify:
1827#endif
John Fastabend25d8c0d2014-09-12 20:05:27 -07001828 for (; tp; tp = rcu_dereference_bh(tp->next)) {
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001829 int err;
1830
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001831 if (tp->protocol != protocol &&
1832 tp->protocol != htons(ETH_P_ALL))
1833 continue;
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001834
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001835 err = tp->classify(skb, tp, res);
1836#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmannc1b3b192015-08-28 18:46:39 +02001837 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode))
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001838 goto reset;
1839#endif
Florian Westphale578d9c2015-05-11 19:50:41 +02001840 if (err >= 0)
Patrick McHardy73ca4912007-07-15 00:02:31 -07001841 return err;
Patrick McHardy73ca4912007-07-15 00:02:31 -07001842 }
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001843
Patrick McHardy73ca4912007-07-15 00:02:31 -07001844 return -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001845#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001846reset:
1847 if (unlikely(limit++ >= MAX_REC_LOOP)) {
Daniel Borkmannc1b3b192015-08-28 18:46:39 +02001848 net_notice_ratelimited("%s: reclassify loop, rule prio %u, protocol %02x\n",
1849 tp->q->ops->id, tp->prio & 0xffff,
1850 ntohs(tp->protocol));
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001851 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001852 }
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02001853
1854 tp = old_tp;
1855 goto reclassify;
Patrick McHardy73ca4912007-07-15 00:02:31 -07001856#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857}
Patrick McHardy73ca4912007-07-15 00:02:31 -07001858EXPORT_SYMBOL(tc_classify);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859
Cong Wang1e052be2015-03-06 11:47:59 -08001860bool tcf_destroy(struct tcf_proto *tp, bool force)
Patrick McHardya48b5a62007-03-23 11:29:43 -07001861{
Cong Wang1e052be2015-03-06 11:47:59 -08001862 if (tp->ops->destroy(tp, force)) {
1863 module_put(tp->ops->owner);
1864 kfree_rcu(tp, rcu);
1865 return true;
1866 }
1867
1868 return false;
Patrick McHardya48b5a62007-03-23 11:29:43 -07001869}
1870
John Fastabend25d8c0d2014-09-12 20:05:27 -07001871void tcf_destroy_chain(struct tcf_proto __rcu **fl)
Patrick McHardya48b5a62007-03-23 11:29:43 -07001872{
1873 struct tcf_proto *tp;
1874
John Fastabend25d8c0d2014-09-12 20:05:27 -07001875 while ((tp = rtnl_dereference(*fl)) != NULL) {
1876 RCU_INIT_POINTER(*fl, tp->next);
Cong Wang1e052be2015-03-06 11:47:59 -08001877 tcf_destroy(tp, true);
Patrick McHardya48b5a62007-03-23 11:29:43 -07001878 }
1879}
1880EXPORT_SYMBOL(tcf_destroy_chain);
1881
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882#ifdef CONFIG_PROC_FS
1883static int psched_show(struct seq_file *seq, void *v)
1884{
1885 seq_printf(seq, "%08x %08x %08x %08x\n",
Jarek Poplawskica44d6e2009-06-15 02:31:47 -07001886 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
Patrick McHardy514bca32007-03-16 12:34:52 -07001887 1000000,
Thomas Gleixner1e317682015-04-14 21:08:28 +00001888 (u32)NSEC_PER_SEC / hrtimer_resolution);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001889
1890 return 0;
1891}
1892
1893static int psched_open(struct inode *inode, struct file *file)
1894{
Tom Goff7e5ab152010-03-30 19:44:56 -07001895 return single_open(file, psched_show, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896}
1897
Arjan van de Venda7071d2007-02-12 00:55:36 -08001898static const struct file_operations psched_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899 .owner = THIS_MODULE,
1900 .open = psched_open,
1901 .read = seq_read,
1902 .llseek = seq_lseek,
1903 .release = single_release,
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001904};
Tom Goff7316ae82010-03-19 15:40:13 +00001905
1906static int __net_init psched_net_init(struct net *net)
1907{
1908 struct proc_dir_entry *e;
1909
Gao fengd4beaa62013-02-18 01:34:54 +00001910 e = proc_create("psched", 0, net->proc_net, &psched_fops);
Tom Goff7316ae82010-03-19 15:40:13 +00001911 if (e == NULL)
1912 return -ENOMEM;
1913
1914 return 0;
1915}
1916
1917static void __net_exit psched_net_exit(struct net *net)
1918{
Gao fengece31ff2013-02-18 01:34:56 +00001919 remove_proc_entry("psched", net->proc_net);
Tom Goff7316ae82010-03-19 15:40:13 +00001920}
1921#else
1922static int __net_init psched_net_init(struct net *net)
1923{
1924 return 0;
1925}
1926
1927static void __net_exit psched_net_exit(struct net *net)
1928{
1929}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001930#endif
1931
Tom Goff7316ae82010-03-19 15:40:13 +00001932static struct pernet_operations psched_net_ops = {
1933 .init = psched_net_init,
1934 .exit = psched_net_exit,
1935};
1936
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937static int __init pktsched_init(void)
1938{
Tom Goff7316ae82010-03-19 15:40:13 +00001939 int err;
1940
1941 err = register_pernet_subsys(&psched_net_ops);
1942 if (err) {
Eric Dumazetcc7ec452011-01-19 19:26:56 +00001943 pr_err("pktsched_init: "
Tom Goff7316ae82010-03-19 15:40:13 +00001944 "cannot initialize per netns operations\n");
1945 return err;
1946 }
1947
stephen hemminger6da7c8f2013-08-27 16:19:08 -07001948 register_qdisc(&pfifo_fast_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001949 register_qdisc(&pfifo_qdisc_ops);
1950 register_qdisc(&bfifo_qdisc_ops);
Hagen Paul Pfeifer57dbb2d2010-01-24 12:30:59 +00001951 register_qdisc(&pfifo_head_drop_qdisc_ops);
David S. Miller6ec1c692009-09-06 01:58:51 -07001952 register_qdisc(&mq_qdisc_ops);
Phil Sutterd66d6c32015-08-27 21:21:38 +02001953 register_qdisc(&noqueue_qdisc_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954
Greg Rosec7ac8672011-06-10 01:27:09 +00001955 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, NULL);
1956 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, NULL);
1957 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc, NULL);
1958 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, NULL);
1959 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, NULL);
1960 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass, NULL);
Thomas Grafbe577dd2007-03-22 11:55:50 -07001961
Linus Torvalds1da177e2005-04-16 15:20:36 -07001962 return 0;
1963}
1964
1965subsys_initcall(pktsched_init);