blob: 9a2151f5d5935a50f2bfa7fed6fec23626c8a457 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* flow.c: Generic flow cache.
2 *
3 * Copyright (C) 2003 Alexey N. Kuznetsov (kuznet@ms2.inr.ac.ru)
4 * Copyright (C) 2003 David S. Miller (davem@redhat.com)
5 */
6
7#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/list.h>
10#include <linux/jhash.h>
11#include <linux/interrupt.h>
12#include <linux/mm.h>
13#include <linux/random.h>
14#include <linux/init.h>
15#include <linux/slab.h>
16#include <linux/smp.h>
17#include <linux/completion.h>
18#include <linux/percpu.h>
19#include <linux/bitops.h>
20#include <linux/notifier.h>
21#include <linux/cpu.h>
22#include <linux/cpumask.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080023#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <net/flow.h>
Arun Sharma600634972011-07-26 16:09:06 -070025#include <linux/atomic.h>
Trent Jaegerdf718372005-12-13 23:12:27 -080026#include <linux/security.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
28struct flow_cache_entry {
Timo Teräs8e4795602010-04-07 00:30:07 +000029 union {
30 struct hlist_node hlist;
31 struct list_head gc_list;
32 } u;
dpward0542b692011-08-31 06:05:27 +000033 struct net *net;
Timo Teräsfe1a5f02010-04-07 00:30:04 +000034 u16 family;
35 u8 dir;
36 u32 genid;
37 struct flowi key;
38 struct flow_cache_object *object;
Linus Torvalds1da177e2005-04-16 15:20:36 -070039};
40
Timo Teräsd7997fe2010-03-31 00:17:06 +000041struct flow_cache_percpu {
Timo Teräs8e4795602010-04-07 00:30:07 +000042 struct hlist_head *hash_table;
Timo Teräsd7997fe2010-03-31 00:17:06 +000043 int hash_count;
44 u32 hash_rnd;
45 int hash_rnd_recalc;
46 struct tasklet_struct flush_tasklet;
Eric Dumazet5f58a5c2008-02-07 18:03:18 -080047};
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
49struct flow_flush_info {
Timo Teräsfe1a5f02010-04-07 00:30:04 +000050 struct flow_cache *cache;
Timo Teräsd7997fe2010-03-31 00:17:06 +000051 atomic_t cpuleft;
52 struct completion completion;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053};
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
Timo Teräsd7997fe2010-03-31 00:17:06 +000055struct flow_cache {
56 u32 hash_shift;
Eric Dumazet83b6b1f2010-09-10 07:00:25 +000057 struct flow_cache_percpu __percpu *percpu;
Timo Teräsd7997fe2010-03-31 00:17:06 +000058 struct notifier_block hotcpu_notifier;
59 int low_watermark;
60 int high_watermark;
61 struct timer_list rnd_timer;
62};
63
64atomic_t flow_cache_genid = ATOMIC_INIT(0);
Eric Dumazet9e34a5b2010-07-09 21:22:04 +000065EXPORT_SYMBOL(flow_cache_genid);
Timo Teräsd7997fe2010-03-31 00:17:06 +000066static struct flow_cache flow_cache_global;
Eric Dumazet83b6b1f2010-09-10 07:00:25 +000067static struct kmem_cache *flow_cachep __read_mostly;
Timo Teräsd7997fe2010-03-31 00:17:06 +000068
Timo Teräs8e4795602010-04-07 00:30:07 +000069static DEFINE_SPINLOCK(flow_cache_gc_lock);
70static LIST_HEAD(flow_cache_gc_list);
71
Timo Teräsd7997fe2010-03-31 00:17:06 +000072#define flow_cache_hash_size(cache) (1 << (cache)->hash_shift)
73#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ)
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
75static void flow_cache_new_hashrnd(unsigned long arg)
76{
Timo Teräsd7997fe2010-03-31 00:17:06 +000077 struct flow_cache *fc = (void *) arg;
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 int i;
79
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -070080 for_each_possible_cpu(i)
Timo Teräsd7997fe2010-03-31 00:17:06 +000081 per_cpu_ptr(fc->percpu, i)->hash_rnd_recalc = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Timo Teräsd7997fe2010-03-31 00:17:06 +000083 fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD;
84 add_timer(&fc->rnd_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -070085}
86
Timo Teräsfe1a5f02010-04-07 00:30:04 +000087static int flow_entry_valid(struct flow_cache_entry *fle)
88{
89 if (atomic_read(&flow_cache_genid) != fle->genid)
90 return 0;
91 if (fle->object && !fle->object->ops->check(fle->object))
92 return 0;
93 return 1;
94}
95
Timo Teräs8e4795602010-04-07 00:30:07 +000096static void flow_entry_kill(struct flow_cache_entry *fle)
James Morris134b0fc2006-10-05 15:42:27 -050097{
98 if (fle->object)
Timo Teräsfe1a5f02010-04-07 00:30:04 +000099 fle->object->ops->delete(fle->object);
James Morris134b0fc2006-10-05 15:42:27 -0500100 kmem_cache_free(flow_cachep, fle);
Timo Teräs8e4795602010-04-07 00:30:07 +0000101}
102
103static void flow_cache_gc_task(struct work_struct *work)
104{
105 struct list_head gc_list;
106 struct flow_cache_entry *fce, *n;
107
108 INIT_LIST_HEAD(&gc_list);
109 spin_lock_bh(&flow_cache_gc_lock);
110 list_splice_tail_init(&flow_cache_gc_list, &gc_list);
111 spin_unlock_bh(&flow_cache_gc_lock);
112
113 list_for_each_entry_safe(fce, n, &gc_list, u.gc_list)
114 flow_entry_kill(fce);
115}
116static DECLARE_WORK(flow_cache_gc_work, flow_cache_gc_task);
117
118static void flow_cache_queue_garbage(struct flow_cache_percpu *fcp,
119 int deleted, struct list_head *gc_list)
120{
121 if (deleted) {
122 fcp->hash_count -= deleted;
123 spin_lock_bh(&flow_cache_gc_lock);
124 list_splice_tail(gc_list, &flow_cache_gc_list);
125 spin_unlock_bh(&flow_cache_gc_lock);
126 schedule_work(&flow_cache_gc_work);
127 }
James Morris134b0fc2006-10-05 15:42:27 -0500128}
129
Timo Teräsd7997fe2010-03-31 00:17:06 +0000130static void __flow_cache_shrink(struct flow_cache *fc,
131 struct flow_cache_percpu *fcp,
132 int shrink_to)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133{
Timo Teräs8e4795602010-04-07 00:30:07 +0000134 struct flow_cache_entry *fle;
Sasha Levinb67bfe02013-02-27 17:06:00 -0800135 struct hlist_node *tmp;
Timo Teräs8e4795602010-04-07 00:30:07 +0000136 LIST_HEAD(gc_list);
137 int i, deleted = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Timo Teräsd7997fe2010-03-31 00:17:06 +0000139 for (i = 0; i < flow_cache_hash_size(fc); i++) {
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000140 int saved = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141
Sasha Levinb67bfe02013-02-27 17:06:00 -0800142 hlist_for_each_entry_safe(fle, tmp,
Timo Teräs8e4795602010-04-07 00:30:07 +0000143 &fcp->hash_table[i], u.hlist) {
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000144 if (saved < shrink_to &&
145 flow_entry_valid(fle)) {
146 saved++;
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000147 } else {
Timo Teräs8e4795602010-04-07 00:30:07 +0000148 deleted++;
149 hlist_del(&fle->u.hlist);
150 list_add_tail(&fle->u.gc_list, &gc_list);
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000151 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 }
153 }
Timo Teräs8e4795602010-04-07 00:30:07 +0000154
155 flow_cache_queue_garbage(fcp, deleted, &gc_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156}
157
Timo Teräsd7997fe2010-03-31 00:17:06 +0000158static void flow_cache_shrink(struct flow_cache *fc,
159 struct flow_cache_percpu *fcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000161 int shrink_to = fc->low_watermark / flow_cache_hash_size(fc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162
Timo Teräsd7997fe2010-03-31 00:17:06 +0000163 __flow_cache_shrink(fc, fcp, shrink_to);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164}
165
Timo Teräsd7997fe2010-03-31 00:17:06 +0000166static void flow_new_hash_rnd(struct flow_cache *fc,
167 struct flow_cache_percpu *fcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000169 get_random_bytes(&fcp->hash_rnd, sizeof(u32));
170 fcp->hash_rnd_recalc = 0;
171 __flow_cache_shrink(fc, fcp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172}
173
Timo Teräsd7997fe2010-03-31 00:17:06 +0000174static u32 flow_hash_code(struct flow_cache *fc,
175 struct flow_cache_percpu *fcp,
dpwardaa1c3662011-09-05 16:47:24 +0000176 const struct flowi *key,
177 size_t keysize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178{
David S. Millerdee9f4b2011-02-22 18:44:31 -0800179 const u32 *k = (const u32 *) key;
dpwardaa1c3662011-09-05 16:47:24 +0000180 const u32 length = keysize * sizeof(flow_compare_t) / sizeof(u32);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
dpwardaa1c3662011-09-05 16:47:24 +0000182 return jhash2(k, length, fcp->hash_rnd)
Eric Dumazeta02cec22010-09-22 20:43:57 +0000183 & (flow_cache_hash_size(fc) - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184}
185
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186/* I hear what you're saying, use memcmp. But memcmp cannot make
dpwardaa1c3662011-09-05 16:47:24 +0000187 * important assumptions that we can here, such as alignment.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 */
dpwardaa1c3662011-09-05 16:47:24 +0000189static int flow_key_compare(const struct flowi *key1, const struct flowi *key2,
190 size_t keysize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191{
David S. Millerdee9f4b2011-02-22 18:44:31 -0800192 const flow_compare_t *k1, *k1_lim, *k2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
David S. Millerdee9f4b2011-02-22 18:44:31 -0800194 k1 = (const flow_compare_t *) key1;
dpwardaa1c3662011-09-05 16:47:24 +0000195 k1_lim = k1 + keysize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
David S. Millerdee9f4b2011-02-22 18:44:31 -0800197 k2 = (const flow_compare_t *) key2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198
199 do {
200 if (*k1++ != *k2++)
201 return 1;
202 } while (k1 < k1_lim);
203
204 return 0;
205}
206
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000207struct flow_cache_object *
David S. Millerdee9f4b2011-02-22 18:44:31 -0800208flow_cache_lookup(struct net *net, const struct flowi *key, u16 family, u8 dir,
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000209 flow_resolve_t resolver, void *ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000211 struct flow_cache *fc = &flow_cache_global;
212 struct flow_cache_percpu *fcp;
Timo Teräs8e4795602010-04-07 00:30:07 +0000213 struct flow_cache_entry *fle, *tfle;
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000214 struct flow_cache_object *flo;
dpwardaa1c3662011-09-05 16:47:24 +0000215 size_t keysize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
218 local_bh_disable();
Eric Dumazet7a9b2d52010-06-24 00:52:37 +0000219 fcp = this_cpu_ptr(fc->percpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
221 fle = NULL;
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000222 flo = NULL;
dpwardaa1c3662011-09-05 16:47:24 +0000223
224 keysize = flow_key_size(family);
225 if (!keysize)
226 goto nocache;
227
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 /* Packet really early in init? Making flow_cache_init a
229 * pre-smp initcall would solve this. --RR */
Timo Teräsd7997fe2010-03-31 00:17:06 +0000230 if (!fcp->hash_table)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 goto nocache;
232
Timo Teräsd7997fe2010-03-31 00:17:06 +0000233 if (fcp->hash_rnd_recalc)
234 flow_new_hash_rnd(fc, fcp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235
dpwardaa1c3662011-09-05 16:47:24 +0000236 hash = flow_hash_code(fc, fcp, key, keysize);
Sasha Levinb67bfe02013-02-27 17:06:00 -0800237 hlist_for_each_entry(tfle, &fcp->hash_table[hash], u.hlist) {
dpward0542b692011-08-31 06:05:27 +0000238 if (tfle->net == net &&
239 tfle->family == family &&
Timo Teräs8e4795602010-04-07 00:30:07 +0000240 tfle->dir == dir &&
dpwardaa1c3662011-09-05 16:47:24 +0000241 flow_key_compare(key, &tfle->key, keysize) == 0) {
Timo Teräs8e4795602010-04-07 00:30:07 +0000242 fle = tfle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 break;
Timo Teräs8e4795602010-04-07 00:30:07 +0000244 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 }
246
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000247 if (unlikely(!fle)) {
Timo Teräsd7997fe2010-03-31 00:17:06 +0000248 if (fcp->hash_count > fc->high_watermark)
249 flow_cache_shrink(fc, fcp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250
Christoph Lameter54e6ecb2006-12-06 20:33:16 -0800251 fle = kmem_cache_alloc(flow_cachep, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 if (fle) {
dpward0542b692011-08-31 06:05:27 +0000253 fle->net = net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 fle->family = family;
255 fle->dir = dir;
dpwardaa1c3662011-09-05 16:47:24 +0000256 memcpy(&fle->key, key, keysize * sizeof(flow_compare_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 fle->object = NULL;
Timo Teräs8e4795602010-04-07 00:30:07 +0000258 hlist_add_head(&fle->u.hlist, &fcp->hash_table[hash]);
Timo Teräsd7997fe2010-03-31 00:17:06 +0000259 fcp->hash_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 }
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000261 } else if (likely(fle->genid == atomic_read(&flow_cache_genid))) {
262 flo = fle->object;
263 if (!flo)
264 goto ret_object;
265 flo = flo->ops->get(flo);
266 if (flo)
267 goto ret_object;
268 } else if (fle->object) {
269 flo = fle->object;
270 flo->ops->delete(flo);
271 fle->object = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 }
273
274nocache:
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000275 flo = NULL;
276 if (fle) {
277 flo = fle->object;
278 fle->object = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 }
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000280 flo = resolver(net, key, family, dir, flo, ctx);
281 if (fle) {
282 fle->genid = atomic_read(&flow_cache_genid);
283 if (!IS_ERR(flo))
284 fle->object = flo;
285 else
286 fle->genid--;
287 } else {
YOSHIFUJI Hideaki / 吉藤英明8fbcec22013-01-22 06:32:44 +0000288 if (!IS_ERR_OR_NULL(flo))
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000289 flo->ops->delete(flo);
290 }
291ret_object:
292 local_bh_enable();
293 return flo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +0000295EXPORT_SYMBOL(flow_cache_lookup);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296
297static void flow_cache_flush_tasklet(unsigned long data)
298{
299 struct flow_flush_info *info = (void *)data;
Timo Teräsd7997fe2010-03-31 00:17:06 +0000300 struct flow_cache *fc = info->cache;
301 struct flow_cache_percpu *fcp;
Timo Teräs8e4795602010-04-07 00:30:07 +0000302 struct flow_cache_entry *fle;
Sasha Levinb67bfe02013-02-27 17:06:00 -0800303 struct hlist_node *tmp;
Timo Teräs8e4795602010-04-07 00:30:07 +0000304 LIST_HEAD(gc_list);
305 int i, deleted = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
Eric Dumazet7a9b2d52010-06-24 00:52:37 +0000307 fcp = this_cpu_ptr(fc->percpu);
Timo Teräsd7997fe2010-03-31 00:17:06 +0000308 for (i = 0; i < flow_cache_hash_size(fc); i++) {
Sasha Levinb67bfe02013-02-27 17:06:00 -0800309 hlist_for_each_entry_safe(fle, tmp,
Timo Teräs8e4795602010-04-07 00:30:07 +0000310 &fcp->hash_table[i], u.hlist) {
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000311 if (flow_entry_valid(fle))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 continue;
313
Timo Teräs8e4795602010-04-07 00:30:07 +0000314 deleted++;
315 hlist_del(&fle->u.hlist);
316 list_add_tail(&fle->u.gc_list, &gc_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 }
318 }
319
Timo Teräs8e4795602010-04-07 00:30:07 +0000320 flow_cache_queue_garbage(fcp, deleted, &gc_list);
321
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 if (atomic_dec_and_test(&info->cpuleft))
323 complete(&info->completion);
324}
325
Chris Metcalf8fdc9292013-03-19 11:35:58 +0000326/*
327 * Return whether a cpu needs flushing. Conservatively, we assume
328 * the presence of any entries means the core may require flushing,
329 * since the flow_cache_ops.check() function may assume it's running
330 * on the same core as the per-cpu cache component.
331 */
332static int flow_cache_percpu_empty(struct flow_cache *fc, int cpu)
333{
334 struct flow_cache_percpu *fcp;
335 int i;
336
Li RongQing27815032013-03-28 02:24:11 +0000337 fcp = per_cpu_ptr(fc->percpu, cpu);
Chris Metcalf8fdc9292013-03-19 11:35:58 +0000338 for (i = 0; i < flow_cache_hash_size(fc); i++)
339 if (!hlist_empty(&fcp->hash_table[i]))
340 return 0;
341 return 1;
342}
343
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344static void flow_cache_flush_per_cpu(void *data)
345{
346 struct flow_flush_info *info = data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 struct tasklet_struct *tasklet;
348
Li RongQing50eab052013-03-27 23:42:41 +0000349 tasklet = &this_cpu_ptr(info->cache->percpu)->flush_tasklet;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 tasklet->data = (unsigned long)info;
351 tasklet_schedule(tasklet);
352}
353
354void flow_cache_flush(void)
355{
356 struct flow_flush_info info;
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -0800357 static DEFINE_MUTEX(flow_flush_sem);
Chris Metcalf8fdc9292013-03-19 11:35:58 +0000358 cpumask_var_t mask;
359 int i, self;
360
361 /* Track which cpus need flushing to avoid disturbing all cores. */
362 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
363 return;
364 cpumask_clear(mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365
366 /* Don't want cpus going down or up during this. */
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100367 get_online_cpus();
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -0800368 mutex_lock(&flow_flush_sem);
Timo Teräsd7997fe2010-03-31 00:17:06 +0000369 info.cache = &flow_cache_global;
Chris Metcalf8fdc9292013-03-19 11:35:58 +0000370 for_each_online_cpu(i)
371 if (!flow_cache_percpu_empty(info.cache, i))
372 cpumask_set_cpu(i, mask);
373 atomic_set(&info.cpuleft, cpumask_weight(mask));
374 if (atomic_read(&info.cpuleft) == 0)
375 goto done;
376
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 init_completion(&info.completion);
378
379 local_bh_disable();
Chris Metcalf8fdc9292013-03-19 11:35:58 +0000380 self = cpumask_test_and_clear_cpu(smp_processor_id(), mask);
381 on_each_cpu_mask(mask, flow_cache_flush_per_cpu, &info, 0);
382 if (self)
383 flow_cache_flush_tasklet((unsigned long)&info);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 local_bh_enable();
385
386 wait_for_completion(&info.completion);
Chris Metcalf8fdc9292013-03-19 11:35:58 +0000387
388done:
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -0800389 mutex_unlock(&flow_flush_sem);
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100390 put_online_cpus();
Chris Metcalf8fdc9292013-03-19 11:35:58 +0000391 free_cpumask_var(mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392}
393
Steffen Klassertc0ed1c12011-12-21 16:48:08 -0500394static void flow_cache_flush_task(struct work_struct *work)
395{
396 flow_cache_flush();
397}
398
399static DECLARE_WORK(flow_cache_flush_work, flow_cache_flush_task);
400
401void flow_cache_flush_deferred(void)
402{
403 schedule_work(&flow_cache_flush_work);
404}
405
Paul Gortmaker013dbb32013-06-19 14:32:33 -0400406static int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407{
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000408 struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
409 size_t sz = sizeof(struct hlist_head) * flow_cache_hash_size(fc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000411 if (!fcp->hash_table) {
412 fcp->hash_table = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
413 if (!fcp->hash_table) {
414 pr_err("NET: failed to allocate flow cache sz %zu\n", sz);
415 return -ENOMEM;
416 }
417 fcp->hash_rnd_recalc = 1;
418 fcp->hash_count = 0;
419 tasklet_init(&fcp->flush_tasklet, flow_cache_flush_tasklet, 0);
420 }
421 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422}
423
Paul Gortmaker013dbb32013-06-19 14:32:33 -0400424static int flow_cache_cpu(struct notifier_block *nfb,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 unsigned long action,
426 void *hcpu)
427{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000428 struct flow_cache *fc = container_of(nfb, struct flow_cache, hotcpu_notifier);
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000429 int res, cpu = (unsigned long) hcpu;
Timo Teräsd7997fe2010-03-31 00:17:06 +0000430 struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
431
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000432 switch (action) {
433 case CPU_UP_PREPARE:
434 case CPU_UP_PREPARE_FROZEN:
435 res = flow_cache_cpu_prepare(fc, cpu);
436 if (res)
437 return notifier_from_errno(res);
438 break;
439 case CPU_DEAD:
440 case CPU_DEAD_FROZEN:
Timo Teräsd7997fe2010-03-31 00:17:06 +0000441 __flow_cache_shrink(fc, fcp, 0);
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000442 break;
443 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 return NOTIFY_OK;
445}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000447static int __init flow_cache_init(struct flow_cache *fc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448{
449 int i;
450
Timo Teräsd7997fe2010-03-31 00:17:06 +0000451 fc->hash_shift = 10;
452 fc->low_watermark = 2 * flow_cache_hash_size(fc);
453 fc->high_watermark = 4 * flow_cache_hash_size(fc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454
Timo Teräsd7997fe2010-03-31 00:17:06 +0000455 fc->percpu = alloc_percpu(struct flow_cache_percpu);
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000456 if (!fc->percpu)
457 return -ENOMEM;
458
Srivatsa S. Bhate30a2932014-03-11 02:12:51 +0530459 cpu_notifier_register_begin();
460
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000461 for_each_online_cpu(i) {
462 if (flow_cache_cpu_prepare(fc, i))
huajun li6ccc3ab2011-09-27 22:51:39 +0000463 goto err;
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000464 }
465 fc->hotcpu_notifier = (struct notifier_block){
466 .notifier_call = flow_cache_cpu,
467 };
Srivatsa S. Bhate30a2932014-03-11 02:12:51 +0530468 __register_hotcpu_notifier(&fc->hotcpu_notifier);
469
470 cpu_notifier_register_done();
Timo Teräsd7997fe2010-03-31 00:17:06 +0000471
472 setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd,
473 (unsigned long) fc);
474 fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD;
475 add_timer(&fc->rnd_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 return 0;
huajun li6ccc3ab2011-09-27 22:51:39 +0000478
479err:
480 for_each_possible_cpu(i) {
481 struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i);
482 kfree(fcp->hash_table);
483 fcp->hash_table = NULL;
484 }
485
Srivatsa S. Bhate30a2932014-03-11 02:12:51 +0530486 cpu_notifier_register_done();
487
huajun li6ccc3ab2011-09-27 22:51:39 +0000488 free_percpu(fc->percpu);
489 fc->percpu = NULL;
490
491 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492}
493
Timo Teräsd7997fe2010-03-31 00:17:06 +0000494static int __init flow_cache_init_global(void)
495{
496 flow_cachep = kmem_cache_create("flow_cache",
497 sizeof(struct flow_cache_entry),
498 0, SLAB_PANIC, NULL);
499
500 return flow_cache_init(&flow_cache_global);
501}
502
503module_init(flow_cache_init_global);