blob: 8ae42de9c79e78379691f20207987c5b2ea338b6 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* flow.c: Generic flow cache.
2 *
3 * Copyright (C) 2003 Alexey N. Kuznetsov (kuznet@ms2.inr.ac.ru)
4 * Copyright (C) 2003 David S. Miller (davem@redhat.com)
5 */
6
7#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/list.h>
10#include <linux/jhash.h>
11#include <linux/interrupt.h>
12#include <linux/mm.h>
13#include <linux/random.h>
14#include <linux/init.h>
15#include <linux/slab.h>
16#include <linux/smp.h>
17#include <linux/completion.h>
18#include <linux/percpu.h>
19#include <linux/bitops.h>
20#include <linux/notifier.h>
21#include <linux/cpu.h>
22#include <linux/cpumask.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080023#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <net/flow.h>
Arun Sharma600634972011-07-26 16:09:06 -070025#include <linux/atomic.h>
Trent Jaegerdf718372005-12-13 23:12:27 -080026#include <linux/security.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
28struct flow_cache_entry {
Timo Teräs8e4795602010-04-07 00:30:07 +000029 union {
30 struct hlist_node hlist;
31 struct list_head gc_list;
32 } u;
dpward0542b692011-08-31 06:05:27 +000033 struct net *net;
Timo Teräsfe1a5f02010-04-07 00:30:04 +000034 u16 family;
35 u8 dir;
36 u32 genid;
37 struct flowi key;
38 struct flow_cache_object *object;
Linus Torvalds1da177e2005-04-16 15:20:36 -070039};
40
Timo Teräsd7997fe2010-03-31 00:17:06 +000041struct flow_cache_percpu {
Timo Teräs8e4795602010-04-07 00:30:07 +000042 struct hlist_head *hash_table;
Timo Teräsd7997fe2010-03-31 00:17:06 +000043 int hash_count;
44 u32 hash_rnd;
45 int hash_rnd_recalc;
46 struct tasklet_struct flush_tasklet;
Eric Dumazet5f58a5c2008-02-07 18:03:18 -080047};
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
49struct flow_flush_info {
Timo Teräsfe1a5f02010-04-07 00:30:04 +000050 struct flow_cache *cache;
Timo Teräsd7997fe2010-03-31 00:17:06 +000051 atomic_t cpuleft;
52 struct completion completion;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053};
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
Timo Teräsd7997fe2010-03-31 00:17:06 +000055struct flow_cache {
56 u32 hash_shift;
Eric Dumazet83b6b1f2010-09-10 07:00:25 +000057 struct flow_cache_percpu __percpu *percpu;
Timo Teräsd7997fe2010-03-31 00:17:06 +000058 struct notifier_block hotcpu_notifier;
59 int low_watermark;
60 int high_watermark;
61 struct timer_list rnd_timer;
62};
63
64atomic_t flow_cache_genid = ATOMIC_INIT(0);
Eric Dumazet9e34a5b2010-07-09 21:22:04 +000065EXPORT_SYMBOL(flow_cache_genid);
Timo Teräsd7997fe2010-03-31 00:17:06 +000066static struct flow_cache flow_cache_global;
Eric Dumazet83b6b1f2010-09-10 07:00:25 +000067static struct kmem_cache *flow_cachep __read_mostly;
Timo Teräsd7997fe2010-03-31 00:17:06 +000068
Timo Teräs8e4795602010-04-07 00:30:07 +000069static DEFINE_SPINLOCK(flow_cache_gc_lock);
70static LIST_HEAD(flow_cache_gc_list);
71
Timo Teräsd7997fe2010-03-31 00:17:06 +000072#define flow_cache_hash_size(cache) (1 << (cache)->hash_shift)
73#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ)
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
75static void flow_cache_new_hashrnd(unsigned long arg)
76{
Timo Teräsd7997fe2010-03-31 00:17:06 +000077 struct flow_cache *fc = (void *) arg;
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 int i;
79
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -070080 for_each_possible_cpu(i)
Timo Teräsd7997fe2010-03-31 00:17:06 +000081 per_cpu_ptr(fc->percpu, i)->hash_rnd_recalc = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Timo Teräsd7997fe2010-03-31 00:17:06 +000083 fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD;
84 add_timer(&fc->rnd_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -070085}
86
Timo Teräsfe1a5f02010-04-07 00:30:04 +000087static int flow_entry_valid(struct flow_cache_entry *fle)
88{
89 if (atomic_read(&flow_cache_genid) != fle->genid)
90 return 0;
91 if (fle->object && !fle->object->ops->check(fle->object))
92 return 0;
93 return 1;
94}
95
Timo Teräs8e4795602010-04-07 00:30:07 +000096static void flow_entry_kill(struct flow_cache_entry *fle)
James Morris134b0fc2006-10-05 15:42:27 -050097{
98 if (fle->object)
Timo Teräsfe1a5f02010-04-07 00:30:04 +000099 fle->object->ops->delete(fle->object);
James Morris134b0fc2006-10-05 15:42:27 -0500100 kmem_cache_free(flow_cachep, fle);
Timo Teräs8e4795602010-04-07 00:30:07 +0000101}
102
103static void flow_cache_gc_task(struct work_struct *work)
104{
105 struct list_head gc_list;
106 struct flow_cache_entry *fce, *n;
107
108 INIT_LIST_HEAD(&gc_list);
109 spin_lock_bh(&flow_cache_gc_lock);
110 list_splice_tail_init(&flow_cache_gc_list, &gc_list);
111 spin_unlock_bh(&flow_cache_gc_lock);
112
113 list_for_each_entry_safe(fce, n, &gc_list, u.gc_list)
114 flow_entry_kill(fce);
115}
116static DECLARE_WORK(flow_cache_gc_work, flow_cache_gc_task);
117
118static void flow_cache_queue_garbage(struct flow_cache_percpu *fcp,
119 int deleted, struct list_head *gc_list)
120{
121 if (deleted) {
122 fcp->hash_count -= deleted;
123 spin_lock_bh(&flow_cache_gc_lock);
124 list_splice_tail(gc_list, &flow_cache_gc_list);
125 spin_unlock_bh(&flow_cache_gc_lock);
126 schedule_work(&flow_cache_gc_work);
127 }
James Morris134b0fc2006-10-05 15:42:27 -0500128}
129
Timo Teräsd7997fe2010-03-31 00:17:06 +0000130static void __flow_cache_shrink(struct flow_cache *fc,
131 struct flow_cache_percpu *fcp,
132 int shrink_to)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133{
Timo Teräs8e4795602010-04-07 00:30:07 +0000134 struct flow_cache_entry *fle;
135 struct hlist_node *entry, *tmp;
136 LIST_HEAD(gc_list);
137 int i, deleted = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Timo Teräsd7997fe2010-03-31 00:17:06 +0000139 for (i = 0; i < flow_cache_hash_size(fc); i++) {
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000140 int saved = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141
Timo Teräs8e4795602010-04-07 00:30:07 +0000142 hlist_for_each_entry_safe(fle, entry, tmp,
143 &fcp->hash_table[i], u.hlist) {
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000144 if (saved < shrink_to &&
145 flow_entry_valid(fle)) {
146 saved++;
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000147 } else {
Timo Teräs8e4795602010-04-07 00:30:07 +0000148 deleted++;
149 hlist_del(&fle->u.hlist);
150 list_add_tail(&fle->u.gc_list, &gc_list);
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000151 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 }
153 }
Timo Teräs8e4795602010-04-07 00:30:07 +0000154
155 flow_cache_queue_garbage(fcp, deleted, &gc_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156}
157
Timo Teräsd7997fe2010-03-31 00:17:06 +0000158static void flow_cache_shrink(struct flow_cache *fc,
159 struct flow_cache_percpu *fcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000161 int shrink_to = fc->low_watermark / flow_cache_hash_size(fc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162
Timo Teräsd7997fe2010-03-31 00:17:06 +0000163 __flow_cache_shrink(fc, fcp, shrink_to);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164}
165
Timo Teräsd7997fe2010-03-31 00:17:06 +0000166static void flow_new_hash_rnd(struct flow_cache *fc,
167 struct flow_cache_percpu *fcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000169 get_random_bytes(&fcp->hash_rnd, sizeof(u32));
170 fcp->hash_rnd_recalc = 0;
171 __flow_cache_shrink(fc, fcp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172}
173
Timo Teräsd7997fe2010-03-31 00:17:06 +0000174static u32 flow_hash_code(struct flow_cache *fc,
175 struct flow_cache_percpu *fcp,
dpwardaa1c3662011-09-05 16:47:24 +0000176 const struct flowi *key,
177 size_t keysize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178{
David S. Millerdee9f4b2011-02-22 18:44:31 -0800179 const u32 *k = (const u32 *) key;
dpwardaa1c3662011-09-05 16:47:24 +0000180 const u32 length = keysize * sizeof(flow_compare_t) / sizeof(u32);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
dpwardaa1c3662011-09-05 16:47:24 +0000182 return jhash2(k, length, fcp->hash_rnd)
Eric Dumazeta02cec22010-09-22 20:43:57 +0000183 & (flow_cache_hash_size(fc) - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184}
185
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186/* I hear what you're saying, use memcmp. But memcmp cannot make
dpwardaa1c3662011-09-05 16:47:24 +0000187 * important assumptions that we can here, such as alignment.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 */
dpwardaa1c3662011-09-05 16:47:24 +0000189static int flow_key_compare(const struct flowi *key1, const struct flowi *key2,
190 size_t keysize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191{
David S. Millerdee9f4b2011-02-22 18:44:31 -0800192 const flow_compare_t *k1, *k1_lim, *k2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
David S. Millerdee9f4b2011-02-22 18:44:31 -0800194 k1 = (const flow_compare_t *) key1;
dpwardaa1c3662011-09-05 16:47:24 +0000195 k1_lim = k1 + keysize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
David S. Millerdee9f4b2011-02-22 18:44:31 -0800197 k2 = (const flow_compare_t *) key2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198
199 do {
200 if (*k1++ != *k2++)
201 return 1;
202 } while (k1 < k1_lim);
203
204 return 0;
205}
206
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000207struct flow_cache_object *
David S. Millerdee9f4b2011-02-22 18:44:31 -0800208flow_cache_lookup(struct net *net, const struct flowi *key, u16 family, u8 dir,
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000209 flow_resolve_t resolver, void *ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000211 struct flow_cache *fc = &flow_cache_global;
212 struct flow_cache_percpu *fcp;
Timo Teräs8e4795602010-04-07 00:30:07 +0000213 struct flow_cache_entry *fle, *tfle;
214 struct hlist_node *entry;
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000215 struct flow_cache_object *flo;
dpwardaa1c3662011-09-05 16:47:24 +0000216 size_t keysize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
219 local_bh_disable();
Eric Dumazet7a9b2d52010-06-24 00:52:37 +0000220 fcp = this_cpu_ptr(fc->percpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221
222 fle = NULL;
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000223 flo = NULL;
dpwardaa1c3662011-09-05 16:47:24 +0000224
225 keysize = flow_key_size(family);
226 if (!keysize)
227 goto nocache;
228
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 /* Packet really early in init? Making flow_cache_init a
230 * pre-smp initcall would solve this. --RR */
Timo Teräsd7997fe2010-03-31 00:17:06 +0000231 if (!fcp->hash_table)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 goto nocache;
233
Timo Teräsd7997fe2010-03-31 00:17:06 +0000234 if (fcp->hash_rnd_recalc)
235 flow_new_hash_rnd(fc, fcp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236
dpwardaa1c3662011-09-05 16:47:24 +0000237 hash = flow_hash_code(fc, fcp, key, keysize);
Timo Teräs8e4795602010-04-07 00:30:07 +0000238 hlist_for_each_entry(tfle, entry, &fcp->hash_table[hash], u.hlist) {
dpward0542b692011-08-31 06:05:27 +0000239 if (tfle->net == net &&
240 tfle->family == family &&
Timo Teräs8e4795602010-04-07 00:30:07 +0000241 tfle->dir == dir &&
dpwardaa1c3662011-09-05 16:47:24 +0000242 flow_key_compare(key, &tfle->key, keysize) == 0) {
Timo Teräs8e4795602010-04-07 00:30:07 +0000243 fle = tfle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 break;
Timo Teräs8e4795602010-04-07 00:30:07 +0000245 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 }
247
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000248 if (unlikely(!fle)) {
Timo Teräsd7997fe2010-03-31 00:17:06 +0000249 if (fcp->hash_count > fc->high_watermark)
250 flow_cache_shrink(fc, fcp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251
Christoph Lameter54e6ecb2006-12-06 20:33:16 -0800252 fle = kmem_cache_alloc(flow_cachep, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253 if (fle) {
dpward0542b692011-08-31 06:05:27 +0000254 fle->net = net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 fle->family = family;
256 fle->dir = dir;
dpwardaa1c3662011-09-05 16:47:24 +0000257 memcpy(&fle->key, key, keysize * sizeof(flow_compare_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 fle->object = NULL;
Timo Teräs8e4795602010-04-07 00:30:07 +0000259 hlist_add_head(&fle->u.hlist, &fcp->hash_table[hash]);
Timo Teräsd7997fe2010-03-31 00:17:06 +0000260 fcp->hash_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 }
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000262 } else if (likely(fle->genid == atomic_read(&flow_cache_genid))) {
263 flo = fle->object;
264 if (!flo)
265 goto ret_object;
266 flo = flo->ops->get(flo);
267 if (flo)
268 goto ret_object;
269 } else if (fle->object) {
270 flo = fle->object;
271 flo->ops->delete(flo);
272 fle->object = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 }
274
275nocache:
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000276 flo = NULL;
277 if (fle) {
278 flo = fle->object;
279 fle->object = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 }
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000281 flo = resolver(net, key, family, dir, flo, ctx);
282 if (fle) {
283 fle->genid = atomic_read(&flow_cache_genid);
284 if (!IS_ERR(flo))
285 fle->object = flo;
286 else
287 fle->genid--;
288 } else {
289 if (flo && !IS_ERR(flo))
290 flo->ops->delete(flo);
291 }
292ret_object:
293 local_bh_enable();
294 return flo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295}
Eric Dumazet9e34a5b2010-07-09 21:22:04 +0000296EXPORT_SYMBOL(flow_cache_lookup);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297
298static void flow_cache_flush_tasklet(unsigned long data)
299{
300 struct flow_flush_info *info = (void *)data;
Timo Teräsd7997fe2010-03-31 00:17:06 +0000301 struct flow_cache *fc = info->cache;
302 struct flow_cache_percpu *fcp;
Timo Teräs8e4795602010-04-07 00:30:07 +0000303 struct flow_cache_entry *fle;
304 struct hlist_node *entry, *tmp;
305 LIST_HEAD(gc_list);
306 int i, deleted = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
Eric Dumazet7a9b2d52010-06-24 00:52:37 +0000308 fcp = this_cpu_ptr(fc->percpu);
Timo Teräsd7997fe2010-03-31 00:17:06 +0000309 for (i = 0; i < flow_cache_hash_size(fc); i++) {
Timo Teräs8e4795602010-04-07 00:30:07 +0000310 hlist_for_each_entry_safe(fle, entry, tmp,
311 &fcp->hash_table[i], u.hlist) {
Timo Teräsfe1a5f02010-04-07 00:30:04 +0000312 if (flow_entry_valid(fle))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313 continue;
314
Timo Teräs8e4795602010-04-07 00:30:07 +0000315 deleted++;
316 hlist_del(&fle->u.hlist);
317 list_add_tail(&fle->u.gc_list, &gc_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 }
319 }
320
Timo Teräs8e4795602010-04-07 00:30:07 +0000321 flow_cache_queue_garbage(fcp, deleted, &gc_list);
322
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 if (atomic_dec_and_test(&info->cpuleft))
324 complete(&info->completion);
325}
326
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327static void flow_cache_flush_per_cpu(void *data)
328{
329 struct flow_flush_info *info = data;
330 int cpu;
331 struct tasklet_struct *tasklet;
332
333 cpu = smp_processor_id();
Timo Teräsd7997fe2010-03-31 00:17:06 +0000334 tasklet = &per_cpu_ptr(info->cache->percpu, cpu)->flush_tasklet;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335 tasklet->data = (unsigned long)info;
336 tasklet_schedule(tasklet);
337}
338
339void flow_cache_flush(void)
340{
341 struct flow_flush_info info;
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -0800342 static DEFINE_MUTEX(flow_flush_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343
344 /* Don't want cpus going down or up during this. */
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100345 get_online_cpus();
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -0800346 mutex_lock(&flow_flush_sem);
Timo Teräsd7997fe2010-03-31 00:17:06 +0000347 info.cache = &flow_cache_global;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 atomic_set(&info.cpuleft, num_online_cpus());
349 init_completion(&info.completion);
350
351 local_bh_disable();
Jens Axboe8691e5a2008-06-06 11:18:06 +0200352 smp_call_function(flow_cache_flush_per_cpu, &info, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 flow_cache_flush_tasklet((unsigned long)&info);
354 local_bh_enable();
355
356 wait_for_completion(&info.completion);
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -0800357 mutex_unlock(&flow_flush_sem);
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100358 put_online_cpus();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359}
360
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000361static int __cpuinit flow_cache_cpu_prepare(struct flow_cache *fc, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362{
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000363 struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
364 size_t sz = sizeof(struct hlist_head) * flow_cache_hash_size(fc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000366 if (!fcp->hash_table) {
367 fcp->hash_table = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
368 if (!fcp->hash_table) {
369 pr_err("NET: failed to allocate flow cache sz %zu\n", sz);
370 return -ENOMEM;
371 }
372 fcp->hash_rnd_recalc = 1;
373 fcp->hash_count = 0;
374 tasklet_init(&fcp->flush_tasklet, flow_cache_flush_tasklet, 0);
375 }
376 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377}
378
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000379static int __cpuinit flow_cache_cpu(struct notifier_block *nfb,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 unsigned long action,
381 void *hcpu)
382{
Timo Teräsd7997fe2010-03-31 00:17:06 +0000383 struct flow_cache *fc = container_of(nfb, struct flow_cache, hotcpu_notifier);
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000384 int res, cpu = (unsigned long) hcpu;
Timo Teräsd7997fe2010-03-31 00:17:06 +0000385 struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
386
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000387 switch (action) {
388 case CPU_UP_PREPARE:
389 case CPU_UP_PREPARE_FROZEN:
390 res = flow_cache_cpu_prepare(fc, cpu);
391 if (res)
392 return notifier_from_errno(res);
393 break;
394 case CPU_DEAD:
395 case CPU_DEAD_FROZEN:
Timo Teräsd7997fe2010-03-31 00:17:06 +0000396 __flow_cache_shrink(fc, fcp, 0);
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000397 break;
398 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 return NOTIFY_OK;
400}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000402static int __init flow_cache_init(struct flow_cache *fc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403{
404 int i;
405
Timo Teräsd7997fe2010-03-31 00:17:06 +0000406 fc->hash_shift = 10;
407 fc->low_watermark = 2 * flow_cache_hash_size(fc);
408 fc->high_watermark = 4 * flow_cache_hash_size(fc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409
Timo Teräsd7997fe2010-03-31 00:17:06 +0000410 fc->percpu = alloc_percpu(struct flow_cache_percpu);
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000411 if (!fc->percpu)
412 return -ENOMEM;
413
414 for_each_online_cpu(i) {
415 if (flow_cache_cpu_prepare(fc, i))
huajun li6ccc3ab2011-09-27 22:51:39 +0000416 goto err;
Eric Dumazet83b6b1f2010-09-10 07:00:25 +0000417 }
418 fc->hotcpu_notifier = (struct notifier_block){
419 .notifier_call = flow_cache_cpu,
420 };
421 register_hotcpu_notifier(&fc->hotcpu_notifier);
Timo Teräsd7997fe2010-03-31 00:17:06 +0000422
423 setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd,
424 (unsigned long) fc);
425 fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD;
426 add_timer(&fc->rnd_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 return 0;
huajun li6ccc3ab2011-09-27 22:51:39 +0000429
430err:
431 for_each_possible_cpu(i) {
432 struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i);
433 kfree(fcp->hash_table);
434 fcp->hash_table = NULL;
435 }
436
437 free_percpu(fc->percpu);
438 fc->percpu = NULL;
439
440 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441}
442
Timo Teräsd7997fe2010-03-31 00:17:06 +0000443static int __init flow_cache_init_global(void)
444{
445 flow_cachep = kmem_cache_create("flow_cache",
446 sizeof(struct flow_cache_entry),
447 0, SLAB_PANIC, NULL);
448
449 return flow_cache_init(&flow_cache_global);
450}
451
452module_init(flow_cache_init_global);