ipv6: Create fast inline ipv6 neigh lookup just like ipv4.
Also, create and use an rt6_bind_neighbour() in net/ipv6/route.c to
consolidate some common logic.
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/include/net/ndisc.h b/include/net/ndisc.h
index e9c3002..e3133c2 100644
--- a/include/net/ndisc.h
+++ b/include/net/ndisc.h
@@ -89,6 +89,33 @@
(p32[3] * hash_rnd[3]));
}
+static inline struct neighbour *__ipv6_neigh_lookup(struct neigh_table *tbl, struct net_device *dev, const void *pkey)
+{
+ struct neigh_hash_table *nht;
+ const u32 *p32 = pkey;
+ struct neighbour *n;
+ u32 hash_val;
+
+ rcu_read_lock_bh();
+ nht = rcu_dereference_bh(tbl->nht);
+ hash_val = ndisc_hashfn(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
+ for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]);
+ n != NULL;
+ n = rcu_dereference_bh(n->next)) {
+ u32 *n32 = (u32 *) n->primary_key;
+ if (n->dev == dev &&
+ ((n32[0] ^ p32[0]) | (n32[1] ^ p32[1]) |
+ (n32[2] ^ p32[2]) | (n32[3] ^ p32[3])) == 0) {
+ if (!atomic_inc_not_zero(&n->refcnt))
+ n = NULL;
+ break;
+ }
+ }
+ rcu_read_unlock_bh();
+
+ return n;
+}
+
extern int ndisc_init(void);
extern void ndisc_cleanup(void);
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 35b07cc..6bf6094 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -123,7 +123,20 @@
static struct neighbour *ip6_neigh_lookup(const struct dst_entry *dst, const void *daddr)
{
- return __neigh_lookup_errno(&nd_tbl, daddr, dst->dev);
+ struct neighbour *n = __ipv6_neigh_lookup(&nd_tbl, dst->dev, daddr);
+ if (n)
+ return n;
+ return neigh_create(&nd_tbl, daddr, dst->dev);
+}
+
+static int rt6_bind_neighbour(struct rt6_info *rt)
+{
+ struct neighbour *n = ip6_neigh_lookup(&rt->dst, &rt->rt6i_gateway);
+ if (IS_ERR(n))
+ return PTR_ERR(n);
+ dst_set_neighbour(&rt->dst, n);
+
+ return 0;
}
static struct dst_ops ip6_dst_ops_template = {
@@ -714,7 +727,6 @@
rt = ip6_rt_copy(ort, daddr);
if (rt) {
- struct neighbour *neigh;
int attempts = !in_softirq();
if (!(rt->rt6i_flags & RTF_GATEWAY)) {
@@ -734,9 +746,7 @@
#endif
retry:
- neigh = __neigh_lookup_errno(&nd_tbl, &rt->rt6i_gateway,
- rt->rt6i_dev);
- if (IS_ERR(neigh)) {
+ if (rt6_bind_neighbour(rt)) {
struct net *net = dev_net(rt->rt6i_dev);
int saved_rt_min_interval =
net->ipv6.sysctl.ip6_rt_gc_min_interval;
@@ -762,8 +772,6 @@
dst_free(&rt->dst);
return NULL;
}
- dst_set_neighbour(&rt->dst, neigh);
-
}
return rt;
@@ -1078,7 +1086,7 @@
if (neigh)
neigh_hold(neigh);
else {
- neigh = __neigh_lookup_errno(&nd_tbl, &fl6->daddr, dev);
+ neigh = ip6_neigh_lookup(&rt->dst, &fl6->daddr);
if (IS_ERR(neigh)) {
dst_free(&rt->dst);
return ERR_CAST(neigh);
@@ -1389,12 +1397,9 @@
rt->rt6i_prefsrc.plen = 0;
if (cfg->fc_flags & (RTF_GATEWAY | RTF_NONEXTHOP)) {
- struct neighbour *n = __neigh_lookup_errno(&nd_tbl, &rt->rt6i_gateway, dev);
- if (IS_ERR(n)) {
- err = PTR_ERR(n);
+ err = rt6_bind_neighbour(rt);
+ if (err)
goto out;
- }
- dst_set_neighbour(&rt->dst, n);
}
rt->rt6i_flags = cfg->fc_flags;
@@ -2057,7 +2062,7 @@
struct net *net = dev_net(idev->dev);
struct rt6_info *rt = ip6_dst_alloc(&net->ipv6.ip6_dst_ops,
net->loopback_dev, 0);
- struct neighbour *neigh;
+ int err;
if (!rt) {
if (net_ratelimit())
@@ -2079,13 +2084,11 @@
rt->rt6i_flags |= RTF_ANYCAST;
else
rt->rt6i_flags |= RTF_LOCAL;
- neigh = __neigh_lookup_errno(&nd_tbl, &rt->rt6i_gateway, rt->rt6i_dev);
- if (IS_ERR(neigh)) {
+ err = rt6_bind_neighbour(rt);
+ if (err) {
dst_free(&rt->dst);
-
- return ERR_CAST(neigh);
+ return ERR_PTR(err);
}
- dst_set_neighbour(&rt->dst, neigh);
rt->rt6i_dst.addr = *addr;
rt->rt6i_dst.plen = 128;