[INET]: Generalise the TCP sock ID lookup routines
And also some TIME_WAIT functions.
[acme@toy net-2.6.14]$ grep built-in /tmp/before.size /tmp/after.size
/tmp/before.size: 282955 13122 9312 305389 4a8ed net/ipv4/built-in.o
/tmp/after.size: 281566 13122 9312 304000 4a380 net/ipv4/built-in.o
[acme@toy net-2.6.14]$
I kept them still inlined, will uninline at some point to see what
would be the performance difference.
Signed-off-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index c38c637..b5c0d64 100644
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -30,6 +30,7 @@
#include <net/tcp_states.h>
#include <asm/atomic.h>
+#include <asm/byteorder.h>
/* This is for all connections with a full identity, no wildcards.
* New scheme, half the table is for TIME_WAIT, the other half is
@@ -285,13 +286,13 @@
const int dif);
/* Optimize the common listener case. */
-static inline struct sock *inet_lookup_listener(struct inet_hashinfo *hashinfo,
- const u32 daddr,
- const unsigned short hnum,
- const int dif)
+static inline struct sock *
+ inet_lookup_listener(struct inet_hashinfo *hashinfo,
+ const u32 daddr,
+ const unsigned short hnum, const int dif)
{
struct sock *sk = NULL;
- struct hlist_head *head;
+ const struct hlist_head *head;
read_lock(&hashinfo->lhash_lock);
head = &hashinfo->listening_hash[inet_lhashfn(hnum)];
@@ -351,4 +352,70 @@
((*((__u32 *)&(inet_twsk(__sk)->tw_dport))) == (__ports)) && \
(!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
#endif /* 64-bit arch */
+
+/*
+ * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need
+ * not check it for lookups anymore, thanks Alexey. -DaveM
+ *
+ * Local BH must be disabled here.
+ */
+static inline struct sock *
+ __inet_lookup_established(struct inet_hashinfo *hashinfo,
+ const u32 saddr, const u16 sport,
+ const u32 daddr, const u16 hnum,
+ const int dif)
+{
+ INET_ADDR_COOKIE(acookie, saddr, daddr)
+ const __u32 ports = INET_COMBINED_PORTS(sport, hnum);
+ struct sock *sk;
+ const struct hlist_node *node;
+ /* Optimize here for direct hit, only listening connections can
+ * have wildcards anyways.
+ */
+ const int hash = inet_ehashfn(daddr, hnum, saddr, sport, hashinfo->ehash_size);
+ struct inet_ehash_bucket *head = &hashinfo->ehash[hash];
+
+ read_lock(&head->lock);
+ sk_for_each(sk, node, &head->chain) {
+ if (INET_MATCH(sk, acookie, saddr, daddr, ports, dif))
+ goto hit; /* You sunk my battleship! */
+ }
+
+ /* Must check for a TIME_WAIT'er before going to listener hash. */
+ sk_for_each(sk, node, &(head + hashinfo->ehash_size)->chain) {
+ if (INET_TW_MATCH(sk, acookie, saddr, daddr, ports, dif))
+ goto hit;
+ }
+ sk = NULL;
+out:
+ read_unlock(&head->lock);
+ return sk;
+hit:
+ sock_hold(sk);
+ goto out;
+}
+
+static inline struct sock *__inet_lookup(struct inet_hashinfo *hashinfo,
+ const u32 saddr, const u16 sport,
+ const u32 daddr, const u16 hnum,
+ const int dif)
+{
+ struct sock *sk = __inet_lookup_established(hashinfo, saddr, sport, daddr,
+ hnum, dif);
+ return sk ? : inet_lookup_listener(hashinfo, daddr, hnum, dif);
+}
+
+static inline struct sock *inet_lookup(struct inet_hashinfo *hashinfo,
+ const u32 saddr, const u16 sport,
+ const u32 daddr, const u16 dport,
+ const int dif)
+{
+ struct sock *sk;
+
+ local_bh_disable();
+ sk = __inet_lookup(hashinfo, saddr, sport, daddr, ntohs(dport), dif);
+ local_bh_enable();
+
+ return sk;
+}
#endif /* _INET_HASHTABLES_H */
diff --git a/include/net/inet_timewait_sock.h b/include/net/inet_timewait_sock.h
index ce11704..020f280 100644
--- a/include/net/inet_timewait_sock.h
+++ b/include/net/inet_timewait_sock.h
@@ -17,6 +17,7 @@
#include <linux/config.h>
+#include <linux/ip.h>
#include <linux/list.h>
#include <linux/types.h>
@@ -32,6 +33,7 @@
#endif
struct inet_bind_bucket;
+struct inet_hashinfo;
/*
* This is a TIME_WAIT sock. It works around the memory consumption
@@ -139,4 +141,11 @@
kmem_cache_free(tw->tw_prot->twsk_slab, tw);
}
}
+
+extern void __inet_twsk_kill(struct inet_timewait_sock *tw,
+ struct inet_hashinfo *hashinfo);
+
+extern void __inet_twsk_hashdance(struct inet_timewait_sock *tw,
+ struct sock *sk,
+ struct inet_hashinfo *hashinfo);
#endif /* _INET_TIMEWAIT_SOCK_ */
diff --git a/include/net/sock.h b/include/net/sock.h
index c902c57..bdae0a5 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -255,28 +255,28 @@
/*
* Hashed lists helper routines
*/
-static inline struct sock *__sk_head(struct hlist_head *head)
+static inline struct sock *__sk_head(const struct hlist_head *head)
{
return hlist_entry(head->first, struct sock, sk_node);
}
-static inline struct sock *sk_head(struct hlist_head *head)
+static inline struct sock *sk_head(const struct hlist_head *head)
{
return hlist_empty(head) ? NULL : __sk_head(head);
}
-static inline struct sock *sk_next(struct sock *sk)
+static inline struct sock *sk_next(const struct sock *sk)
{
return sk->sk_node.next ?
hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
}
-static inline int sk_unhashed(struct sock *sk)
+static inline int sk_unhashed(const struct sock *sk)
{
return hlist_unhashed(&sk->sk_node);
}
-static inline int sk_hashed(struct sock *sk)
+static inline int sk_hashed(const struct sock *sk)
{
return sk->sk_node.pprev != NULL;
}
@@ -494,7 +494,7 @@
struct request_sock_ops;
/* Here is the right place to enable sock refcounting debugging */
-#define SOCK_REFCNT_DEBUG
+//#define SOCK_REFCNT_DEBUG
/* Networking protocol blocks we attach to sockets.
* socket layer -> transport layer interface