blob: 278f607ab603ff90d243472c1efc0a1735742558 [file] [log] [blame]
Andy Grover00e0f342009-02-24 15:30:23 +00001/*
2 * Copyright (c) 2006 Oracle. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33#include <linux/kernel.h>
34#include <linux/list.h>
35#include <net/inet_hashtables.h>
36
37#include "rds.h"
38#include "loop.h"
39#include "rdma.h"
40
41#define RDS_CONNECTION_HASH_BITS 12
42#define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS)
43#define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1)
44
45/* converting this to RCU is a chore for another day.. */
46static DEFINE_SPINLOCK(rds_conn_lock);
47static unsigned long rds_conn_count;
48static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES];
49static struct kmem_cache *rds_conn_slab;
50
51static struct hlist_head *rds_conn_bucket(__be32 laddr, __be32 faddr)
52{
53 /* Pass NULL, don't need struct net for hash */
54 unsigned long hash = inet_ehashfn(NULL,
55 be32_to_cpu(laddr), 0,
56 be32_to_cpu(faddr), 0);
57 return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK];
58}
59
60#define rds_conn_info_set(var, test, suffix) do { \
61 if (test) \
62 var |= RDS_INFO_CONNECTION_FLAG_##suffix; \
63} while (0)
64
65static inline int rds_conn_is_sending(struct rds_connection *conn)
66{
67 int ret = 0;
68
69 if (!mutex_trylock(&conn->c_send_lock))
70 ret = 1;
71 else
72 mutex_unlock(&conn->c_send_lock);
73
74 return ret;
75}
76
77static struct rds_connection *rds_conn_lookup(struct hlist_head *head,
78 __be32 laddr, __be32 faddr,
79 struct rds_transport *trans)
80{
81 struct rds_connection *conn, *ret = NULL;
82 struct hlist_node *pos;
83
84 hlist_for_each_entry(conn, pos, head, c_hash_node) {
85 if (conn->c_faddr == faddr && conn->c_laddr == laddr &&
86 conn->c_trans == trans) {
87 ret = conn;
88 break;
89 }
90 }
91 rdsdebug("returning conn %p for %pI4 -> %pI4\n", ret,
92 &laddr, &faddr);
93 return ret;
94}
95
96/*
97 * This is called by transports as they're bringing down a connection.
98 * It clears partial message state so that the transport can start sending
99 * and receiving over this connection again in the future. It is up to
100 * the transport to have serialized this call with its send and recv.
101 */
102void rds_conn_reset(struct rds_connection *conn)
103{
104 rdsdebug("connection %pI4 to %pI4 reset\n",
105 &conn->c_laddr, &conn->c_faddr);
106
107 rds_stats_inc(s_conn_reset);
108 rds_send_reset(conn);
109 conn->c_flags = 0;
110
111 /* Do not clear next_rx_seq here, else we cannot distinguish
112 * retransmitted packets from new packets, and will hand all
113 * of them to the application. That is not consistent with the
114 * reliability guarantees of RDS. */
115}
116
117/*
118 * There is only every one 'conn' for a given pair of addresses in the
119 * system at a time. They contain messages to be retransmitted and so
120 * span the lifetime of the actual underlying transport connections.
121 *
122 * For now they are not garbage collected once they're created. They
123 * are torn down as the module is removed, if ever.
124 */
125static struct rds_connection *__rds_conn_create(__be32 laddr, __be32 faddr,
126 struct rds_transport *trans, gfp_t gfp,
127 int is_outgoing)
128{
Andy Grovercb244052009-07-17 13:13:36 +0000129 struct rds_connection *conn, *parent = NULL;
Andy Grover00e0f342009-02-24 15:30:23 +0000130 struct hlist_head *head = rds_conn_bucket(laddr, faddr);
131 unsigned long flags;
132 int ret;
133
134 spin_lock_irqsave(&rds_conn_lock, flags);
135 conn = rds_conn_lookup(head, laddr, faddr, trans);
Joe Perchesf64f9e72009-11-29 16:55:45 -0800136 if (conn && conn->c_loopback && conn->c_trans != &rds_loop_transport &&
137 !is_outgoing) {
Andy Grover00e0f342009-02-24 15:30:23 +0000138 /* This is a looped back IB connection, and we're
139 * called by the code handling the incoming connect.
140 * We need a second connection object into which we
141 * can stick the other QP. */
142 parent = conn;
143 conn = parent->c_passive;
144 }
145 spin_unlock_irqrestore(&rds_conn_lock, flags);
146 if (conn)
147 goto out;
148
Wei Yongjun05a178e2009-04-09 14:09:44 +0000149 conn = kmem_cache_zalloc(rds_conn_slab, gfp);
Andy Grover00e0f342009-02-24 15:30:23 +0000150 if (conn == NULL) {
151 conn = ERR_PTR(-ENOMEM);
152 goto out;
153 }
154
Andy Grover00e0f342009-02-24 15:30:23 +0000155 INIT_HLIST_NODE(&conn->c_hash_node);
Andy Grover00e0f342009-02-24 15:30:23 +0000156 conn->c_laddr = laddr;
157 conn->c_faddr = faddr;
158 spin_lock_init(&conn->c_lock);
159 conn->c_next_tx_seq = 1;
160
161 mutex_init(&conn->c_send_lock);
162 INIT_LIST_HEAD(&conn->c_send_queue);
163 INIT_LIST_HEAD(&conn->c_retrans);
164
165 ret = rds_cong_get_maps(conn);
166 if (ret) {
167 kmem_cache_free(rds_conn_slab, conn);
168 conn = ERR_PTR(ret);
169 goto out;
170 }
171
172 /*
173 * This is where a connection becomes loopback. If *any* RDS sockets
174 * can bind to the destination address then we'd rather the messages
175 * flow through loopback rather than either transport.
176 */
177 if (rds_trans_get_preferred(faddr)) {
178 conn->c_loopback = 1;
179 if (is_outgoing && trans->t_prefer_loopback) {
180 /* "outgoing" connection - and the transport
181 * says it wants the connection handled by the
182 * loopback transport. This is what TCP does.
183 */
184 trans = &rds_loop_transport;
185 }
186 }
187
188 conn->c_trans = trans;
189
190 ret = trans->conn_alloc(conn, gfp);
191 if (ret) {
192 kmem_cache_free(rds_conn_slab, conn);
193 conn = ERR_PTR(ret);
194 goto out;
195 }
196
197 atomic_set(&conn->c_state, RDS_CONN_DOWN);
198 conn->c_reconnect_jiffies = 0;
199 INIT_DELAYED_WORK(&conn->c_send_w, rds_send_worker);
200 INIT_DELAYED_WORK(&conn->c_recv_w, rds_recv_worker);
201 INIT_DELAYED_WORK(&conn->c_conn_w, rds_connect_worker);
202 INIT_WORK(&conn->c_down_w, rds_shutdown_worker);
203 mutex_init(&conn->c_cm_lock);
204 conn->c_flags = 0;
205
206 rdsdebug("allocated conn %p for %pI4 -> %pI4 over %s %s\n",
207 conn, &laddr, &faddr,
208 trans->t_name ? trans->t_name : "[unknown]",
209 is_outgoing ? "(outgoing)" : "");
210
Andy Grovercb244052009-07-17 13:13:36 +0000211 /*
212 * Since we ran without holding the conn lock, someone could
213 * have created the same conn (either normal or passive) in the
214 * interim. We check while holding the lock. If we won, we complete
215 * init and return our conn. If we lost, we rollback and return the
216 * other one.
217 */
Andy Grover00e0f342009-02-24 15:30:23 +0000218 spin_lock_irqsave(&rds_conn_lock, flags);
Andy Grovercb244052009-07-17 13:13:36 +0000219 if (parent) {
220 /* Creating passive conn */
221 if (parent->c_passive) {
222 trans->conn_free(conn->c_transport_data);
223 kmem_cache_free(rds_conn_slab, conn);
224 conn = parent->c_passive;
225 } else {
Andy Grover00e0f342009-02-24 15:30:23 +0000226 parent->c_passive = conn;
Andy Grovercb244052009-07-17 13:13:36 +0000227 rds_cong_add_conn(conn);
228 rds_conn_count++;
229 }
Andy Grover00e0f342009-02-24 15:30:23 +0000230 } else {
Andy Grovercb244052009-07-17 13:13:36 +0000231 /* Creating normal conn */
232 struct rds_connection *found;
Andy Grover00e0f342009-02-24 15:30:23 +0000233
Andy Grovercb244052009-07-17 13:13:36 +0000234 found = rds_conn_lookup(head, laddr, faddr, trans);
235 if (found) {
236 trans->conn_free(conn->c_transport_data);
237 kmem_cache_free(rds_conn_slab, conn);
238 conn = found;
239 } else {
240 hlist_add_head(&conn->c_hash_node, head);
241 rds_cong_add_conn(conn);
242 rds_conn_count++;
243 }
244 }
Andy Grover00e0f342009-02-24 15:30:23 +0000245 spin_unlock_irqrestore(&rds_conn_lock, flags);
246
247out:
248 return conn;
249}
250
251struct rds_connection *rds_conn_create(__be32 laddr, __be32 faddr,
252 struct rds_transport *trans, gfp_t gfp)
253{
254 return __rds_conn_create(laddr, faddr, trans, gfp, 0);
255}
Andy Grover616b7572009-08-21 12:28:32 +0000256EXPORT_SYMBOL_GPL(rds_conn_create);
Andy Grover00e0f342009-02-24 15:30:23 +0000257
258struct rds_connection *rds_conn_create_outgoing(__be32 laddr, __be32 faddr,
259 struct rds_transport *trans, gfp_t gfp)
260{
261 return __rds_conn_create(laddr, faddr, trans, gfp, 1);
262}
Andy Grover616b7572009-08-21 12:28:32 +0000263EXPORT_SYMBOL_GPL(rds_conn_create_outgoing);
Andy Grover00e0f342009-02-24 15:30:23 +0000264
265void rds_conn_destroy(struct rds_connection *conn)
266{
267 struct rds_message *rm, *rtmp;
268
269 rdsdebug("freeing conn %p for %pI4 -> "
270 "%pI4\n", conn, &conn->c_laddr,
271 &conn->c_faddr);
272
273 hlist_del_init(&conn->c_hash_node);
274
275 /* wait for the rds thread to shut it down */
276 atomic_set(&conn->c_state, RDS_CONN_ERROR);
277 cancel_delayed_work(&conn->c_conn_w);
278 queue_work(rds_wq, &conn->c_down_w);
279 flush_workqueue(rds_wq);
280
281 /* tear down queued messages */
282 list_for_each_entry_safe(rm, rtmp,
283 &conn->c_send_queue,
284 m_conn_item) {
285 list_del_init(&rm->m_conn_item);
286 BUG_ON(!list_empty(&rm->m_sock_item));
287 rds_message_put(rm);
288 }
289 if (conn->c_xmit_rm)
290 rds_message_put(conn->c_xmit_rm);
291
292 conn->c_trans->conn_free(conn->c_transport_data);
293
294 /*
295 * The congestion maps aren't freed up here. They're
296 * freed by rds_cong_exit() after all the connections
297 * have been freed.
298 */
299 rds_cong_remove_conn(conn);
300
301 BUG_ON(!list_empty(&conn->c_retrans));
302 kmem_cache_free(rds_conn_slab, conn);
303
304 rds_conn_count--;
305}
Andy Grover616b7572009-08-21 12:28:32 +0000306EXPORT_SYMBOL_GPL(rds_conn_destroy);
Andy Grover00e0f342009-02-24 15:30:23 +0000307
308static void rds_conn_message_info(struct socket *sock, unsigned int len,
309 struct rds_info_iterator *iter,
310 struct rds_info_lengths *lens,
311 int want_send)
312{
313 struct hlist_head *head;
314 struct hlist_node *pos;
315 struct list_head *list;
316 struct rds_connection *conn;
317 struct rds_message *rm;
318 unsigned long flags;
319 unsigned int total = 0;
320 size_t i;
321
322 len /= sizeof(struct rds_info_message);
323
324 spin_lock_irqsave(&rds_conn_lock, flags);
325
326 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
327 i++, head++) {
328 hlist_for_each_entry(conn, pos, head, c_hash_node) {
329 if (want_send)
330 list = &conn->c_send_queue;
331 else
332 list = &conn->c_retrans;
333
334 spin_lock(&conn->c_lock);
335
336 /* XXX too lazy to maintain counts.. */
337 list_for_each_entry(rm, list, m_conn_item) {
338 total++;
339 if (total <= len)
340 rds_inc_info_copy(&rm->m_inc, iter,
341 conn->c_laddr,
342 conn->c_faddr, 0);
343 }
344
345 spin_unlock(&conn->c_lock);
346 }
347 }
348
349 spin_unlock_irqrestore(&rds_conn_lock, flags);
350
351 lens->nr = total;
352 lens->each = sizeof(struct rds_info_message);
353}
354
355static void rds_conn_message_info_send(struct socket *sock, unsigned int len,
356 struct rds_info_iterator *iter,
357 struct rds_info_lengths *lens)
358{
359 rds_conn_message_info(sock, len, iter, lens, 1);
360}
361
362static void rds_conn_message_info_retrans(struct socket *sock,
363 unsigned int len,
364 struct rds_info_iterator *iter,
365 struct rds_info_lengths *lens)
366{
367 rds_conn_message_info(sock, len, iter, lens, 0);
368}
369
370void rds_for_each_conn_info(struct socket *sock, unsigned int len,
371 struct rds_info_iterator *iter,
372 struct rds_info_lengths *lens,
373 int (*visitor)(struct rds_connection *, void *),
374 size_t item_len)
375{
376 uint64_t buffer[(item_len + 7) / 8];
377 struct hlist_head *head;
378 struct hlist_node *pos;
379 struct hlist_node *tmp;
380 struct rds_connection *conn;
381 unsigned long flags;
382 size_t i;
383
384 spin_lock_irqsave(&rds_conn_lock, flags);
385
386 lens->nr = 0;
387 lens->each = item_len;
388
389 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
390 i++, head++) {
391 hlist_for_each_entry_safe(conn, pos, tmp, head, c_hash_node) {
392
393 /* XXX no c_lock usage.. */
394 if (!visitor(conn, buffer))
395 continue;
396
397 /* We copy as much as we can fit in the buffer,
398 * but we count all items so that the caller
399 * can resize the buffer. */
400 if (len >= item_len) {
401 rds_info_copy(iter, buffer, item_len);
402 len -= item_len;
403 }
404 lens->nr++;
405 }
406 }
407
408 spin_unlock_irqrestore(&rds_conn_lock, flags);
409}
Andy Grover616b7572009-08-21 12:28:32 +0000410EXPORT_SYMBOL_GPL(rds_for_each_conn_info);
Andy Grover00e0f342009-02-24 15:30:23 +0000411
412static int rds_conn_info_visitor(struct rds_connection *conn,
413 void *buffer)
414{
415 struct rds_info_connection *cinfo = buffer;
416
417 cinfo->next_tx_seq = conn->c_next_tx_seq;
418 cinfo->next_rx_seq = conn->c_next_rx_seq;
419 cinfo->laddr = conn->c_laddr;
420 cinfo->faddr = conn->c_faddr;
421 strncpy(cinfo->transport, conn->c_trans->t_name,
422 sizeof(cinfo->transport));
423 cinfo->flags = 0;
424
425 rds_conn_info_set(cinfo->flags,
426 rds_conn_is_sending(conn), SENDING);
427 /* XXX Future: return the state rather than these funky bits */
428 rds_conn_info_set(cinfo->flags,
429 atomic_read(&conn->c_state) == RDS_CONN_CONNECTING,
430 CONNECTING);
431 rds_conn_info_set(cinfo->flags,
432 atomic_read(&conn->c_state) == RDS_CONN_UP,
433 CONNECTED);
434 return 1;
435}
436
437static void rds_conn_info(struct socket *sock, unsigned int len,
438 struct rds_info_iterator *iter,
439 struct rds_info_lengths *lens)
440{
441 rds_for_each_conn_info(sock, len, iter, lens,
442 rds_conn_info_visitor,
443 sizeof(struct rds_info_connection));
444}
445
446int __init rds_conn_init(void)
447{
448 rds_conn_slab = kmem_cache_create("rds_connection",
449 sizeof(struct rds_connection),
450 0, 0, NULL);
451 if (rds_conn_slab == NULL)
452 return -ENOMEM;
453
454 rds_info_register_func(RDS_INFO_CONNECTIONS, rds_conn_info);
455 rds_info_register_func(RDS_INFO_SEND_MESSAGES,
456 rds_conn_message_info_send);
457 rds_info_register_func(RDS_INFO_RETRANS_MESSAGES,
458 rds_conn_message_info_retrans);
459
460 return 0;
461}
462
463void rds_conn_exit(void)
464{
465 rds_loop_exit();
466
467 WARN_ON(!hlist_empty(rds_conn_hash));
468
469 kmem_cache_destroy(rds_conn_slab);
470
471 rds_info_deregister_func(RDS_INFO_CONNECTIONS, rds_conn_info);
472 rds_info_deregister_func(RDS_INFO_SEND_MESSAGES,
473 rds_conn_message_info_send);
474 rds_info_deregister_func(RDS_INFO_RETRANS_MESSAGES,
475 rds_conn_message_info_retrans);
476}
477
478/*
479 * Force a disconnect
480 */
481void rds_conn_drop(struct rds_connection *conn)
482{
483 atomic_set(&conn->c_state, RDS_CONN_ERROR);
484 queue_work(rds_wq, &conn->c_down_w);
485}
Andy Grover616b7572009-08-21 12:28:32 +0000486EXPORT_SYMBOL_GPL(rds_conn_drop);
Andy Grover00e0f342009-02-24 15:30:23 +0000487
488/*
489 * An error occurred on the connection
490 */
491void
492__rds_conn_error(struct rds_connection *conn, const char *fmt, ...)
493{
494 va_list ap;
495
496 va_start(ap, fmt);
497 vprintk(fmt, ap);
498 va_end(ap);
499
500 rds_conn_drop(conn);
501}