Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* XDP sockets |
| 3 | * |
| 4 | * AF_XDP sockets allows a channel between XDP programs and userspace |
| 5 | * applications. |
| 6 | * Copyright(c) 2018 Intel Corporation. |
| 7 | * |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 8 | * Author(s): Björn Töpel <bjorn.topel@intel.com> |
| 9 | * Magnus Karlsson <magnus.karlsson@intel.com> |
| 10 | */ |
| 11 | |
| 12 | #define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__ |
| 13 | |
| 14 | #include <linux/if_xdp.h> |
| 15 | #include <linux/init.h> |
| 16 | #include <linux/sched/mm.h> |
| 17 | #include <linux/sched/signal.h> |
| 18 | #include <linux/sched/task.h> |
| 19 | #include <linux/socket.h> |
| 20 | #include <linux/file.h> |
| 21 | #include <linux/uaccess.h> |
| 22 | #include <linux/net.h> |
| 23 | #include <linux/netdevice.h> |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 24 | #include <linux/rculist.h> |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 25 | #include <net/xdp_sock.h> |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 26 | #include <net/xdp.h> |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 27 | |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 28 | #include "xsk_queue.h" |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 29 | #include "xdp_umem.h" |
| 30 | |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 31 | #define TX_BATCH_SIZE 16 |
| 32 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 33 | static struct xdp_sock *xdp_sk(struct sock *sk) |
| 34 | { |
| 35 | return (struct xdp_sock *)sk; |
| 36 | } |
| 37 | |
Björn Töpel | fbfc504 | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 38 | bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) |
| 39 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 40 | return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) && |
| 41 | READ_ONCE(xs->umem->fq); |
Björn Töpel | fbfc504 | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 42 | } |
| 43 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 44 | u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr) |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 45 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 46 | return xskq_peek_addr(umem->fq, addr); |
| 47 | } |
| 48 | EXPORT_SYMBOL(xsk_umem_peek_addr); |
| 49 | |
| 50 | void xsk_umem_discard_addr(struct xdp_umem *umem) |
| 51 | { |
| 52 | xskq_discard_addr(umem->fq); |
| 53 | } |
| 54 | EXPORT_SYMBOL(xsk_umem_discard_addr); |
| 55 | |
| 56 | static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) |
| 57 | { |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 58 | void *buffer; |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 59 | u64 addr; |
Björn Töpel | 4e64c83 | 2018-06-04 13:57:11 +0200 | [diff] [blame] | 60 | int err; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 61 | |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 62 | if (!xskq_peek_addr(xs->umem->fq, &addr) || |
| 63 | len > xs->umem->chunk_size_nohr) { |
Björn Töpel | a509a95 | 2018-06-04 13:57:12 +0200 | [diff] [blame] | 64 | xs->rx_dropped++; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 65 | return -ENOSPC; |
Björn Töpel | a509a95 | 2018-06-04 13:57:12 +0200 | [diff] [blame] | 66 | } |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 67 | |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 68 | addr += xs->umem->headroom; |
| 69 | |
| 70 | buffer = xdp_umem_get_data(xs->umem, addr); |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 71 | memcpy(buffer, xdp->data, len); |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 72 | err = xskq_produce_batch_desc(xs->rx, addr, len); |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 73 | if (!err) { |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 74 | xskq_discard_addr(xs->umem->fq); |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 75 | xdp_return_buff(xdp); |
| 76 | return 0; |
| 77 | } |
| 78 | |
| 79 | xs->rx_dropped++; |
| 80 | return err; |
| 81 | } |
| 82 | |
| 83 | static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) |
| 84 | { |
| 85 | int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len); |
| 86 | |
| 87 | if (err) { |
| 88 | xdp_return_buff(xdp); |
Björn Töpel | a509a95 | 2018-06-04 13:57:12 +0200 | [diff] [blame] | 89 | xs->rx_dropped++; |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 90 | } |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 91 | |
| 92 | return err; |
| 93 | } |
| 94 | |
| 95 | int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) |
| 96 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 97 | u32 len; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 98 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 99 | if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) |
| 100 | return -EINVAL; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 101 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 102 | len = xdp->data_end - xdp->data; |
| 103 | |
| 104 | return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ? |
| 105 | __xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len); |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 106 | } |
| 107 | |
| 108 | void xsk_flush(struct xdp_sock *xs) |
| 109 | { |
| 110 | xskq_produce_flush_desc(xs->rx); |
| 111 | xs->sk.sk_data_ready(&xs->sk); |
| 112 | } |
| 113 | |
| 114 | int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) |
| 115 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 116 | u32 len = xdp->data_end - xdp->data; |
| 117 | void *buffer; |
| 118 | u64 addr; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 119 | int err; |
| 120 | |
Björn Töpel | 5d90237 | 2018-06-12 12:02:56 +0200 | [diff] [blame] | 121 | if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) |
| 122 | return -EINVAL; |
| 123 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 124 | if (!xskq_peek_addr(xs->umem->fq, &addr) || |
| 125 | len > xs->umem->chunk_size_nohr) { |
| 126 | xs->rx_dropped++; |
| 127 | return -ENOSPC; |
| 128 | } |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 129 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 130 | addr += xs->umem->headroom; |
| 131 | |
| 132 | buffer = xdp_umem_get_data(xs->umem, addr); |
| 133 | memcpy(buffer, xdp->data, len); |
| 134 | err = xskq_produce_batch_desc(xs->rx, addr, len); |
| 135 | if (!err) { |
| 136 | xskq_discard_addr(xs->umem->fq); |
| 137 | xsk_flush(xs); |
| 138 | return 0; |
| 139 | } |
| 140 | |
| 141 | xs->rx_dropped++; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 142 | return err; |
| 143 | } |
| 144 | |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 145 | void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries) |
| 146 | { |
| 147 | xskq_produce_flush_addr_n(umem->cq, nb_entries); |
| 148 | } |
| 149 | EXPORT_SYMBOL(xsk_umem_complete_tx); |
| 150 | |
| 151 | void xsk_umem_consume_tx_done(struct xdp_umem *umem) |
| 152 | { |
| 153 | struct xdp_sock *xs; |
| 154 | |
| 155 | rcu_read_lock(); |
| 156 | list_for_each_entry_rcu(xs, &umem->xsk_list, list) { |
| 157 | xs->sk.sk_write_space(&xs->sk); |
| 158 | } |
| 159 | rcu_read_unlock(); |
| 160 | } |
| 161 | EXPORT_SYMBOL(xsk_umem_consume_tx_done); |
| 162 | |
| 163 | bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len) |
| 164 | { |
| 165 | struct xdp_desc desc; |
| 166 | struct xdp_sock *xs; |
| 167 | |
| 168 | rcu_read_lock(); |
| 169 | list_for_each_entry_rcu(xs, &umem->xsk_list, list) { |
| 170 | if (!xskq_peek_desc(xs->tx, &desc)) |
| 171 | continue; |
| 172 | |
| 173 | if (xskq_produce_addr_lazy(umem->cq, desc.addr)) |
| 174 | goto out; |
| 175 | |
| 176 | *dma = xdp_umem_get_dma(umem, desc.addr); |
| 177 | *len = desc.len; |
| 178 | |
| 179 | xskq_discard_desc(xs->tx); |
| 180 | rcu_read_unlock(); |
| 181 | return true; |
| 182 | } |
| 183 | |
| 184 | out: |
| 185 | rcu_read_unlock(); |
| 186 | return false; |
| 187 | } |
| 188 | EXPORT_SYMBOL(xsk_umem_consume_tx); |
| 189 | |
| 190 | static int xsk_zc_xmit(struct sock *sk) |
| 191 | { |
| 192 | struct xdp_sock *xs = xdp_sk(sk); |
| 193 | struct net_device *dev = xs->dev; |
| 194 | |
| 195 | return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id); |
| 196 | } |
| 197 | |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 198 | static void xsk_destruct_skb(struct sk_buff *skb) |
| 199 | { |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 200 | u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 201 | struct xdp_sock *xs = xdp_sk(skb->sk); |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 202 | unsigned long flags; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 203 | |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 204 | spin_lock_irqsave(&xs->tx_completion_lock, flags); |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 205 | WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr)); |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 206 | spin_unlock_irqrestore(&xs->tx_completion_lock, flags); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 207 | |
| 208 | sock_wfree(skb); |
| 209 | } |
| 210 | |
| 211 | static int xsk_generic_xmit(struct sock *sk, struct msghdr *m, |
| 212 | size_t total_len) |
| 213 | { |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 214 | u32 max_batch = TX_BATCH_SIZE; |
| 215 | struct xdp_sock *xs = xdp_sk(sk); |
| 216 | bool sent_frame = false; |
| 217 | struct xdp_desc desc; |
| 218 | struct sk_buff *skb; |
| 219 | int err = 0; |
| 220 | |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 221 | mutex_lock(&xs->mutex); |
| 222 | |
| 223 | while (xskq_peek_desc(xs->tx, &desc)) { |
| 224 | char *buffer; |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 225 | u64 addr; |
| 226 | u32 len; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 227 | |
| 228 | if (max_batch-- == 0) { |
| 229 | err = -EAGAIN; |
| 230 | goto out; |
| 231 | } |
| 232 | |
Magnus Karlsson | 9684f5e | 2018-07-11 10:12:50 +0200 | [diff] [blame] | 233 | if (xskq_reserve_addr(xs->umem->cq)) |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 234 | goto out; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 235 | |
| 236 | len = desc.len; |
| 237 | if (unlikely(len > xs->dev->mtu)) { |
| 238 | err = -EMSGSIZE; |
| 239 | goto out; |
| 240 | } |
| 241 | |
Magnus Karlsson | 509d764 | 2018-07-11 10:12:49 +0200 | [diff] [blame] | 242 | if (xs->queue_id >= xs->dev->real_num_tx_queues) |
Magnus Karlsson | 2e59dd5 | 2018-05-22 09:34:58 +0200 | [diff] [blame] | 243 | goto out; |
Magnus Karlsson | 2e59dd5 | 2018-05-22 09:34:58 +0200 | [diff] [blame] | 244 | |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 245 | skb = sock_alloc_send_skb(sk, len, 1, &err); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 246 | if (unlikely(!skb)) { |
| 247 | err = -EAGAIN; |
| 248 | goto out; |
| 249 | } |
| 250 | |
| 251 | skb_put(skb, len); |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 252 | addr = desc.addr; |
| 253 | buffer = xdp_umem_get_data(xs->umem, addr); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 254 | err = skb_store_bits(skb, 0, buffer, len); |
| 255 | if (unlikely(err)) { |
| 256 | kfree_skb(skb); |
| 257 | goto out; |
| 258 | } |
| 259 | |
| 260 | skb->dev = xs->dev; |
| 261 | skb->priority = sk->sk_priority; |
| 262 | skb->mark = sk->sk_mark; |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 263 | skb_shinfo(skb)->destructor_arg = (void *)(long)addr; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 264 | skb->destructor = xsk_destruct_skb; |
| 265 | |
| 266 | err = dev_direct_xmit(skb, xs->queue_id); |
Magnus Karlsson | fe58868 | 2018-06-29 09:48:18 +0200 | [diff] [blame] | 267 | xskq_discard_desc(xs->tx); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 268 | /* Ignore NET_XMIT_CN as packet might have been sent */ |
| 269 | if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { |
Magnus Karlsson | fe58868 | 2018-06-29 09:48:18 +0200 | [diff] [blame] | 270 | /* SKB completed but not sent */ |
| 271 | err = -EBUSY; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 272 | goto out; |
| 273 | } |
| 274 | |
| 275 | sent_frame = true; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 276 | } |
| 277 | |
| 278 | out: |
| 279 | if (sent_frame) |
| 280 | sk->sk_write_space(sk); |
| 281 | |
| 282 | mutex_unlock(&xs->mutex); |
| 283 | return err; |
| 284 | } |
| 285 | |
| 286 | static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) |
| 287 | { |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 288 | bool need_wait = !(m->msg_flags & MSG_DONTWAIT); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 289 | struct sock *sk = sock->sk; |
| 290 | struct xdp_sock *xs = xdp_sk(sk); |
| 291 | |
| 292 | if (unlikely(!xs->dev)) |
| 293 | return -ENXIO; |
| 294 | if (unlikely(!(xs->dev->flags & IFF_UP))) |
| 295 | return -ENETDOWN; |
Magnus Karlsson | 6efb443 | 2018-07-11 10:12:51 +0200 | [diff] [blame^] | 296 | if (unlikely(!xs->tx)) |
| 297 | return -ENOBUFS; |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 298 | if (need_wait) |
| 299 | return -EOPNOTSUPP; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 300 | |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 301 | return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 302 | } |
| 303 | |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 304 | static unsigned int xsk_poll(struct file *file, struct socket *sock, |
| 305 | struct poll_table_struct *wait) |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 306 | { |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 307 | unsigned int mask = datagram_poll(file, sock, wait); |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 308 | struct sock *sk = sock->sk; |
| 309 | struct xdp_sock *xs = xdp_sk(sk); |
| 310 | |
| 311 | if (xs->rx && !xskq_empty_desc(xs->rx)) |
| 312 | mask |= POLLIN | POLLRDNORM; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 313 | if (xs->tx && !xskq_full_desc(xs->tx)) |
| 314 | mask |= POLLOUT | POLLWRNORM; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 315 | |
| 316 | return mask; |
| 317 | } |
| 318 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 319 | static int xsk_init_queue(u32 entries, struct xsk_queue **queue, |
| 320 | bool umem_queue) |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 321 | { |
| 322 | struct xsk_queue *q; |
| 323 | |
| 324 | if (entries == 0 || *queue || !is_power_of_2(entries)) |
| 325 | return -EINVAL; |
| 326 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 327 | q = xskq_create(entries, umem_queue); |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 328 | if (!q) |
| 329 | return -ENOMEM; |
| 330 | |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 331 | /* Make sure queue is ready before it can be seen by others */ |
| 332 | smp_wmb(); |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 333 | *queue = q; |
| 334 | return 0; |
| 335 | } |
| 336 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 337 | static int xsk_release(struct socket *sock) |
| 338 | { |
| 339 | struct sock *sk = sock->sk; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 340 | struct xdp_sock *xs = xdp_sk(sk); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 341 | struct net *net; |
| 342 | |
| 343 | if (!sk) |
| 344 | return 0; |
| 345 | |
| 346 | net = sock_net(sk); |
| 347 | |
| 348 | local_bh_disable(); |
| 349 | sock_prot_inuse_add(net, sk->sk_prot, -1); |
| 350 | local_bh_enable(); |
| 351 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 352 | if (xs->dev) { |
Björn Töpel | 959b71d | 2018-05-22 09:34:56 +0200 | [diff] [blame] | 353 | /* Wait for driver to stop using the xdp socket. */ |
| 354 | synchronize_net(); |
| 355 | dev_put(xs->dev); |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 356 | xs->dev = NULL; |
| 357 | } |
| 358 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 359 | sock_orphan(sk); |
| 360 | sock->sk = NULL; |
| 361 | |
| 362 | sk_refcnt_debug_release(sk); |
| 363 | sock_put(sk); |
| 364 | |
| 365 | return 0; |
| 366 | } |
| 367 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 368 | static struct socket *xsk_lookup_xsk_from_fd(int fd) |
| 369 | { |
| 370 | struct socket *sock; |
| 371 | int err; |
| 372 | |
| 373 | sock = sockfd_lookup(fd, &err); |
| 374 | if (!sock) |
| 375 | return ERR_PTR(-ENOTSOCK); |
| 376 | |
| 377 | if (sock->sk->sk_family != PF_XDP) { |
| 378 | sockfd_put(sock); |
| 379 | return ERR_PTR(-ENOPROTOOPT); |
| 380 | } |
| 381 | |
| 382 | return sock; |
| 383 | } |
| 384 | |
| 385 | static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) |
| 386 | { |
| 387 | struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr; |
| 388 | struct sock *sk = sock->sk; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 389 | struct xdp_sock *xs = xdp_sk(sk); |
Björn Töpel | 959b71d | 2018-05-22 09:34:56 +0200 | [diff] [blame] | 390 | struct net_device *dev; |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 391 | u32 flags, qid; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 392 | int err = 0; |
| 393 | |
| 394 | if (addr_len < sizeof(struct sockaddr_xdp)) |
| 395 | return -EINVAL; |
| 396 | if (sxdp->sxdp_family != AF_XDP) |
| 397 | return -EINVAL; |
| 398 | |
| 399 | mutex_lock(&xs->mutex); |
Björn Töpel | 959b71d | 2018-05-22 09:34:56 +0200 | [diff] [blame] | 400 | if (xs->dev) { |
| 401 | err = -EBUSY; |
| 402 | goto out_release; |
| 403 | } |
| 404 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 405 | dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex); |
| 406 | if (!dev) { |
| 407 | err = -ENODEV; |
| 408 | goto out_release; |
| 409 | } |
| 410 | |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 411 | if (!xs->rx && !xs->tx) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 412 | err = -EINVAL; |
| 413 | goto out_unlock; |
| 414 | } |
| 415 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 416 | qid = sxdp->sxdp_queue_id; |
| 417 | |
| 418 | if ((xs->rx && qid >= dev->real_num_rx_queues) || |
| 419 | (xs->tx && qid >= dev->real_num_tx_queues)) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 420 | err = -EINVAL; |
| 421 | goto out_unlock; |
| 422 | } |
| 423 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 424 | flags = sxdp->sxdp_flags; |
| 425 | |
| 426 | if (flags & XDP_SHARED_UMEM) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 427 | struct xdp_sock *umem_xs; |
| 428 | struct socket *sock; |
| 429 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 430 | if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY)) { |
| 431 | /* Cannot specify flags for shared sockets. */ |
| 432 | err = -EINVAL; |
| 433 | goto out_unlock; |
| 434 | } |
| 435 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 436 | if (xs->umem) { |
| 437 | /* We have already our own. */ |
| 438 | err = -EINVAL; |
| 439 | goto out_unlock; |
| 440 | } |
| 441 | |
| 442 | sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd); |
| 443 | if (IS_ERR(sock)) { |
| 444 | err = PTR_ERR(sock); |
| 445 | goto out_unlock; |
| 446 | } |
| 447 | |
| 448 | umem_xs = xdp_sk(sock->sk); |
| 449 | if (!umem_xs->umem) { |
| 450 | /* No umem to inherit. */ |
| 451 | err = -EBADF; |
| 452 | sockfd_put(sock); |
| 453 | goto out_unlock; |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 454 | } else if (umem_xs->dev != dev || umem_xs->queue_id != qid) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 455 | err = -EINVAL; |
| 456 | sockfd_put(sock); |
| 457 | goto out_unlock; |
| 458 | } |
| 459 | |
| 460 | xdp_get_umem(umem_xs->umem); |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 461 | xs->umem = umem_xs->umem; |
| 462 | sockfd_put(sock); |
| 463 | } else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) { |
| 464 | err = -EINVAL; |
| 465 | goto out_unlock; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 466 | } else { |
| 467 | /* This xsk has its own umem. */ |
| 468 | xskq_set_umem(xs->umem->fq, &xs->umem->props); |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 469 | xskq_set_umem(xs->umem->cq, &xs->umem->props); |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 470 | |
| 471 | err = xdp_umem_assign_dev(xs->umem, dev, qid, flags); |
| 472 | if (err) |
| 473 | goto out_unlock; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 474 | } |
| 475 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 476 | xs->dev = dev; |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 477 | xs->zc = xs->umem->zc; |
| 478 | xs->queue_id = qid; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 479 | xskq_set_umem(xs->rx, &xs->umem->props); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 480 | xskq_set_umem(xs->tx, &xs->umem->props); |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 481 | xdp_add_sk_umem(xs->umem, xs); |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 482 | |
| 483 | out_unlock: |
| 484 | if (err) |
| 485 | dev_put(dev); |
| 486 | out_release: |
| 487 | mutex_unlock(&xs->mutex); |
| 488 | return err; |
| 489 | } |
| 490 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 491 | static int xsk_setsockopt(struct socket *sock, int level, int optname, |
| 492 | char __user *optval, unsigned int optlen) |
| 493 | { |
| 494 | struct sock *sk = sock->sk; |
| 495 | struct xdp_sock *xs = xdp_sk(sk); |
| 496 | int err; |
| 497 | |
| 498 | if (level != SOL_XDP) |
| 499 | return -ENOPROTOOPT; |
| 500 | |
| 501 | switch (optname) { |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 502 | case XDP_RX_RING: |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 503 | case XDP_TX_RING: |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 504 | { |
| 505 | struct xsk_queue **q; |
| 506 | int entries; |
| 507 | |
| 508 | if (optlen < sizeof(entries)) |
| 509 | return -EINVAL; |
| 510 | if (copy_from_user(&entries, optval, sizeof(entries))) |
| 511 | return -EFAULT; |
| 512 | |
| 513 | mutex_lock(&xs->mutex); |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 514 | q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx; |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 515 | err = xsk_init_queue(entries, q, false); |
| 516 | mutex_unlock(&xs->mutex); |
| 517 | return err; |
| 518 | } |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 519 | case XDP_UMEM_REG: |
| 520 | { |
| 521 | struct xdp_umem_reg mr; |
| 522 | struct xdp_umem *umem; |
| 523 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 524 | if (copy_from_user(&mr, optval, sizeof(mr))) |
| 525 | return -EFAULT; |
| 526 | |
| 527 | mutex_lock(&xs->mutex); |
Björn Töpel | a49049e | 2018-05-22 09:35:02 +0200 | [diff] [blame] | 528 | if (xs->umem) { |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 529 | mutex_unlock(&xs->mutex); |
Björn Töpel | a49049e | 2018-05-22 09:35:02 +0200 | [diff] [blame] | 530 | return -EBUSY; |
| 531 | } |
| 532 | |
| 533 | umem = xdp_umem_create(&mr); |
| 534 | if (IS_ERR(umem)) { |
| 535 | mutex_unlock(&xs->mutex); |
| 536 | return PTR_ERR(umem); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 537 | } |
| 538 | |
| 539 | /* Make sure umem is ready before it can be seen by others */ |
| 540 | smp_wmb(); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 541 | xs->umem = umem; |
| 542 | mutex_unlock(&xs->mutex); |
| 543 | return 0; |
| 544 | } |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 545 | case XDP_UMEM_FILL_RING: |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 546 | case XDP_UMEM_COMPLETION_RING: |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 547 | { |
| 548 | struct xsk_queue **q; |
| 549 | int entries; |
| 550 | |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 551 | if (copy_from_user(&entries, optval, sizeof(entries))) |
| 552 | return -EFAULT; |
| 553 | |
| 554 | mutex_lock(&xs->mutex); |
Björn Töpel | a49049e | 2018-05-22 09:35:02 +0200 | [diff] [blame] | 555 | if (!xs->umem) { |
| 556 | mutex_unlock(&xs->mutex); |
| 557 | return -EINVAL; |
| 558 | } |
| 559 | |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 560 | q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq : |
| 561 | &xs->umem->cq; |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 562 | err = xsk_init_queue(entries, q, true); |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 563 | mutex_unlock(&xs->mutex); |
| 564 | return err; |
| 565 | } |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 566 | default: |
| 567 | break; |
| 568 | } |
| 569 | |
| 570 | return -ENOPROTOOPT; |
| 571 | } |
| 572 | |
Magnus Karlsson | af75d9e | 2018-05-02 13:01:35 +0200 | [diff] [blame] | 573 | static int xsk_getsockopt(struct socket *sock, int level, int optname, |
| 574 | char __user *optval, int __user *optlen) |
| 575 | { |
| 576 | struct sock *sk = sock->sk; |
| 577 | struct xdp_sock *xs = xdp_sk(sk); |
| 578 | int len; |
| 579 | |
| 580 | if (level != SOL_XDP) |
| 581 | return -ENOPROTOOPT; |
| 582 | |
| 583 | if (get_user(len, optlen)) |
| 584 | return -EFAULT; |
| 585 | if (len < 0) |
| 586 | return -EINVAL; |
| 587 | |
| 588 | switch (optname) { |
| 589 | case XDP_STATISTICS: |
| 590 | { |
| 591 | struct xdp_statistics stats; |
| 592 | |
| 593 | if (len < sizeof(stats)) |
| 594 | return -EINVAL; |
| 595 | |
| 596 | mutex_lock(&xs->mutex); |
| 597 | stats.rx_dropped = xs->rx_dropped; |
| 598 | stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx); |
| 599 | stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx); |
| 600 | mutex_unlock(&xs->mutex); |
| 601 | |
| 602 | if (copy_to_user(optval, &stats, sizeof(stats))) |
| 603 | return -EFAULT; |
| 604 | if (put_user(sizeof(stats), optlen)) |
| 605 | return -EFAULT; |
| 606 | |
| 607 | return 0; |
| 608 | } |
Björn Töpel | b3a9e0b | 2018-05-22 09:34:59 +0200 | [diff] [blame] | 609 | case XDP_MMAP_OFFSETS: |
| 610 | { |
| 611 | struct xdp_mmap_offsets off; |
| 612 | |
| 613 | if (len < sizeof(off)) |
| 614 | return -EINVAL; |
| 615 | |
| 616 | off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer); |
| 617 | off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer); |
| 618 | off.rx.desc = offsetof(struct xdp_rxtx_ring, desc); |
| 619 | off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer); |
| 620 | off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer); |
| 621 | off.tx.desc = offsetof(struct xdp_rxtx_ring, desc); |
| 622 | |
| 623 | off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer); |
| 624 | off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer); |
| 625 | off.fr.desc = offsetof(struct xdp_umem_ring, desc); |
| 626 | off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer); |
| 627 | off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer); |
| 628 | off.cr.desc = offsetof(struct xdp_umem_ring, desc); |
| 629 | |
| 630 | len = sizeof(off); |
| 631 | if (copy_to_user(optval, &off, len)) |
| 632 | return -EFAULT; |
| 633 | if (put_user(len, optlen)) |
| 634 | return -EFAULT; |
| 635 | |
| 636 | return 0; |
| 637 | } |
Magnus Karlsson | af75d9e | 2018-05-02 13:01:35 +0200 | [diff] [blame] | 638 | default: |
| 639 | break; |
| 640 | } |
| 641 | |
| 642 | return -EOPNOTSUPP; |
| 643 | } |
| 644 | |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 645 | static int xsk_mmap(struct file *file, struct socket *sock, |
| 646 | struct vm_area_struct *vma) |
| 647 | { |
Geert Uytterhoeven | a5a16e4 | 2018-06-07 15:37:34 +0200 | [diff] [blame] | 648 | loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 649 | unsigned long size = vma->vm_end - vma->vm_start; |
| 650 | struct xdp_sock *xs = xdp_sk(sock->sk); |
| 651 | struct xsk_queue *q = NULL; |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 652 | struct xdp_umem *umem; |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 653 | unsigned long pfn; |
| 654 | struct page *qpg; |
| 655 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 656 | if (offset == XDP_PGOFF_RX_RING) { |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 657 | q = READ_ONCE(xs->rx); |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 658 | } else if (offset == XDP_PGOFF_TX_RING) { |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 659 | q = READ_ONCE(xs->tx); |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 660 | } else { |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 661 | umem = READ_ONCE(xs->umem); |
| 662 | if (!umem) |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 663 | return -EINVAL; |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 664 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 665 | if (offset == XDP_UMEM_PGOFF_FILL_RING) |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 666 | q = READ_ONCE(umem->fq); |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 667 | else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING) |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 668 | q = READ_ONCE(umem->cq); |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 669 | } |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 670 | |
| 671 | if (!q) |
| 672 | return -EINVAL; |
| 673 | |
| 674 | qpg = virt_to_head_page(q->ring); |
| 675 | if (size > (PAGE_SIZE << compound_order(qpg))) |
| 676 | return -EINVAL; |
| 677 | |
| 678 | pfn = virt_to_phys(q->ring) >> PAGE_SHIFT; |
| 679 | return remap_pfn_range(vma, vma->vm_start, pfn, |
| 680 | size, vma->vm_page_prot); |
| 681 | } |
| 682 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 683 | static struct proto xsk_proto = { |
| 684 | .name = "XDP", |
| 685 | .owner = THIS_MODULE, |
| 686 | .obj_size = sizeof(struct xdp_sock), |
| 687 | }; |
| 688 | |
| 689 | static const struct proto_ops xsk_proto_ops = { |
Björn Töpel | c2f4374 | 2018-05-18 14:00:24 +0200 | [diff] [blame] | 690 | .family = PF_XDP, |
| 691 | .owner = THIS_MODULE, |
| 692 | .release = xsk_release, |
| 693 | .bind = xsk_bind, |
| 694 | .connect = sock_no_connect, |
| 695 | .socketpair = sock_no_socketpair, |
| 696 | .accept = sock_no_accept, |
| 697 | .getname = sock_no_getname, |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 698 | .poll = xsk_poll, |
Björn Töpel | c2f4374 | 2018-05-18 14:00:24 +0200 | [diff] [blame] | 699 | .ioctl = sock_no_ioctl, |
| 700 | .listen = sock_no_listen, |
| 701 | .shutdown = sock_no_shutdown, |
| 702 | .setsockopt = xsk_setsockopt, |
| 703 | .getsockopt = xsk_getsockopt, |
| 704 | .sendmsg = xsk_sendmsg, |
| 705 | .recvmsg = sock_no_recvmsg, |
| 706 | .mmap = xsk_mmap, |
| 707 | .sendpage = sock_no_sendpage, |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 708 | }; |
| 709 | |
| 710 | static void xsk_destruct(struct sock *sk) |
| 711 | { |
| 712 | struct xdp_sock *xs = xdp_sk(sk); |
| 713 | |
| 714 | if (!sock_flag(sk, SOCK_DEAD)) |
| 715 | return; |
| 716 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 717 | xskq_destroy(xs->rx); |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 718 | xskq_destroy(xs->tx); |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 719 | xdp_del_sk_umem(xs->umem, xs); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 720 | xdp_put_umem(xs->umem); |
| 721 | |
| 722 | sk_refcnt_debug_dec(sk); |
| 723 | } |
| 724 | |
| 725 | static int xsk_create(struct net *net, struct socket *sock, int protocol, |
| 726 | int kern) |
| 727 | { |
| 728 | struct sock *sk; |
| 729 | struct xdp_sock *xs; |
| 730 | |
| 731 | if (!ns_capable(net->user_ns, CAP_NET_RAW)) |
| 732 | return -EPERM; |
| 733 | if (sock->type != SOCK_RAW) |
| 734 | return -ESOCKTNOSUPPORT; |
| 735 | |
| 736 | if (protocol) |
| 737 | return -EPROTONOSUPPORT; |
| 738 | |
| 739 | sock->state = SS_UNCONNECTED; |
| 740 | |
| 741 | sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern); |
| 742 | if (!sk) |
| 743 | return -ENOBUFS; |
| 744 | |
| 745 | sock->ops = &xsk_proto_ops; |
| 746 | |
| 747 | sock_init_data(sock, sk); |
| 748 | |
| 749 | sk->sk_family = PF_XDP; |
| 750 | |
| 751 | sk->sk_destruct = xsk_destruct; |
| 752 | sk_refcnt_debug_inc(sk); |
| 753 | |
| 754 | xs = xdp_sk(sk); |
| 755 | mutex_init(&xs->mutex); |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 756 | spin_lock_init(&xs->tx_completion_lock); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 757 | |
| 758 | local_bh_disable(); |
| 759 | sock_prot_inuse_add(net, &xsk_proto, 1); |
| 760 | local_bh_enable(); |
| 761 | |
| 762 | return 0; |
| 763 | } |
| 764 | |
| 765 | static const struct net_proto_family xsk_family_ops = { |
| 766 | .family = PF_XDP, |
| 767 | .create = xsk_create, |
| 768 | .owner = THIS_MODULE, |
| 769 | }; |
| 770 | |
| 771 | static int __init xsk_init(void) |
| 772 | { |
| 773 | int err; |
| 774 | |
| 775 | err = proto_register(&xsk_proto, 0 /* no slab */); |
| 776 | if (err) |
| 777 | goto out; |
| 778 | |
| 779 | err = sock_register(&xsk_family_ops); |
| 780 | if (err) |
| 781 | goto out_proto; |
| 782 | |
| 783 | return 0; |
| 784 | |
| 785 | out_proto: |
| 786 | proto_unregister(&xsk_proto); |
| 787 | out: |
| 788 | return err; |
| 789 | } |
| 790 | |
| 791 | fs_initcall(xsk_init); |