Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* XDP sockets |
| 3 | * |
| 4 | * AF_XDP sockets allows a channel between XDP programs and userspace |
| 5 | * applications. |
| 6 | * Copyright(c) 2018 Intel Corporation. |
| 7 | * |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 8 | * Author(s): Björn Töpel <bjorn.topel@intel.com> |
| 9 | * Magnus Karlsson <magnus.karlsson@intel.com> |
| 10 | */ |
| 11 | |
| 12 | #define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__ |
| 13 | |
| 14 | #include <linux/if_xdp.h> |
| 15 | #include <linux/init.h> |
| 16 | #include <linux/sched/mm.h> |
| 17 | #include <linux/sched/signal.h> |
| 18 | #include <linux/sched/task.h> |
| 19 | #include <linux/socket.h> |
| 20 | #include <linux/file.h> |
| 21 | #include <linux/uaccess.h> |
| 22 | #include <linux/net.h> |
| 23 | #include <linux/netdevice.h> |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 24 | #include <linux/rculist.h> |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 25 | #include <net/xdp_sock.h> |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 26 | #include <net/xdp.h> |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 27 | |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 28 | #include "xsk_queue.h" |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 29 | #include "xdp_umem.h" |
| 30 | |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 31 | #define TX_BATCH_SIZE 16 |
| 32 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 33 | static struct xdp_sock *xdp_sk(struct sock *sk) |
| 34 | { |
| 35 | return (struct xdp_sock *)sk; |
| 36 | } |
| 37 | |
Björn Töpel | fbfc504 | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 38 | bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) |
| 39 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 40 | return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) && |
| 41 | READ_ONCE(xs->umem->fq); |
Björn Töpel | fbfc504 | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 42 | } |
| 43 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 44 | u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr) |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 45 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 46 | return xskq_peek_addr(umem->fq, addr); |
| 47 | } |
| 48 | EXPORT_SYMBOL(xsk_umem_peek_addr); |
| 49 | |
| 50 | void xsk_umem_discard_addr(struct xdp_umem *umem) |
| 51 | { |
| 52 | xskq_discard_addr(umem->fq); |
| 53 | } |
| 54 | EXPORT_SYMBOL(xsk_umem_discard_addr); |
| 55 | |
| 56 | static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) |
| 57 | { |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 58 | void *buffer; |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 59 | u64 addr; |
Björn Töpel | 4e64c83 | 2018-06-04 13:57:11 +0200 | [diff] [blame] | 60 | int err; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 61 | |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 62 | if (!xskq_peek_addr(xs->umem->fq, &addr) || |
| 63 | len > xs->umem->chunk_size_nohr) { |
Björn Töpel | a509a95 | 2018-06-04 13:57:12 +0200 | [diff] [blame] | 64 | xs->rx_dropped++; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 65 | return -ENOSPC; |
Björn Töpel | a509a95 | 2018-06-04 13:57:12 +0200 | [diff] [blame] | 66 | } |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 67 | |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 68 | addr += xs->umem->headroom; |
| 69 | |
| 70 | buffer = xdp_umem_get_data(xs->umem, addr); |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 71 | memcpy(buffer, xdp->data, len); |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 72 | err = xskq_produce_batch_desc(xs->rx, addr, len); |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 73 | if (!err) { |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 74 | xskq_discard_addr(xs->umem->fq); |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 75 | xdp_return_buff(xdp); |
| 76 | return 0; |
| 77 | } |
| 78 | |
| 79 | xs->rx_dropped++; |
| 80 | return err; |
| 81 | } |
| 82 | |
| 83 | static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) |
| 84 | { |
| 85 | int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len); |
| 86 | |
Jakub Kicinski | 2d55d61 | 2018-07-27 20:20:08 -0700 | [diff] [blame] | 87 | if (err) |
Björn Töpel | a509a95 | 2018-06-04 13:57:12 +0200 | [diff] [blame] | 88 | xs->rx_dropped++; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 89 | |
| 90 | return err; |
| 91 | } |
| 92 | |
| 93 | int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) |
| 94 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 95 | u32 len; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 96 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 97 | if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) |
| 98 | return -EINVAL; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 99 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 100 | len = xdp->data_end - xdp->data; |
| 101 | |
| 102 | return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ? |
| 103 | __xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len); |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 104 | } |
| 105 | |
| 106 | void xsk_flush(struct xdp_sock *xs) |
| 107 | { |
| 108 | xskq_produce_flush_desc(xs->rx); |
| 109 | xs->sk.sk_data_ready(&xs->sk); |
| 110 | } |
| 111 | |
| 112 | int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) |
| 113 | { |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 114 | u32 len = xdp->data_end - xdp->data; |
| 115 | void *buffer; |
| 116 | u64 addr; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 117 | int err; |
| 118 | |
Björn Töpel | 5d90237 | 2018-06-12 12:02:56 +0200 | [diff] [blame] | 119 | if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) |
| 120 | return -EINVAL; |
| 121 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 122 | if (!xskq_peek_addr(xs->umem->fq, &addr) || |
| 123 | len > xs->umem->chunk_size_nohr) { |
| 124 | xs->rx_dropped++; |
| 125 | return -ENOSPC; |
| 126 | } |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 127 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 128 | addr += xs->umem->headroom; |
| 129 | |
| 130 | buffer = xdp_umem_get_data(xs->umem, addr); |
| 131 | memcpy(buffer, xdp->data, len); |
| 132 | err = xskq_produce_batch_desc(xs->rx, addr, len); |
| 133 | if (!err) { |
| 134 | xskq_discard_addr(xs->umem->fq); |
| 135 | xsk_flush(xs); |
| 136 | return 0; |
| 137 | } |
| 138 | |
| 139 | xs->rx_dropped++; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 140 | return err; |
| 141 | } |
| 142 | |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 143 | void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries) |
| 144 | { |
| 145 | xskq_produce_flush_addr_n(umem->cq, nb_entries); |
| 146 | } |
| 147 | EXPORT_SYMBOL(xsk_umem_complete_tx); |
| 148 | |
| 149 | void xsk_umem_consume_tx_done(struct xdp_umem *umem) |
| 150 | { |
| 151 | struct xdp_sock *xs; |
| 152 | |
| 153 | rcu_read_lock(); |
| 154 | list_for_each_entry_rcu(xs, &umem->xsk_list, list) { |
| 155 | xs->sk.sk_write_space(&xs->sk); |
| 156 | } |
| 157 | rcu_read_unlock(); |
| 158 | } |
| 159 | EXPORT_SYMBOL(xsk_umem_consume_tx_done); |
| 160 | |
| 161 | bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len) |
| 162 | { |
| 163 | struct xdp_desc desc; |
| 164 | struct xdp_sock *xs; |
| 165 | |
| 166 | rcu_read_lock(); |
| 167 | list_for_each_entry_rcu(xs, &umem->xsk_list, list) { |
| 168 | if (!xskq_peek_desc(xs->tx, &desc)) |
| 169 | continue; |
| 170 | |
| 171 | if (xskq_produce_addr_lazy(umem->cq, desc.addr)) |
| 172 | goto out; |
| 173 | |
| 174 | *dma = xdp_umem_get_dma(umem, desc.addr); |
| 175 | *len = desc.len; |
| 176 | |
| 177 | xskq_discard_desc(xs->tx); |
| 178 | rcu_read_unlock(); |
| 179 | return true; |
| 180 | } |
| 181 | |
| 182 | out: |
| 183 | rcu_read_unlock(); |
| 184 | return false; |
| 185 | } |
| 186 | EXPORT_SYMBOL(xsk_umem_consume_tx); |
| 187 | |
| 188 | static int xsk_zc_xmit(struct sock *sk) |
| 189 | { |
| 190 | struct xdp_sock *xs = xdp_sk(sk); |
| 191 | struct net_device *dev = xs->dev; |
| 192 | |
| 193 | return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id); |
| 194 | } |
| 195 | |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 196 | static void xsk_destruct_skb(struct sk_buff *skb) |
| 197 | { |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 198 | u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 199 | struct xdp_sock *xs = xdp_sk(skb->sk); |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 200 | unsigned long flags; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 201 | |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 202 | spin_lock_irqsave(&xs->tx_completion_lock, flags); |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 203 | WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr)); |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 204 | spin_unlock_irqrestore(&xs->tx_completion_lock, flags); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 205 | |
| 206 | sock_wfree(skb); |
| 207 | } |
| 208 | |
| 209 | static int xsk_generic_xmit(struct sock *sk, struct msghdr *m, |
| 210 | size_t total_len) |
| 211 | { |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 212 | u32 max_batch = TX_BATCH_SIZE; |
| 213 | struct xdp_sock *xs = xdp_sk(sk); |
| 214 | bool sent_frame = false; |
| 215 | struct xdp_desc desc; |
| 216 | struct sk_buff *skb; |
| 217 | int err = 0; |
| 218 | |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 219 | mutex_lock(&xs->mutex); |
| 220 | |
| 221 | while (xskq_peek_desc(xs->tx, &desc)) { |
| 222 | char *buffer; |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 223 | u64 addr; |
| 224 | u32 len; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 225 | |
| 226 | if (max_batch-- == 0) { |
| 227 | err = -EAGAIN; |
| 228 | goto out; |
| 229 | } |
| 230 | |
Magnus Karlsson | 9684f5e | 2018-07-11 10:12:50 +0200 | [diff] [blame] | 231 | if (xskq_reserve_addr(xs->umem->cq)) |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 232 | goto out; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 233 | |
Magnus Karlsson | 509d764 | 2018-07-11 10:12:49 +0200 | [diff] [blame] | 234 | if (xs->queue_id >= xs->dev->real_num_tx_queues) |
Magnus Karlsson | 2e59dd5 | 2018-05-22 09:34:58 +0200 | [diff] [blame] | 235 | goto out; |
Magnus Karlsson | 2e59dd5 | 2018-05-22 09:34:58 +0200 | [diff] [blame] | 236 | |
Magnus Karlsson | 09210c4 | 2018-07-11 10:12:52 +0200 | [diff] [blame] | 237 | len = desc.len; |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 238 | skb = sock_alloc_send_skb(sk, len, 1, &err); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 239 | if (unlikely(!skb)) { |
| 240 | err = -EAGAIN; |
| 241 | goto out; |
| 242 | } |
| 243 | |
| 244 | skb_put(skb, len); |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 245 | addr = desc.addr; |
| 246 | buffer = xdp_umem_get_data(xs->umem, addr); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 247 | err = skb_store_bits(skb, 0, buffer, len); |
| 248 | if (unlikely(err)) { |
| 249 | kfree_skb(skb); |
| 250 | goto out; |
| 251 | } |
| 252 | |
| 253 | skb->dev = xs->dev; |
| 254 | skb->priority = sk->sk_priority; |
| 255 | skb->mark = sk->sk_mark; |
Björn Töpel | bbff2f3 | 2018-06-04 13:57:13 +0200 | [diff] [blame] | 256 | skb_shinfo(skb)->destructor_arg = (void *)(long)addr; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 257 | skb->destructor = xsk_destruct_skb; |
| 258 | |
| 259 | err = dev_direct_xmit(skb, xs->queue_id); |
Magnus Karlsson | fe58868 | 2018-06-29 09:48:18 +0200 | [diff] [blame] | 260 | xskq_discard_desc(xs->tx); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 261 | /* Ignore NET_XMIT_CN as packet might have been sent */ |
| 262 | if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { |
Magnus Karlsson | fe58868 | 2018-06-29 09:48:18 +0200 | [diff] [blame] | 263 | /* SKB completed but not sent */ |
| 264 | err = -EBUSY; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 265 | goto out; |
| 266 | } |
| 267 | |
| 268 | sent_frame = true; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 269 | } |
| 270 | |
| 271 | out: |
| 272 | if (sent_frame) |
| 273 | sk->sk_write_space(sk); |
| 274 | |
| 275 | mutex_unlock(&xs->mutex); |
| 276 | return err; |
| 277 | } |
| 278 | |
| 279 | static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) |
| 280 | { |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 281 | bool need_wait = !(m->msg_flags & MSG_DONTWAIT); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 282 | struct sock *sk = sock->sk; |
| 283 | struct xdp_sock *xs = xdp_sk(sk); |
| 284 | |
| 285 | if (unlikely(!xs->dev)) |
| 286 | return -ENXIO; |
| 287 | if (unlikely(!(xs->dev->flags & IFF_UP))) |
| 288 | return -ENETDOWN; |
Magnus Karlsson | 6efb443 | 2018-07-11 10:12:51 +0200 | [diff] [blame] | 289 | if (unlikely(!xs->tx)) |
| 290 | return -ENOBUFS; |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 291 | if (need_wait) |
| 292 | return -EOPNOTSUPP; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 293 | |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 294 | return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 295 | } |
| 296 | |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 297 | static unsigned int xsk_poll(struct file *file, struct socket *sock, |
| 298 | struct poll_table_struct *wait) |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 299 | { |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 300 | unsigned int mask = datagram_poll(file, sock, wait); |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 301 | struct sock *sk = sock->sk; |
| 302 | struct xdp_sock *xs = xdp_sk(sk); |
| 303 | |
| 304 | if (xs->rx && !xskq_empty_desc(xs->rx)) |
| 305 | mask |= POLLIN | POLLRDNORM; |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 306 | if (xs->tx && !xskq_full_desc(xs->tx)) |
| 307 | mask |= POLLOUT | POLLWRNORM; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 308 | |
| 309 | return mask; |
| 310 | } |
| 311 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 312 | static int xsk_init_queue(u32 entries, struct xsk_queue **queue, |
| 313 | bool umem_queue) |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 314 | { |
| 315 | struct xsk_queue *q; |
| 316 | |
| 317 | if (entries == 0 || *queue || !is_power_of_2(entries)) |
| 318 | return -EINVAL; |
| 319 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 320 | q = xskq_create(entries, umem_queue); |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 321 | if (!q) |
| 322 | return -ENOMEM; |
| 323 | |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 324 | /* Make sure queue is ready before it can be seen by others */ |
| 325 | smp_wmb(); |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 326 | *queue = q; |
| 327 | return 0; |
| 328 | } |
| 329 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 330 | static int xsk_release(struct socket *sock) |
| 331 | { |
| 332 | struct sock *sk = sock->sk; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 333 | struct xdp_sock *xs = xdp_sk(sk); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 334 | struct net *net; |
| 335 | |
| 336 | if (!sk) |
| 337 | return 0; |
| 338 | |
| 339 | net = sock_net(sk); |
| 340 | |
| 341 | local_bh_disable(); |
| 342 | sock_prot_inuse_add(net, sk->sk_prot, -1); |
| 343 | local_bh_enable(); |
| 344 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 345 | if (xs->dev) { |
Björn Töpel | b17ddbd | 2018-10-05 13:25:15 +0200 | [diff] [blame] | 346 | struct net_device *dev = xs->dev; |
| 347 | |
Björn Töpel | 959b71d | 2018-05-22 09:34:56 +0200 | [diff] [blame] | 348 | /* Wait for driver to stop using the xdp socket. */ |
Björn Töpel | b17ddbd | 2018-10-05 13:25:15 +0200 | [diff] [blame] | 349 | xdp_del_sk_umem(xs->umem, xs); |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 350 | xs->dev = NULL; |
Björn Töpel | b17ddbd | 2018-10-05 13:25:15 +0200 | [diff] [blame] | 351 | synchronize_net(); |
| 352 | dev_put(dev); |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 353 | } |
| 354 | |
Björn Töpel | b17ddbd | 2018-10-05 13:25:15 +0200 | [diff] [blame] | 355 | xskq_destroy(xs->rx); |
| 356 | xskq_destroy(xs->tx); |
| 357 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 358 | sock_orphan(sk); |
| 359 | sock->sk = NULL; |
| 360 | |
| 361 | sk_refcnt_debug_release(sk); |
| 362 | sock_put(sk); |
| 363 | |
| 364 | return 0; |
| 365 | } |
| 366 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 367 | static struct socket *xsk_lookup_xsk_from_fd(int fd) |
| 368 | { |
| 369 | struct socket *sock; |
| 370 | int err; |
| 371 | |
| 372 | sock = sockfd_lookup(fd, &err); |
| 373 | if (!sock) |
| 374 | return ERR_PTR(-ENOTSOCK); |
| 375 | |
| 376 | if (sock->sk->sk_family != PF_XDP) { |
| 377 | sockfd_put(sock); |
| 378 | return ERR_PTR(-ENOPROTOOPT); |
| 379 | } |
| 380 | |
| 381 | return sock; |
| 382 | } |
| 383 | |
| 384 | static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) |
| 385 | { |
| 386 | struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr; |
| 387 | struct sock *sk = sock->sk; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 388 | struct xdp_sock *xs = xdp_sk(sk); |
Björn Töpel | 959b71d | 2018-05-22 09:34:56 +0200 | [diff] [blame] | 389 | struct net_device *dev; |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 390 | u32 flags, qid; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 391 | int err = 0; |
| 392 | |
| 393 | if (addr_len < sizeof(struct sockaddr_xdp)) |
| 394 | return -EINVAL; |
| 395 | if (sxdp->sxdp_family != AF_XDP) |
| 396 | return -EINVAL; |
| 397 | |
| 398 | mutex_lock(&xs->mutex); |
Björn Töpel | 959b71d | 2018-05-22 09:34:56 +0200 | [diff] [blame] | 399 | if (xs->dev) { |
| 400 | err = -EBUSY; |
| 401 | goto out_release; |
| 402 | } |
| 403 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 404 | dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex); |
| 405 | if (!dev) { |
| 406 | err = -ENODEV; |
| 407 | goto out_release; |
| 408 | } |
| 409 | |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 410 | if (!xs->rx && !xs->tx) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 411 | err = -EINVAL; |
| 412 | goto out_unlock; |
| 413 | } |
| 414 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 415 | qid = sxdp->sxdp_queue_id; |
| 416 | |
| 417 | if ((xs->rx && qid >= dev->real_num_rx_queues) || |
| 418 | (xs->tx && qid >= dev->real_num_tx_queues)) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 419 | err = -EINVAL; |
| 420 | goto out_unlock; |
| 421 | } |
| 422 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 423 | flags = sxdp->sxdp_flags; |
| 424 | |
| 425 | if (flags & XDP_SHARED_UMEM) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 426 | struct xdp_sock *umem_xs; |
| 427 | struct socket *sock; |
| 428 | |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 429 | if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY)) { |
| 430 | /* Cannot specify flags for shared sockets. */ |
| 431 | err = -EINVAL; |
| 432 | goto out_unlock; |
| 433 | } |
| 434 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 435 | if (xs->umem) { |
| 436 | /* We have already our own. */ |
| 437 | err = -EINVAL; |
| 438 | goto out_unlock; |
| 439 | } |
| 440 | |
| 441 | sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd); |
| 442 | if (IS_ERR(sock)) { |
| 443 | err = PTR_ERR(sock); |
| 444 | goto out_unlock; |
| 445 | } |
| 446 | |
| 447 | umem_xs = xdp_sk(sock->sk); |
| 448 | if (!umem_xs->umem) { |
| 449 | /* No umem to inherit. */ |
| 450 | err = -EBADF; |
| 451 | sockfd_put(sock); |
| 452 | goto out_unlock; |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 453 | } else if (umem_xs->dev != dev || umem_xs->queue_id != qid) { |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 454 | err = -EINVAL; |
| 455 | sockfd_put(sock); |
| 456 | goto out_unlock; |
| 457 | } |
| 458 | |
| 459 | xdp_get_umem(umem_xs->umem); |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 460 | xs->umem = umem_xs->umem; |
| 461 | sockfd_put(sock); |
| 462 | } else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) { |
| 463 | err = -EINVAL; |
| 464 | goto out_unlock; |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 465 | } else { |
| 466 | /* This xsk has its own umem. */ |
| 467 | xskq_set_umem(xs->umem->fq, &xs->umem->props); |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 468 | xskq_set_umem(xs->umem->cq, &xs->umem->props); |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 469 | |
| 470 | err = xdp_umem_assign_dev(xs->umem, dev, qid, flags); |
| 471 | if (err) |
| 472 | goto out_unlock; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 473 | } |
| 474 | |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 475 | xs->dev = dev; |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 476 | xs->zc = xs->umem->zc; |
| 477 | xs->queue_id = qid; |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 478 | xskq_set_umem(xs->rx, &xs->umem->props); |
Magnus Karlsson | 35fcde7 | 2018-05-02 13:01:34 +0200 | [diff] [blame] | 479 | xskq_set_umem(xs->tx, &xs->umem->props); |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 480 | xdp_add_sk_umem(xs->umem, xs); |
Magnus Karlsson | 965a990 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 481 | |
| 482 | out_unlock: |
| 483 | if (err) |
| 484 | dev_put(dev); |
| 485 | out_release: |
| 486 | mutex_unlock(&xs->mutex); |
| 487 | return err; |
| 488 | } |
| 489 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 490 | static int xsk_setsockopt(struct socket *sock, int level, int optname, |
| 491 | char __user *optval, unsigned int optlen) |
| 492 | { |
| 493 | struct sock *sk = sock->sk; |
| 494 | struct xdp_sock *xs = xdp_sk(sk); |
| 495 | int err; |
| 496 | |
| 497 | if (level != SOL_XDP) |
| 498 | return -ENOPROTOOPT; |
| 499 | |
| 500 | switch (optname) { |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 501 | case XDP_RX_RING: |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 502 | case XDP_TX_RING: |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 503 | { |
| 504 | struct xsk_queue **q; |
| 505 | int entries; |
| 506 | |
| 507 | if (optlen < sizeof(entries)) |
| 508 | return -EINVAL; |
| 509 | if (copy_from_user(&entries, optval, sizeof(entries))) |
| 510 | return -EFAULT; |
| 511 | |
| 512 | mutex_lock(&xs->mutex); |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 513 | q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx; |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 514 | err = xsk_init_queue(entries, q, false); |
| 515 | mutex_unlock(&xs->mutex); |
| 516 | return err; |
| 517 | } |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 518 | case XDP_UMEM_REG: |
| 519 | { |
| 520 | struct xdp_umem_reg mr; |
| 521 | struct xdp_umem *umem; |
| 522 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 523 | if (copy_from_user(&mr, optval, sizeof(mr))) |
| 524 | return -EFAULT; |
| 525 | |
| 526 | mutex_lock(&xs->mutex); |
Björn Töpel | a49049e | 2018-05-22 09:35:02 +0200 | [diff] [blame] | 527 | if (xs->umem) { |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 528 | mutex_unlock(&xs->mutex); |
Björn Töpel | a49049e | 2018-05-22 09:35:02 +0200 | [diff] [blame] | 529 | return -EBUSY; |
| 530 | } |
| 531 | |
| 532 | umem = xdp_umem_create(&mr); |
| 533 | if (IS_ERR(umem)) { |
| 534 | mutex_unlock(&xs->mutex); |
| 535 | return PTR_ERR(umem); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 536 | } |
| 537 | |
| 538 | /* Make sure umem is ready before it can be seen by others */ |
| 539 | smp_wmb(); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 540 | xs->umem = umem; |
| 541 | mutex_unlock(&xs->mutex); |
| 542 | return 0; |
| 543 | } |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 544 | case XDP_UMEM_FILL_RING: |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 545 | case XDP_UMEM_COMPLETION_RING: |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 546 | { |
| 547 | struct xsk_queue **q; |
| 548 | int entries; |
| 549 | |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 550 | if (copy_from_user(&entries, optval, sizeof(entries))) |
| 551 | return -EFAULT; |
| 552 | |
| 553 | mutex_lock(&xs->mutex); |
Björn Töpel | a49049e | 2018-05-22 09:35:02 +0200 | [diff] [blame] | 554 | if (!xs->umem) { |
| 555 | mutex_unlock(&xs->mutex); |
| 556 | return -EINVAL; |
| 557 | } |
| 558 | |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 559 | q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq : |
| 560 | &xs->umem->cq; |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 561 | err = xsk_init_queue(entries, q, true); |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 562 | mutex_unlock(&xs->mutex); |
| 563 | return err; |
| 564 | } |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 565 | default: |
| 566 | break; |
| 567 | } |
| 568 | |
| 569 | return -ENOPROTOOPT; |
| 570 | } |
| 571 | |
Magnus Karlsson | af75d9e | 2018-05-02 13:01:35 +0200 | [diff] [blame] | 572 | static int xsk_getsockopt(struct socket *sock, int level, int optname, |
| 573 | char __user *optval, int __user *optlen) |
| 574 | { |
| 575 | struct sock *sk = sock->sk; |
| 576 | struct xdp_sock *xs = xdp_sk(sk); |
| 577 | int len; |
| 578 | |
| 579 | if (level != SOL_XDP) |
| 580 | return -ENOPROTOOPT; |
| 581 | |
| 582 | if (get_user(len, optlen)) |
| 583 | return -EFAULT; |
| 584 | if (len < 0) |
| 585 | return -EINVAL; |
| 586 | |
| 587 | switch (optname) { |
| 588 | case XDP_STATISTICS: |
| 589 | { |
| 590 | struct xdp_statistics stats; |
| 591 | |
| 592 | if (len < sizeof(stats)) |
| 593 | return -EINVAL; |
| 594 | |
| 595 | mutex_lock(&xs->mutex); |
| 596 | stats.rx_dropped = xs->rx_dropped; |
| 597 | stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx); |
| 598 | stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx); |
| 599 | mutex_unlock(&xs->mutex); |
| 600 | |
| 601 | if (copy_to_user(optval, &stats, sizeof(stats))) |
| 602 | return -EFAULT; |
| 603 | if (put_user(sizeof(stats), optlen)) |
| 604 | return -EFAULT; |
| 605 | |
| 606 | return 0; |
| 607 | } |
Björn Töpel | b3a9e0b | 2018-05-22 09:34:59 +0200 | [diff] [blame] | 608 | case XDP_MMAP_OFFSETS: |
| 609 | { |
| 610 | struct xdp_mmap_offsets off; |
| 611 | |
| 612 | if (len < sizeof(off)) |
| 613 | return -EINVAL; |
| 614 | |
| 615 | off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer); |
| 616 | off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer); |
| 617 | off.rx.desc = offsetof(struct xdp_rxtx_ring, desc); |
| 618 | off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer); |
| 619 | off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer); |
| 620 | off.tx.desc = offsetof(struct xdp_rxtx_ring, desc); |
| 621 | |
| 622 | off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer); |
| 623 | off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer); |
| 624 | off.fr.desc = offsetof(struct xdp_umem_ring, desc); |
| 625 | off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer); |
| 626 | off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer); |
| 627 | off.cr.desc = offsetof(struct xdp_umem_ring, desc); |
| 628 | |
| 629 | len = sizeof(off); |
| 630 | if (copy_to_user(optval, &off, len)) |
| 631 | return -EFAULT; |
| 632 | if (put_user(len, optlen)) |
| 633 | return -EFAULT; |
| 634 | |
| 635 | return 0; |
| 636 | } |
Magnus Karlsson | af75d9e | 2018-05-02 13:01:35 +0200 | [diff] [blame] | 637 | default: |
| 638 | break; |
| 639 | } |
| 640 | |
| 641 | return -EOPNOTSUPP; |
| 642 | } |
| 643 | |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 644 | static int xsk_mmap(struct file *file, struct socket *sock, |
| 645 | struct vm_area_struct *vma) |
| 646 | { |
Geert Uytterhoeven | a5a16e4 | 2018-06-07 15:37:34 +0200 | [diff] [blame] | 647 | loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 648 | unsigned long size = vma->vm_end - vma->vm_start; |
| 649 | struct xdp_sock *xs = xdp_sk(sock->sk); |
| 650 | struct xsk_queue *q = NULL; |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 651 | struct xdp_umem *umem; |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 652 | unsigned long pfn; |
| 653 | struct page *qpg; |
| 654 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 655 | if (offset == XDP_PGOFF_RX_RING) { |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 656 | q = READ_ONCE(xs->rx); |
Magnus Karlsson | f614590 | 2018-05-02 13:01:32 +0200 | [diff] [blame] | 657 | } else if (offset == XDP_PGOFF_TX_RING) { |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 658 | q = READ_ONCE(xs->tx); |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 659 | } else { |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 660 | umem = READ_ONCE(xs->umem); |
| 661 | if (!umem) |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 662 | return -EINVAL; |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 663 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 664 | if (offset == XDP_UMEM_PGOFF_FILL_RING) |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 665 | q = READ_ONCE(umem->fq); |
Magnus Karlsson | fe23083 | 2018-05-02 13:01:31 +0200 | [diff] [blame] | 666 | else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING) |
Björn Töpel | 37b0769 | 2018-05-22 09:35:01 +0200 | [diff] [blame] | 667 | q = READ_ONCE(umem->cq); |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 668 | } |
Magnus Karlsson | 423f383 | 2018-05-02 13:01:24 +0200 | [diff] [blame] | 669 | |
| 670 | if (!q) |
| 671 | return -EINVAL; |
| 672 | |
| 673 | qpg = virt_to_head_page(q->ring); |
| 674 | if (size > (PAGE_SIZE << compound_order(qpg))) |
| 675 | return -EINVAL; |
| 676 | |
| 677 | pfn = virt_to_phys(q->ring) >> PAGE_SHIFT; |
| 678 | return remap_pfn_range(vma, vma->vm_start, pfn, |
| 679 | size, vma->vm_page_prot); |
| 680 | } |
| 681 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 682 | static struct proto xsk_proto = { |
| 683 | .name = "XDP", |
| 684 | .owner = THIS_MODULE, |
| 685 | .obj_size = sizeof(struct xdp_sock), |
| 686 | }; |
| 687 | |
| 688 | static const struct proto_ops xsk_proto_ops = { |
Björn Töpel | c2f4374 | 2018-05-18 14:00:24 +0200 | [diff] [blame] | 689 | .family = PF_XDP, |
| 690 | .owner = THIS_MODULE, |
| 691 | .release = xsk_release, |
| 692 | .bind = xsk_bind, |
| 693 | .connect = sock_no_connect, |
| 694 | .socketpair = sock_no_socketpair, |
| 695 | .accept = sock_no_accept, |
| 696 | .getname = sock_no_getname, |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 697 | .poll = xsk_poll, |
Björn Töpel | c2f4374 | 2018-05-18 14:00:24 +0200 | [diff] [blame] | 698 | .ioctl = sock_no_ioctl, |
| 699 | .listen = sock_no_listen, |
| 700 | .shutdown = sock_no_shutdown, |
| 701 | .setsockopt = xsk_setsockopt, |
| 702 | .getsockopt = xsk_getsockopt, |
| 703 | .sendmsg = xsk_sendmsg, |
| 704 | .recvmsg = sock_no_recvmsg, |
| 705 | .mmap = xsk_mmap, |
| 706 | .sendpage = sock_no_sendpage, |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 707 | }; |
| 708 | |
| 709 | static void xsk_destruct(struct sock *sk) |
| 710 | { |
| 711 | struct xdp_sock *xs = xdp_sk(sk); |
| 712 | |
| 713 | if (!sock_flag(sk, SOCK_DEAD)) |
| 714 | return; |
| 715 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 716 | xdp_put_umem(xs->umem); |
| 717 | |
| 718 | sk_refcnt_debug_dec(sk); |
| 719 | } |
| 720 | |
| 721 | static int xsk_create(struct net *net, struct socket *sock, int protocol, |
| 722 | int kern) |
| 723 | { |
| 724 | struct sock *sk; |
| 725 | struct xdp_sock *xs; |
| 726 | |
| 727 | if (!ns_capable(net->user_ns, CAP_NET_RAW)) |
| 728 | return -EPERM; |
| 729 | if (sock->type != SOCK_RAW) |
| 730 | return -ESOCKTNOSUPPORT; |
| 731 | |
| 732 | if (protocol) |
| 733 | return -EPROTONOSUPPORT; |
| 734 | |
| 735 | sock->state = SS_UNCONNECTED; |
| 736 | |
| 737 | sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern); |
| 738 | if (!sk) |
| 739 | return -ENOBUFS; |
| 740 | |
| 741 | sock->ops = &xsk_proto_ops; |
| 742 | |
| 743 | sock_init_data(sock, sk); |
| 744 | |
| 745 | sk->sk_family = PF_XDP; |
| 746 | |
| 747 | sk->sk_destruct = xsk_destruct; |
| 748 | sk_refcnt_debug_inc(sk); |
| 749 | |
Björn Töpel | cee2716 | 2018-10-08 19:40:16 +0200 | [diff] [blame] | 750 | sock_set_flag(sk, SOCK_RCU_FREE); |
| 751 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 752 | xs = xdp_sk(sk); |
| 753 | mutex_init(&xs->mutex); |
Magnus Karlsson | a9744f7 | 2018-06-29 09:48:20 +0200 | [diff] [blame] | 754 | spin_lock_init(&xs->tx_completion_lock); |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 755 | |
| 756 | local_bh_disable(); |
| 757 | sock_prot_inuse_add(net, &xsk_proto, 1); |
| 758 | local_bh_enable(); |
| 759 | |
| 760 | return 0; |
| 761 | } |
| 762 | |
| 763 | static const struct net_proto_family xsk_family_ops = { |
| 764 | .family = PF_XDP, |
| 765 | .create = xsk_create, |
| 766 | .owner = THIS_MODULE, |
| 767 | }; |
| 768 | |
| 769 | static int __init xsk_init(void) |
| 770 | { |
| 771 | int err; |
| 772 | |
| 773 | err = proto_register(&xsk_proto, 0 /* no slab */); |
| 774 | if (err) |
| 775 | goto out; |
| 776 | |
| 777 | err = sock_register(&xsk_family_ops); |
| 778 | if (err) |
| 779 | goto out_proto; |
| 780 | |
| 781 | return 0; |
| 782 | |
| 783 | out_proto: |
| 784 | proto_unregister(&xsk_proto); |
| 785 | out: |
| 786 | return err; |
| 787 | } |
| 788 | |
| 789 | fs_initcall(xsk_init); |