blob: b306210b02b7b40c717ae160e4116db8926418ce [file] [log] [blame]
Erwan Yvin0d2e1a22013-03-20 13:52:24 +10301/*
2 * Copyright (C) ST-Ericsson AB 2013
Sjur Brændeland01d779a2013-05-01 11:57:50 +09303 * Authors: Vicram Arv
4 * Dmitry Tarnyagin <dmitry.tarnyagin@lockless.no>
5 * Sjur Brendeland
Erwan Yvin0d2e1a22013-03-20 13:52:24 +10306 * License terms: GNU General Public License (GPL) version 2
7 */
8#include <linux/module.h>
9#include <linux/if_arp.h>
10#include <linux/virtio.h>
11#include <linux/vringh.h>
12#include <linux/debugfs.h>
13#include <linux/spinlock.h>
14#include <linux/genalloc.h>
15#include <linux/interrupt.h>
16#include <linux/netdevice.h>
17#include <linux/rtnetlink.h>
18#include <linux/virtio_ids.h>
19#include <linux/virtio_caif.h>
20#include <linux/virtio_ring.h>
21#include <linux/dma-mapping.h>
22#include <net/caif/caif_dev.h>
23#include <linux/virtio_config.h>
24
25MODULE_LICENSE("GPL v2");
Sjur Brændeland01d779a2013-05-01 11:57:50 +093026MODULE_AUTHOR("Vicram Arv");
27MODULE_AUTHOR("Sjur Brendeland");
Erwan Yvin0d2e1a22013-03-20 13:52:24 +103028MODULE_DESCRIPTION("Virtio CAIF Driver");
29
30/* NAPI schedule quota */
31#define CFV_DEFAULT_QUOTA 32
32
33/* Defaults used if virtio config space is unavailable */
34#define CFV_DEF_MTU_SIZE 4096
35#define CFV_DEF_HEADROOM 32
36#define CFV_DEF_TAILROOM 32
37
38/* Required IP header alignment */
39#define IP_HDR_ALIGN 4
40
41/* struct cfv_napi_contxt - NAPI context info
42 * @riov: IOV holding data read from the ring. Note that riov may
43 * still hold data when cfv_rx_poll() returns.
44 * @head: Last descriptor ID we received from vringh_getdesc_kern.
45 * We use this to put descriptor back on the used ring. USHRT_MAX is
46 * used to indicate invalid head-id.
47 */
48struct cfv_napi_context {
49 struct vringh_kiov riov;
50 unsigned short head;
51};
52
53/* struct cfv_stats - statistics for debugfs
54 * @rx_napi_complete: Number of NAPI completions (RX)
55 * @rx_napi_resched: Number of calls where the full quota was used (RX)
56 * @rx_nomem: Number of SKB alloc failures (RX)
57 * @rx_kicks: Number of RX kicks
58 * @tx_full_ring: Number times TX ring was full
59 * @tx_no_mem: Number of times TX went out of memory
60 * @tx_flow_on: Number of flow on (TX)
61 * @tx_kicks: Number of TX kicks
62 */
63struct cfv_stats {
64 u32 rx_napi_complete;
65 u32 rx_napi_resched;
66 u32 rx_nomem;
67 u32 rx_kicks;
68 u32 tx_full_ring;
69 u32 tx_no_mem;
70 u32 tx_flow_on;
71 u32 tx_kicks;
72};
73
74/* struct cfv_info - Caif Virtio control structure
75 * @cfdev: caif common header
76 * @vdev: Associated virtio device
77 * @vr_rx: rx/downlink host vring
78 * @vq_tx: tx/uplink virtqueue
79 * @ndev: CAIF link layer device
80 * @watermark_tx: indicates number of free descriptors we need
81 * to reopen the tx-queues after overload.
82 * @tx_lock: protects vq_tx from concurrent use
83 * @tx_release_tasklet: Tasklet for freeing consumed TX buffers
84 * @napi: Napi context used in cfv_rx_poll()
85 * @ctx: Context data used in cfv_rx_poll()
86 * @tx_hr: transmit headroom
87 * @rx_hr: receive headroom
88 * @tx_tr: transmit tail room
89 * @rx_tr: receive tail room
90 * @mtu: transmit max size
91 * @mru: receive max size
92 * @allocsz: size of dma memory reserved for TX buffers
93 * @alloc_addr: virtual address to dma memory for TX buffers
94 * @alloc_dma: dma address to dma memory for TX buffers
95 * @genpool: Gen Pool used for allocating TX buffers
96 * @reserved_mem: Pointer to memory reserve allocated from genpool
97 * @reserved_size: Size of memory reserve allocated from genpool
98 * @stats: Statistics exposed in sysfs
99 * @debugfs: Debugfs dentry for statistic counters
100 */
101struct cfv_info {
102 struct caif_dev_common cfdev;
103 struct virtio_device *vdev;
104 struct vringh *vr_rx;
105 struct virtqueue *vq_tx;
106 struct net_device *ndev;
107 unsigned int watermark_tx;
108 /* Protect access to vq_tx */
109 spinlock_t tx_lock;
110 struct tasklet_struct tx_release_tasklet;
111 struct napi_struct napi;
112 struct cfv_napi_context ctx;
113 u16 tx_hr;
114 u16 rx_hr;
115 u16 tx_tr;
116 u16 rx_tr;
117 u32 mtu;
118 u32 mru;
119 size_t allocsz;
120 void *alloc_addr;
121 dma_addr_t alloc_dma;
122 struct gen_pool *genpool;
123 unsigned long reserved_mem;
124 size_t reserved_size;
125 struct cfv_stats stats;
126 struct dentry *debugfs;
127};
128
129/* struct buf_info - maintains transmit buffer data handle
130 * @size: size of transmit buffer
131 * @dma_handle: handle to allocated dma device memory area
132 * @vaddr: virtual address mapping to allocated memory area
133 */
134struct buf_info {
135 size_t size;
136 u8 *vaddr;
137};
138
139/* Called from virtio device, in IRQ context */
140static void cfv_release_cb(struct virtqueue *vq_tx)
141{
142 struct cfv_info *cfv = vq_tx->vdev->priv;
143
144 ++cfv->stats.tx_kicks;
145 tasklet_schedule(&cfv->tx_release_tasklet);
146}
147
148static void free_buf_info(struct cfv_info *cfv, struct buf_info *buf_info)
149{
150 if (!buf_info)
151 return;
152 gen_pool_free(cfv->genpool, (unsigned long) buf_info->vaddr,
153 buf_info->size);
154 kfree(buf_info);
155}
156
157/* This is invoked whenever the remote processor completed processing
158 * a TX msg we just sent, and the buffer is put back to the used ring.
159 */
160static void cfv_release_used_buf(struct virtqueue *vq_tx)
161{
162 struct cfv_info *cfv = vq_tx->vdev->priv;
163 unsigned long flags;
164
165 BUG_ON(vq_tx != cfv->vq_tx);
166
167 for (;;) {
168 unsigned int len;
169 struct buf_info *buf_info;
170
171 /* Get used buffer from used ring to recycle used descriptors */
172 spin_lock_irqsave(&cfv->tx_lock, flags);
173 buf_info = virtqueue_get_buf(vq_tx, &len);
174 spin_unlock_irqrestore(&cfv->tx_lock, flags);
175
176 /* Stop looping if there are no more buffers to free */
177 if (!buf_info)
178 break;
179
180 free_buf_info(cfv, buf_info);
181
182 /* watermark_tx indicates if we previously stopped the tx
183 * queues. If we have enough free stots in the virtio ring,
184 * re-establish memory reserved and open up tx queues.
185 */
186 if (cfv->vq_tx->num_free <= cfv->watermark_tx)
187 continue;
188
189 /* Re-establish memory reserve */
190 if (cfv->reserved_mem == 0 && cfv->genpool)
191 cfv->reserved_mem =
192 gen_pool_alloc(cfv->genpool,
193 cfv->reserved_size);
194
195 /* Open up the tx queues */
196 if (cfv->reserved_mem) {
197 cfv->watermark_tx =
198 virtqueue_get_vring_size(cfv->vq_tx);
199 netif_tx_wake_all_queues(cfv->ndev);
200 /* Buffers are recycled in cfv_netdev_tx, so
201 * disable notifications when queues are opened.
202 */
203 virtqueue_disable_cb(cfv->vq_tx);
204 ++cfv->stats.tx_flow_on;
205 } else {
206 /* if no memory reserve, wait for more free slots */
207 WARN_ON(cfv->watermark_tx >
208 virtqueue_get_vring_size(cfv->vq_tx));
209 cfv->watermark_tx +=
210 virtqueue_get_vring_size(cfv->vq_tx) / 4;
211 }
212 }
213}
214
215/* Allocate a SKB and copy packet data to it */
216static struct sk_buff *cfv_alloc_and_copy_skb(int *err,
217 struct cfv_info *cfv,
218 u8 *frm, u32 frm_len)
219{
220 struct sk_buff *skb;
221 u32 cfpkt_len, pad_len;
222
223 *err = 0;
224 /* Verify that packet size with down-link header and mtu size */
225 if (frm_len > cfv->mru || frm_len <= cfv->rx_hr + cfv->rx_tr) {
226 netdev_err(cfv->ndev,
227 "Invalid frmlen:%u mtu:%u hr:%d tr:%d\n",
228 frm_len, cfv->mru, cfv->rx_hr,
229 cfv->rx_tr);
230 *err = -EPROTO;
231 return NULL;
232 }
233
234 cfpkt_len = frm_len - (cfv->rx_hr + cfv->rx_tr);
235 pad_len = (unsigned long)(frm + cfv->rx_hr) & (IP_HDR_ALIGN - 1);
236
237 skb = netdev_alloc_skb(cfv->ndev, frm_len + pad_len);
238 if (!skb) {
239 *err = -ENOMEM;
240 return NULL;
241 }
242
243 skb_reserve(skb, cfv->rx_hr + pad_len);
244
245 memcpy(skb_put(skb, cfpkt_len), frm + cfv->rx_hr, cfpkt_len);
246 return skb;
247}
248
249/* Get packets from the host vring */
250static int cfv_rx_poll(struct napi_struct *napi, int quota)
251{
252 struct cfv_info *cfv = container_of(napi, struct cfv_info, napi);
253 int rxcnt = 0;
254 int err = 0;
255 void *buf;
256 struct sk_buff *skb;
257 struct vringh_kiov *riov = &cfv->ctx.riov;
258 unsigned int skb_len;
259
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030260 do {
261 skb = NULL;
262
263 /* Put the previous iovec back on the used ring and
264 * fetch a new iovec if we have processed all elements.
265 */
266 if (riov->i == riov->used) {
267 if (cfv->ctx.head != USHRT_MAX) {
268 vringh_complete_kern(cfv->vr_rx,
269 cfv->ctx.head,
270 0);
271 cfv->ctx.head = USHRT_MAX;
272 }
273
274 err = vringh_getdesc_kern(
275 cfv->vr_rx,
276 riov,
277 NULL,
278 &cfv->ctx.head,
279 GFP_ATOMIC);
280
281 if (err <= 0)
282 goto exit;
283 }
284
285 buf = phys_to_virt((unsigned long) riov->iov[riov->i].iov_base);
286 /* TODO: Add check on valid buffer address */
287
288 skb = cfv_alloc_and_copy_skb(&err, cfv, buf,
289 riov->iov[riov->i].iov_len);
290 if (unlikely(err))
291 goto exit;
292
293 /* Push received packet up the stack. */
294 skb_len = skb->len;
295 skb->protocol = htons(ETH_P_CAIF);
296 skb_reset_mac_header(skb);
297 skb->dev = cfv->ndev;
298 err = netif_receive_skb(skb);
299 if (unlikely(err)) {
300 ++cfv->ndev->stats.rx_dropped;
301 } else {
302 ++cfv->ndev->stats.rx_packets;
303 cfv->ndev->stats.rx_bytes += skb_len;
304 }
305
306 ++riov->i;
307 ++rxcnt;
308 } while (rxcnt < quota);
309
310 ++cfv->stats.rx_napi_resched;
311 goto out;
312
313exit:
314 switch (err) {
315 case 0:
316 ++cfv->stats.rx_napi_complete;
317
318 /* Really out of patckets? (stolen from virtio_net)*/
319 napi_complete(napi);
Sjur Brændelandb2273be2013-03-24 14:19:44 +1030320 if (unlikely(!vringh_notify_enable_kern(cfv->vr_rx)) &&
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030321 napi_schedule_prep(napi)) {
322 vringh_notify_disable_kern(cfv->vr_rx);
323 __napi_schedule(napi);
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030324 }
325 break;
326
327 case -ENOMEM:
328 ++cfv->stats.rx_nomem;
329 dev_kfree_skb(skb);
330 /* Stop NAPI poll on OOM, we hope to be polled later */
331 napi_complete(napi);
332 vringh_notify_enable_kern(cfv->vr_rx);
333 break;
334
335 default:
336 /* We're doomed, any modem fault is fatal */
337 netdev_warn(cfv->ndev, "Bad ring, disable device\n");
338 cfv->ndev->stats.rx_dropped = riov->used - riov->i;
339 napi_complete(napi);
340 vringh_notify_disable_kern(cfv->vr_rx);
341 netif_carrier_off(cfv->ndev);
342 break;
343 }
344out:
345 if (rxcnt && vringh_need_notify_kern(cfv->vr_rx) > 0)
346 vringh_notify(cfv->vr_rx);
347 return rxcnt;
348}
349
350static void cfv_recv(struct virtio_device *vdev, struct vringh *vr_rx)
351{
352 struct cfv_info *cfv = vdev->priv;
353
354 ++cfv->stats.rx_kicks;
355 vringh_notify_disable_kern(cfv->vr_rx);
356 napi_schedule(&cfv->napi);
357}
358
359static void cfv_destroy_genpool(struct cfv_info *cfv)
360{
361 if (cfv->alloc_addr)
362 dma_free_coherent(cfv->vdev->dev.parent->parent,
363 cfv->allocsz, cfv->alloc_addr,
364 cfv->alloc_dma);
365
366 if (!cfv->genpool)
367 return;
368 gen_pool_free(cfv->genpool, cfv->reserved_mem,
369 cfv->reserved_size);
370 gen_pool_destroy(cfv->genpool);
371 cfv->genpool = NULL;
372}
373
374static int cfv_create_genpool(struct cfv_info *cfv)
375{
376 int err;
377
378 /* dma_alloc can only allocate whole pages, and we need a more
379 * fine graned allocation so we use genpool. We ask for space needed
380 * by IP and a full ring. If the dma allcoation fails we retry with a
381 * smaller allocation size.
382 */
383 err = -ENOMEM;
384 cfv->allocsz = (virtqueue_get_vring_size(cfv->vq_tx) *
385 (ETH_DATA_LEN + cfv->tx_hr + cfv->tx_tr) * 11)/10;
386 if (cfv->allocsz <= (num_possible_cpus() + 1) * cfv->ndev->mtu)
387 return -EINVAL;
388
389 for (;;) {
390 if (cfv->allocsz <= num_possible_cpus() * cfv->ndev->mtu) {
391 netdev_info(cfv->ndev, "Not enough device memory\n");
392 return -ENOMEM;
393 }
394
395 cfv->alloc_addr = dma_alloc_coherent(
396 cfv->vdev->dev.parent->parent,
397 cfv->allocsz, &cfv->alloc_dma,
398 GFP_ATOMIC);
399 if (cfv->alloc_addr)
400 break;
401
402 cfv->allocsz = (cfv->allocsz * 3) >> 2;
403 }
404
405 netdev_dbg(cfv->ndev, "Allocated %zd bytes from dma-memory\n",
406 cfv->allocsz);
407
408 /* Allocate on 128 bytes boundaries (1 << 7)*/
409 cfv->genpool = gen_pool_create(7, -1);
410 if (!cfv->genpool)
411 goto err;
412
413 err = gen_pool_add_virt(cfv->genpool, (unsigned long)cfv->alloc_addr,
414 (phys_addr_t)virt_to_phys(cfv->alloc_addr),
415 cfv->allocsz, -1);
416 if (err)
417 goto err;
418
419 /* Reserve some memory for low memory situations. If we hit the roof
420 * in the memory pool, we stop TX flow and release the reserve.
421 */
422 cfv->reserved_size = num_possible_cpus() * cfv->ndev->mtu;
423 cfv->reserved_mem = gen_pool_alloc(cfv->genpool,
424 cfv->reserved_size);
Wei Yongjun1aef76e2013-04-02 16:45:56 +1030425 if (!cfv->reserved_mem) {
426 err = -ENOMEM;
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030427 goto err;
Wei Yongjun1aef76e2013-04-02 16:45:56 +1030428 }
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030429
430 cfv->watermark_tx = virtqueue_get_vring_size(cfv->vq_tx);
431 return 0;
432err:
433 cfv_destroy_genpool(cfv);
434 return err;
435}
436
437/* Enable the CAIF interface and allocate the memory-pool */
438static int cfv_netdev_open(struct net_device *netdev)
439{
440 struct cfv_info *cfv = netdev_priv(netdev);
441
442 if (cfv_create_genpool(cfv))
443 return -ENOMEM;
444
445 netif_carrier_on(netdev);
446 napi_enable(&cfv->napi);
447
448 /* Schedule NAPI to read any pending packets */
449 napi_schedule(&cfv->napi);
450 return 0;
451}
452
453/* Disable the CAIF interface and free the memory-pool */
454static int cfv_netdev_close(struct net_device *netdev)
455{
456 struct cfv_info *cfv = netdev_priv(netdev);
457 unsigned long flags;
458 struct buf_info *buf_info;
459
460 /* Disable interrupts, queues and NAPI polling */
461 netif_carrier_off(netdev);
462 virtqueue_disable_cb(cfv->vq_tx);
463 vringh_notify_disable_kern(cfv->vr_rx);
464 napi_disable(&cfv->napi);
465
466 /* Release any TX buffers on both used and avilable rings */
467 cfv_release_used_buf(cfv->vq_tx);
468 spin_lock_irqsave(&cfv->tx_lock, flags);
469 while ((buf_info = virtqueue_detach_unused_buf(cfv->vq_tx)))
470 free_buf_info(cfv, buf_info);
471 spin_unlock_irqrestore(&cfv->tx_lock, flags);
472
473 /* Release all dma allocated memory and destroy the pool */
474 cfv_destroy_genpool(cfv);
475 return 0;
476}
477
478/* Allocate a buffer in dma-memory and copy skb to it */
479static struct buf_info *cfv_alloc_and_copy_to_shm(struct cfv_info *cfv,
480 struct sk_buff *skb,
481 struct scatterlist *sg)
482{
483 struct caif_payload_info *info = (void *)&skb->cb;
484 struct buf_info *buf_info = NULL;
485 u8 pad_len, hdr_ofs;
486
487 if (!cfv->genpool)
488 goto err;
489
490 if (unlikely(cfv->tx_hr + skb->len + cfv->tx_tr > cfv->mtu)) {
491 netdev_warn(cfv->ndev, "Invalid packet len (%d > %d)\n",
492 cfv->tx_hr + skb->len + cfv->tx_tr, cfv->mtu);
493 goto err;
494 }
495
496 buf_info = kmalloc(sizeof(struct buf_info), GFP_ATOMIC);
497 if (unlikely(!buf_info))
498 goto err;
499
500 /* Make the IP header aligned in tbe buffer */
501 hdr_ofs = cfv->tx_hr + info->hdr_len;
502 pad_len = hdr_ofs & (IP_HDR_ALIGN - 1);
503 buf_info->size = cfv->tx_hr + skb->len + cfv->tx_tr + pad_len;
504
505 /* allocate dma memory buffer */
506 buf_info->vaddr = (void *)gen_pool_alloc(cfv->genpool, buf_info->size);
507 if (unlikely(!buf_info->vaddr))
508 goto err;
509
510 /* copy skbuf contents to send buffer */
511 skb_copy_bits(skb, 0, buf_info->vaddr + cfv->tx_hr + pad_len, skb->len);
512 sg_init_one(sg, buf_info->vaddr + pad_len,
513 skb->len + cfv->tx_hr + cfv->rx_hr);
514
515 return buf_info;
516err:
517 kfree(buf_info);
518 return NULL;
519}
520
521/* Put the CAIF packet on the virtio ring and kick the receiver */
522static int cfv_netdev_tx(struct sk_buff *skb, struct net_device *netdev)
523{
524 struct cfv_info *cfv = netdev_priv(netdev);
525 struct buf_info *buf_info;
526 struct scatterlist sg;
527 unsigned long flags;
528 bool flow_off = false;
529 int ret;
530
531 /* garbage collect released buffers */
532 cfv_release_used_buf(cfv->vq_tx);
533 spin_lock_irqsave(&cfv->tx_lock, flags);
534
535 /* Flow-off check takes into account number of cpus to make sure
536 * virtqueue will not be overfilled in any possible smp conditions.
537 *
538 * Flow-on is triggered when sufficient buffers are freed
539 */
540 if (unlikely(cfv->vq_tx->num_free <= num_present_cpus())) {
541 flow_off = true;
542 cfv->stats.tx_full_ring++;
543 }
544
545 /* If we run out of memory, we release the memory reserve and retry
546 * allocation.
547 */
548 buf_info = cfv_alloc_and_copy_to_shm(cfv, skb, &sg);
549 if (unlikely(!buf_info)) {
550 cfv->stats.tx_no_mem++;
551 flow_off = true;
552
553 if (cfv->reserved_mem && cfv->genpool) {
554 gen_pool_free(cfv->genpool, cfv->reserved_mem,
555 cfv->reserved_size);
556 cfv->reserved_mem = 0;
557 buf_info = cfv_alloc_and_copy_to_shm(cfv, skb, &sg);
558 }
559 }
560
561 if (unlikely(flow_off)) {
562 /* Turn flow on when a 1/4 of the descriptors are released */
563 cfv->watermark_tx = virtqueue_get_vring_size(cfv->vq_tx) / 4;
564 /* Enable notifications of recycled TX buffers */
565 virtqueue_enable_cb(cfv->vq_tx);
566 netif_tx_stop_all_queues(netdev);
567 }
568
569 if (unlikely(!buf_info)) {
570 /* If the memory reserve does it's job, this shouldn't happen */
571 netdev_warn(cfv->ndev, "Out of gen_pool memory\n");
572 goto err;
573 }
574
Rusty Russell71bcbec2013-03-20 15:44:29 +1030575 ret = virtqueue_add_outbuf(cfv->vq_tx, &sg, 1, buf_info, GFP_ATOMIC);
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030576 if (unlikely((ret < 0))) {
577 /* If flow control works, this shouldn't happen */
578 netdev_warn(cfv->ndev, "Failed adding buffer to TX vring:%d\n",
579 ret);
580 goto err;
581 }
582
583 /* update netdev statistics */
584 cfv->ndev->stats.tx_packets++;
585 cfv->ndev->stats.tx_bytes += skb->len;
586 spin_unlock_irqrestore(&cfv->tx_lock, flags);
587
588 /* tell the remote processor it has a pending message to read */
589 virtqueue_kick(cfv->vq_tx);
590
591 dev_kfree_skb(skb);
592 return NETDEV_TX_OK;
593err:
594 spin_unlock_irqrestore(&cfv->tx_lock, flags);
595 cfv->ndev->stats.tx_dropped++;
596 free_buf_info(cfv, buf_info);
597 dev_kfree_skb(skb);
598 return NETDEV_TX_OK;
599}
600
601static void cfv_tx_release_tasklet(unsigned long drv)
602{
603 struct cfv_info *cfv = (struct cfv_info *)drv;
604 cfv_release_used_buf(cfv->vq_tx);
605}
606
607static const struct net_device_ops cfv_netdev_ops = {
608 .ndo_open = cfv_netdev_open,
609 .ndo_stop = cfv_netdev_close,
610 .ndo_start_xmit = cfv_netdev_tx,
611};
612
613static void cfv_netdev_setup(struct net_device *netdev)
614{
615 netdev->netdev_ops = &cfv_netdev_ops;
616 netdev->type = ARPHRD_CAIF;
617 netdev->tx_queue_len = 100;
618 netdev->flags = IFF_POINTOPOINT | IFF_NOARP;
619 netdev->mtu = CFV_DEF_MTU_SIZE;
620 netdev->destructor = free_netdev;
621}
622
623/* Create debugfs counters for the device */
624static inline void debugfs_init(struct cfv_info *cfv)
625{
626 cfv->debugfs =
627 debugfs_create_dir(netdev_name(cfv->ndev), NULL);
628
629 if (IS_ERR(cfv->debugfs))
630 return;
631
632 debugfs_create_u32("rx-napi-complete", S_IRUSR, cfv->debugfs,
633 &cfv->stats.rx_napi_complete);
634 debugfs_create_u32("rx-napi-resched", S_IRUSR, cfv->debugfs,
635 &cfv->stats.rx_napi_resched);
636 debugfs_create_u32("rx-nomem", S_IRUSR, cfv->debugfs,
637 &cfv->stats.rx_nomem);
638 debugfs_create_u32("rx-kicks", S_IRUSR, cfv->debugfs,
639 &cfv->stats.rx_kicks);
640 debugfs_create_u32("tx-full-ring", S_IRUSR, cfv->debugfs,
641 &cfv->stats.tx_full_ring);
642 debugfs_create_u32("tx-no-mem", S_IRUSR, cfv->debugfs,
643 &cfv->stats.tx_no_mem);
644 debugfs_create_u32("tx-kicks", S_IRUSR, cfv->debugfs,
645 &cfv->stats.tx_kicks);
646 debugfs_create_u32("tx-flow-on", S_IRUSR, cfv->debugfs,
647 &cfv->stats.tx_flow_on);
648}
649
650/* Setup CAIF for the a virtio device */
651static int cfv_probe(struct virtio_device *vdev)
652{
653 vq_callback_t *vq_cbs = cfv_release_cb;
654 vrh_callback_t *vrh_cbs = cfv_recv;
655 const char *names = "output";
656 const char *cfv_netdev_name = "cfvrt";
657 struct net_device *netdev;
658 struct cfv_info *cfv;
659 int err = -EINVAL;
660
661 netdev = alloc_netdev(sizeof(struct cfv_info), cfv_netdev_name,
Tom Gundersenc835a672014-07-14 16:37:24 +0200662 NET_NAME_UNKNOWN, cfv_netdev_setup);
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030663 if (!netdev)
664 return -ENOMEM;
665
666 cfv = netdev_priv(netdev);
667 cfv->vdev = vdev;
668 cfv->ndev = netdev;
669
670 spin_lock_init(&cfv->tx_lock);
671
672 /* Get the RX virtio ring. This is a "host side vring". */
Sjur Brændelanda8c76872013-03-24 14:19:59 +1030673 err = -ENODEV;
674 if (!vdev->vringh_config || !vdev->vringh_config->find_vrhs)
675 goto err;
676
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030677 err = vdev->vringh_config->find_vrhs(vdev, 1, &cfv->vr_rx, &vrh_cbs);
678 if (err)
679 goto err;
680
681 /* Get the TX virtio ring. This is a "guest side vring". */
682 err = vdev->config->find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names);
683 if (err)
684 goto err;
685
686 /* Get the CAIF configuration from virtio config space, if available */
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030687 if (vdev->config->get) {
Rusty Russell855e0c52013-10-14 18:11:51 +1030688 virtio_cread(vdev, struct virtio_caif_transf_config, headroom,
689 &cfv->tx_hr);
690 virtio_cread(vdev, struct virtio_caif_transf_config, headroom,
691 &cfv->rx_hr);
692 virtio_cread(vdev, struct virtio_caif_transf_config, tailroom,
693 &cfv->tx_tr);
694 virtio_cread(vdev, struct virtio_caif_transf_config, tailroom,
695 &cfv->rx_tr);
696 virtio_cread(vdev, struct virtio_caif_transf_config, mtu,
697 &cfv->mtu);
698 virtio_cread(vdev, struct virtio_caif_transf_config, mtu,
699 &cfv->mru);
Erwan Yvin0d2e1a22013-03-20 13:52:24 +1030700 } else {
701 cfv->tx_hr = CFV_DEF_HEADROOM;
702 cfv->rx_hr = CFV_DEF_HEADROOM;
703 cfv->tx_tr = CFV_DEF_TAILROOM;
704 cfv->rx_tr = CFV_DEF_TAILROOM;
705 cfv->mtu = CFV_DEF_MTU_SIZE;
706 cfv->mru = CFV_DEF_MTU_SIZE;
707 }
708
709 netdev->needed_headroom = cfv->tx_hr;
710 netdev->needed_tailroom = cfv->tx_tr;
711
712 /* Disable buffer release interrupts unless we have stopped TX queues */
713 virtqueue_disable_cb(cfv->vq_tx);
714
715 netdev->mtu = cfv->mtu - cfv->tx_tr;
716 vdev->priv = cfv;
717
718 /* Initialize NAPI poll context data */
719 vringh_kiov_init(&cfv->ctx.riov, NULL, 0);
720 cfv->ctx.head = USHRT_MAX;
721 netif_napi_add(netdev, &cfv->napi, cfv_rx_poll, CFV_DEFAULT_QUOTA);
722
723 tasklet_init(&cfv->tx_release_tasklet,
724 cfv_tx_release_tasklet,
725 (unsigned long)cfv);
726
727 /* Carrier is off until netdevice is opened */
728 netif_carrier_off(netdev);
729
730 /* register Netdev */
731 err = register_netdev(netdev);
732 if (err) {
733 dev_err(&vdev->dev, "Unable to register netdev (%d)\n", err);
734 goto err;
735 }
736
737 debugfs_init(cfv);
738
739 return 0;
740err:
741 netdev_warn(cfv->ndev, "CAIF Virtio probe failed:%d\n", err);
742
743 if (cfv->vr_rx)
744 vdev->vringh_config->del_vrhs(cfv->vdev);
745 if (cfv->vdev)
746 vdev->config->del_vqs(cfv->vdev);
747 free_netdev(netdev);
748 return err;
749}
750
751static void cfv_remove(struct virtio_device *vdev)
752{
753 struct cfv_info *cfv = vdev->priv;
754
755 rtnl_lock();
756 dev_close(cfv->ndev);
757 rtnl_unlock();
758
759 tasklet_kill(&cfv->tx_release_tasklet);
760 debugfs_remove_recursive(cfv->debugfs);
761
762 vringh_kiov_cleanup(&cfv->ctx.riov);
763 vdev->config->reset(vdev);
764 vdev->vringh_config->del_vrhs(cfv->vdev);
765 cfv->vr_rx = NULL;
766 vdev->config->del_vqs(cfv->vdev);
767 unregister_netdev(cfv->ndev);
768}
769
770static struct virtio_device_id id_table[] = {
771 { VIRTIO_ID_CAIF, VIRTIO_DEV_ANY_ID },
772 { 0 },
773};
774
775static unsigned int features[] = {
776};
777
778static struct virtio_driver caif_virtio_driver = {
779 .feature_table = features,
780 .feature_table_size = ARRAY_SIZE(features),
781 .driver.name = KBUILD_MODNAME,
782 .driver.owner = THIS_MODULE,
783 .id_table = id_table,
784 .probe = cfv_probe,
785 .remove = cfv_remove,
786};
787
788module_virtio_driver(caif_virtio_driver);
789MODULE_DEVICE_TABLE(virtio, id_table);