USB: Add support for usbfs zerocopy.
Add a new interface for userspace to preallocate memory that can be
used with usbfs. This gives two primary benefits:
- Zerocopy; data no longer needs to be copied between the userspace
and the kernel, but can instead be read directly by the driver from
userspace's buffers. This works for all kinds of transfers (even if
nonsensical for control and interrupt transfers); isochronous also
no longer need to memset() the buffer to zero to avoid leaking kernel data.
- Once the buffers are allocated, USB transfers can no longer fail due to
memory fragmentation; previously, long-running programs could run into
problems finding a large enough contiguous memory chunk, especially on
embedded systems or at high rates.
Memory is allocated by using mmap() against the usbfs file descriptor,
and similarly deallocated by munmap(). Once memory has been allocated,
using it as pointers to a bulk or isochronous operation means you will
automatically get zerocopy behavior. Note that this also means you cannot
modify outgoing data until the transfer is complete. The same holds for
data on the same cache lines as incoming data; DMA modifying them at the
same time could lead to your changes being overwritten.
There's a new capability USBDEVFS_CAP_MMAP that userspace can query to see
if the running kernel supports this functionality, if just trying mmap() is
not acceptable.
Largely based on a patch by Markus Rechberger with some updates. The original
patch can be found at:
http://sundtek.de/support/devio_mmap_v0.4.diff
Signed-off-by: Steinar H. Gunderson <sesse@google.com>
Signed-off-by: Markus Rechberger <mrechberger@gmail.com>
Acked-by: Alan Stern <stern@rowland.harvard.edu>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
index 13a4b9f..39da166 100644
--- a/drivers/usb/core/devio.c
+++ b/drivers/usb/core/devio.c
@@ -50,6 +50,7 @@
#include <linux/user_namespace.h>
#include <linux/scatterlist.h>
#include <linux/uaccess.h>
+#include <linux/dma-mapping.h>
#include <asm/byteorder.h>
#include <linux/moduleparam.h>
@@ -69,6 +70,7 @@
spinlock_t lock; /* protects the async urb lists */
struct list_head async_pending;
struct list_head async_completed;
+ struct list_head memory_list;
wait_queue_head_t wait; /* wake up if a request completed */
unsigned int discsignr;
struct pid *disc_pid;
@@ -79,6 +81,17 @@
u32 disabled_bulk_eps;
};
+struct usb_memory {
+ struct list_head memlist;
+ int vma_use_count;
+ int urb_use_count;
+ u32 size;
+ void *mem;
+ dma_addr_t dma_handle;
+ unsigned long vm_start;
+ struct usb_dev_state *ps;
+};
+
struct async {
struct list_head asynclist;
struct usb_dev_state *ps;
@@ -89,6 +102,7 @@
void __user *userbuffer;
void __user *userurb;
struct urb *urb;
+ struct usb_memory *usbm;
unsigned int mem_usage;
int status;
u32 secid;
@@ -162,6 +176,111 @@
ps->dev->state != USB_STATE_NOTATTACHED);
}
+static void dec_usb_memory_use_count(struct usb_memory *usbm, int *count)
+{
+ struct usb_dev_state *ps = usbm->ps;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ps->lock, flags);
+ --*count;
+ if (usbm->urb_use_count == 0 && usbm->vma_use_count == 0) {
+ list_del(&usbm->memlist);
+ spin_unlock_irqrestore(&ps->lock, flags);
+
+ usb_free_coherent(ps->dev, usbm->size, usbm->mem,
+ usbm->dma_handle);
+ usbfs_decrease_memory_usage(
+ usbm->size + sizeof(struct usb_memory));
+ kfree(usbm);
+ } else {
+ spin_unlock_irqrestore(&ps->lock, flags);
+ }
+}
+
+static void usbdev_vm_open(struct vm_area_struct *vma)
+{
+ struct usb_memory *usbm = vma->vm_private_data;
+ unsigned long flags;
+
+ spin_lock_irqsave(&usbm->ps->lock, flags);
+ ++usbm->vma_use_count;
+ spin_unlock_irqrestore(&usbm->ps->lock, flags);
+}
+
+static void usbdev_vm_close(struct vm_area_struct *vma)
+{
+ struct usb_memory *usbm = vma->vm_private_data;
+
+ dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
+}
+
+struct vm_operations_struct usbdev_vm_ops = {
+ .open = usbdev_vm_open,
+ .close = usbdev_vm_close
+};
+
+static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct usb_memory *usbm = NULL;
+ struct usb_dev_state *ps = file->private_data;
+ size_t size = vma->vm_end - vma->vm_start;
+ void *mem;
+ unsigned long flags;
+ dma_addr_t dma_handle;
+ int ret;
+
+ ret = usbfs_increase_memory_usage(size + sizeof(struct usb_memory));
+ if (ret)
+ goto error;
+
+ usbm = kzalloc(sizeof(struct usb_memory), GFP_KERNEL);
+ if (!usbm) {
+ ret = -ENOMEM;
+ goto error_decrease_mem;
+ }
+
+ mem = usb_alloc_coherent(ps->dev, size, GFP_USER, &dma_handle);
+ if (!mem) {
+ ret = -ENOMEM;
+ goto error_free_usbm;
+ }
+
+ memset(mem, 0, size);
+
+ usbm->mem = mem;
+ usbm->dma_handle = dma_handle;
+ usbm->size = size;
+ usbm->ps = ps;
+ usbm->vm_start = vma->vm_start;
+ usbm->vma_use_count = 1;
+ INIT_LIST_HEAD(&usbm->memlist);
+
+ if (remap_pfn_range(vma, vma->vm_start,
+ virt_to_phys(usbm->mem) >> PAGE_SHIFT,
+ size, vma->vm_page_prot) < 0) {
+ dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
+ return -EAGAIN;
+ }
+
+ vma->vm_flags |= VM_IO;
+ vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
+ vma->vm_ops = &usbdev_vm_ops;
+ vma->vm_private_data = usbm;
+
+ spin_lock_irqsave(&ps->lock, flags);
+ list_add_tail(&usbm->memlist, &ps->memory_list);
+ spin_unlock_irqrestore(&ps->lock, flags);
+
+ return 0;
+
+error_free_usbm:
+ kfree(usbm);
+error_decrease_mem:
+ usbfs_decrease_memory_usage(size + sizeof(struct usb_memory));
+error:
+ return ret;
+}
+
static ssize_t usbdev_read(struct file *file, char __user *buf, size_t nbytes,
loff_t *ppos)
{
@@ -278,8 +397,13 @@
if (sg_page(&as->urb->sg[i]))
kfree(sg_virt(&as->urb->sg[i]));
}
+
kfree(as->urb->sg);
- kfree(as->urb->transfer_buffer);
+ if (as->usbm == NULL)
+ kfree(as->urb->transfer_buffer);
+ else
+ dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
+
kfree(as->urb->setup_packet);
usb_free_urb(as->urb);
usbfs_decrease_memory_usage(as->mem_usage);
@@ -893,6 +1017,7 @@
INIT_LIST_HEAD(&ps->list);
INIT_LIST_HEAD(&ps->async_pending);
INIT_LIST_HEAD(&ps->async_completed);
+ INIT_LIST_HEAD(&ps->memory_list);
init_waitqueue_head(&ps->wait);
ps->discsignr = 0;
ps->disc_pid = get_pid(task_pid(current));
@@ -945,6 +1070,7 @@
free_async(as);
as = async_getcompleted(ps);
}
+
kfree(ps);
return 0;
}
@@ -1266,6 +1392,31 @@
return status;
}
+static struct usb_memory *
+find_memory_area(struct usb_dev_state *ps, const struct usbdevfs_urb *uurb)
+{
+ struct usb_memory *usbm = NULL, *iter;
+ unsigned long flags;
+ unsigned long uurb_start = (unsigned long)uurb->buffer;
+
+ spin_lock_irqsave(&ps->lock, flags);
+ list_for_each_entry(iter, &ps->memory_list, memlist) {
+ if (uurb_start >= iter->vm_start &&
+ uurb_start < iter->vm_start + iter->size) {
+ if (uurb->buffer_length > iter->vm_start + iter->size -
+ uurb_start) {
+ usbm = ERR_PTR(-EINVAL);
+ } else {
+ usbm = iter;
+ usbm->urb_use_count++;
+ }
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&ps->lock, flags);
+ return usbm;
+}
+
static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb,
struct usbdevfs_iso_packet_desc __user *iso_frame_desc,
void __user *arg)
@@ -1421,6 +1572,19 @@
goto error;
}
+ as->usbm = find_memory_area(ps, uurb);
+ if (IS_ERR(as->usbm)) {
+ ret = PTR_ERR(as->usbm);
+ as->usbm = NULL;
+ goto error;
+ }
+
+ /* do not use SG buffers when memory mapped segments
+ * are in use
+ */
+ if (as->usbm)
+ num_sgs = 0;
+
u += sizeof(struct async) + sizeof(struct urb) + uurb->buffer_length +
num_sgs * sizeof(struct scatterlist);
ret = usbfs_increase_memory_usage(u);
@@ -1458,29 +1622,35 @@
totlen -= u;
}
} else if (uurb->buffer_length > 0) {
- as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
- GFP_KERNEL);
- if (!as->urb->transfer_buffer) {
- ret = -ENOMEM;
- goto error;
- }
+ if (as->usbm) {
+ unsigned long uurb_start = (unsigned long)uurb->buffer;
- if (!is_in) {
- if (copy_from_user(as->urb->transfer_buffer,
- uurb->buffer,
- uurb->buffer_length)) {
- ret = -EFAULT;
+ as->urb->transfer_buffer = as->usbm->mem +
+ (uurb_start - as->usbm->vm_start);
+ } else {
+ as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
+ GFP_KERNEL);
+ if (!as->urb->transfer_buffer) {
+ ret = -ENOMEM;
goto error;
}
- } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
- /*
- * Isochronous input data may end up being
- * discontiguous if some of the packets are short.
- * Clear the buffer so that the gaps don't leak
- * kernel data to userspace.
- */
- memset(as->urb->transfer_buffer, 0,
- uurb->buffer_length);
+ if (!is_in) {
+ if (copy_from_user(as->urb->transfer_buffer,
+ uurb->buffer,
+ uurb->buffer_length)) {
+ ret = -EFAULT;
+ goto error;
+ }
+ } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
+ /*
+ * Isochronous input data may end up being
+ * discontiguous if some of the packets are
+ * short. Clear the buffer so that the gaps
+ * don't leak kernel data to userspace.
+ */
+ memset(as->urb->transfer_buffer, 0,
+ uurb->buffer_length);
+ }
}
}
as->urb->dev = ps->dev;
@@ -1527,10 +1697,14 @@
isopkt = NULL;
as->ps = ps;
as->userurb = arg;
- if (is_in && uurb->buffer_length > 0)
+ if (as->usbm) {
+ unsigned long uurb_start = (unsigned long)uurb->buffer;
+
+ as->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
+ as->urb->transfer_dma = as->usbm->dma_handle +
+ (uurb_start - as->usbm->vm_start);
+ } else if (is_in && uurb->buffer_length > 0)
as->userbuffer = uurb->buffer;
- else
- as->userbuffer = NULL;
as->signr = uurb->signr;
as->ifnum = ifnum;
as->pid = get_pid(task_pid(current));
@@ -1586,6 +1760,8 @@
return 0;
error:
+ if (as && as->usbm)
+ dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
kfree(isopkt);
kfree(dr);
if (as)
@@ -2039,7 +2215,7 @@
__u32 caps;
caps = USBDEVFS_CAP_ZERO_PACKET | USBDEVFS_CAP_NO_PACKET_SIZE_LIM |
- USBDEVFS_CAP_REAP_AFTER_DISCONNECT;
+ USBDEVFS_CAP_REAP_AFTER_DISCONNECT | USBDEVFS_CAP_MMAP;
if (!ps->dev->bus->no_stop_on_short)
caps |= USBDEVFS_CAP_BULK_CONTINUATION;
if (ps->dev->bus->sg_tablesize)
@@ -2365,6 +2541,7 @@
#ifdef CONFIG_COMPAT
.compat_ioctl = usbdev_compat_ioctl,
#endif
+ .mmap = usbdev_mmap,
.open = usbdev_open,
.release = usbdev_release,
};