Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Pierre Ossman | 70f1048 | 2007-07-11 20:04:50 +0200 | [diff] [blame] | 2 | * linux/drivers/mmc/card/queue.c |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2003 Russell King, All Rights Reserved. |
Pierre Ossman | 98ac216 | 2006-12-23 20:03:02 +0100 | [diff] [blame] | 5 | * Copyright 2006-2007 Pierre Ossman |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License version 2 as |
| 9 | * published by the Free Software Foundation. |
| 10 | * |
| 11 | */ |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 12 | #include <linux/slab.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <linux/module.h> |
| 14 | #include <linux/blkdev.h> |
Rafael J. Wysocki | 8314418 | 2007-07-17 04:03:35 -0700 | [diff] [blame] | 15 | #include <linux/freezer.h> |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 16 | #include <linux/kthread.h> |
Jens Axboe | 45711f1 | 2007-10-22 21:19:53 +0200 | [diff] [blame] | 17 | #include <linux/scatterlist.h> |
Santosh Shilimkar | 8e0cb8a | 2013-07-29 14:20:15 +0100 | [diff] [blame] | 18 | #include <linux/dma-mapping.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 19 | |
| 20 | #include <linux/mmc/card.h> |
| 21 | #include <linux/mmc/host.h> |
Tim Murray | 01d8cc5 | 2016-01-19 16:36:40 -0800 | [diff] [blame] | 22 | #include <linux/sched/rt.h> |
Linus Walleij | 29eb7bd | 2016-09-20 11:34:38 +0200 | [diff] [blame] | 23 | |
Pierre Ossman | 98ac216 | 2006-12-23 20:03:02 +0100 | [diff] [blame] | 24 | #include "queue.h" |
Linus Walleij | 29eb7bd | 2016-09-20 11:34:38 +0200 | [diff] [blame] | 25 | #include "block.h" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 27 | #define MMC_QUEUE_BOUNCESZ 65536 |
| 28 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | /* |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 30 | * Prepare a MMC request. This just filters out odd stuff. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | */ |
| 32 | static int mmc_prep_request(struct request_queue *q, struct request *req) |
| 33 | { |
Sujit Reddy Thumma | a8ad82cc | 2011-12-08 14:05:50 +0530 | [diff] [blame] | 34 | struct mmc_queue *mq = q->queuedata; |
| 35 | |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 36 | /* |
Adrian Hunter | bd788c9 | 2010-08-11 14:17:47 -0700 | [diff] [blame] | 37 | * We only like normal block requests and discards. |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 38 | */ |
Adrian Hunter | 7afafc8 | 2016-08-16 10:59:35 +0300 | [diff] [blame] | 39 | if (req->cmd_type != REQ_TYPE_FS && req_op(req) != REQ_OP_DISCARD && |
| 40 | req_op(req) != REQ_OP_SECURE_ERASE) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | blk_dump_rq_flags(req, "MMC bad request"); |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 42 | return BLKPREP_KILL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 43 | } |
| 44 | |
Chuanxiao Dong | 4e93b9a | 2014-08-12 12:01:30 +0800 | [diff] [blame] | 45 | if (mq && (mmc_card_removed(mq->card) || mmc_access_rpmb(mq))) |
Sujit Reddy Thumma | a8ad82cc | 2011-12-08 14:05:50 +0530 | [diff] [blame] | 46 | return BLKPREP_KILL; |
| 47 | |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 48 | req->cmd_flags |= REQ_DONTPREP; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 50 | return BLKPREP_OK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 51 | } |
| 52 | |
| 53 | static int mmc_queue_thread(void *d) |
| 54 | { |
| 55 | struct mmc_queue *mq = d; |
| 56 | struct request_queue *q = mq->queue; |
Tim Murray | 01d8cc5 | 2016-01-19 16:36:40 -0800 | [diff] [blame] | 57 | struct sched_param scheduler_params = {0}; |
| 58 | |
| 59 | scheduler_params.sched_priority = 1; |
| 60 | |
| 61 | sched_setscheduler(current, SCHED_FIFO, &scheduler_params); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | |
Rafael J. Wysocki | 8314418 | 2007-07-17 04:03:35 -0700 | [diff] [blame] | 63 | current->flags |= PF_MEMALLOC; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | down(&mq->thread_sem); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 66 | do { |
| 67 | struct request *req = NULL; |
| 68 | |
| 69 | spin_lock_irq(q->queue_lock); |
| 70 | set_current_state(TASK_INTERRUPTIBLE); |
Jens Axboe | 7eaceac | 2011-03-10 08:52:07 +0100 | [diff] [blame] | 71 | req = blk_fetch_request(q); |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 72 | mq->mqrq_cur->req = req; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | spin_unlock_irq(q->queue_lock); |
| 74 | |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 75 | if (req || mq->mqrq_prev->req) { |
Adrian Hunter | 869c554 | 2016-08-25 14:11:43 -0600 | [diff] [blame] | 76 | bool req_is_special = mmc_req_is_special(req); |
| 77 | |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 78 | set_current_state(TASK_RUNNING); |
Linus Walleij | 29eb7bd | 2016-09-20 11:34:38 +0200 | [diff] [blame] | 79 | mmc_blk_issue_rq(mq, req); |
Rabin Vincent | a8c27c0 | 2015-06-14 19:26:11 +0200 | [diff] [blame] | 80 | cond_resched(); |
Konstantin Dorfman | 2220eed | 2013-01-14 14:28:17 -0500 | [diff] [blame] | 81 | if (mq->flags & MMC_QUEUE_NEW_REQUEST) { |
| 82 | mq->flags &= ~MMC_QUEUE_NEW_REQUEST; |
| 83 | continue; /* fetch again */ |
| 84 | } |
Seungwon Jeon | 45c5a91 | 2012-09-28 19:12:53 +0900 | [diff] [blame] | 85 | |
| 86 | /* |
| 87 | * Current request becomes previous request |
| 88 | * and vice versa. |
Seungwon Jeon | 369d321 | 2012-12-26 10:40:17 +0900 | [diff] [blame] | 89 | * In case of special requests, current request |
| 90 | * has been finished. Do not assign it to previous |
| 91 | * request. |
Seungwon Jeon | 45c5a91 | 2012-09-28 19:12:53 +0900 | [diff] [blame] | 92 | */ |
Adrian Hunter | 869c554 | 2016-08-25 14:11:43 -0600 | [diff] [blame] | 93 | if (req_is_special) |
Seungwon Jeon | 369d321 | 2012-12-26 10:40:17 +0900 | [diff] [blame] | 94 | mq->mqrq_cur->req = NULL; |
| 95 | |
Seungwon Jeon | 45c5a91 | 2012-09-28 19:12:53 +0900 | [diff] [blame] | 96 | mq->mqrq_prev->brq.mrq.data = NULL; |
| 97 | mq->mqrq_prev->req = NULL; |
Fabian Frederick | 7551847 | 2015-06-10 18:30:53 +0200 | [diff] [blame] | 98 | swap(mq->mqrq_prev, mq->mqrq_cur); |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 99 | } else { |
Vitaly Wool | 7b30d28 | 2006-12-07 20:08:02 +0100 | [diff] [blame] | 100 | if (kthread_should_stop()) { |
| 101 | set_current_state(TASK_RUNNING); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | break; |
Vitaly Wool | 7b30d28 | 2006-12-07 20:08:02 +0100 | [diff] [blame] | 103 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | up(&mq->thread_sem); |
| 105 | schedule(); |
| 106 | down(&mq->thread_sem); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 107 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | } while (1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | up(&mq->thread_sem); |
| 110 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 111 | return 0; |
| 112 | } |
| 113 | |
| 114 | /* |
| 115 | * Generic MMC request handler. This is called for any queue on a |
| 116 | * particular host. When the host is not busy, we look for a request |
| 117 | * on any queue on this host, and attempt to issue it. This may |
| 118 | * not be the queue we were asked to process. |
| 119 | */ |
Venkatraman S | 1b50f5f | 2012-04-13 17:54:11 +0530 | [diff] [blame] | 120 | static void mmc_request_fn(struct request_queue *q) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | { |
| 122 | struct mmc_queue *mq = q->queuedata; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 123 | struct request *req; |
Konstantin Dorfman | 2220eed | 2013-01-14 14:28:17 -0500 | [diff] [blame] | 124 | unsigned long flags; |
| 125 | struct mmc_context_info *cntx; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 126 | |
| 127 | if (!mq) { |
Adrian Hunter | 5fa83ce | 2010-01-08 14:43:00 -0800 | [diff] [blame] | 128 | while ((req = blk_fetch_request(q)) != NULL) { |
| 129 | req->cmd_flags |= REQ_QUIET; |
Tejun Heo | 296b2f6 | 2009-05-08 11:54:15 +0900 | [diff] [blame] | 130 | __blk_end_request_all(req, -EIO); |
Adrian Hunter | 5fa83ce | 2010-01-08 14:43:00 -0800 | [diff] [blame] | 131 | } |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 132 | return; |
| 133 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | |
Konstantin Dorfman | 2220eed | 2013-01-14 14:28:17 -0500 | [diff] [blame] | 135 | cntx = &mq->card->host->context_info; |
| 136 | if (!mq->mqrq_cur->req && mq->mqrq_prev->req) { |
| 137 | /* |
| 138 | * New MMC request arrived when MMC thread may be |
| 139 | * blocked on the previous request to be complete |
| 140 | * with no current request fetched |
| 141 | */ |
| 142 | spin_lock_irqsave(&cntx->lock, flags); |
| 143 | if (cntx->is_waiting_last_req) { |
| 144 | cntx->is_new_req = true; |
| 145 | wake_up_interruptible(&cntx->wait); |
| 146 | } |
| 147 | spin_unlock_irqrestore(&cntx->lock, flags); |
| 148 | } else if (!mq->mqrq_cur->req && !mq->mqrq_prev->req) |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 149 | wake_up_process(mq->thread); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 150 | } |
| 151 | |
Venkatraman S | 7513cd7 | 2011-08-23 21:16:02 +0530 | [diff] [blame] | 152 | static struct scatterlist *mmc_alloc_sg(int sg_len, int *err) |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 153 | { |
| 154 | struct scatterlist *sg; |
| 155 | |
| 156 | sg = kmalloc(sizeof(struct scatterlist)*sg_len, GFP_KERNEL); |
| 157 | if (!sg) |
| 158 | *err = -ENOMEM; |
| 159 | else { |
| 160 | *err = 0; |
| 161 | sg_init_table(sg, sg_len); |
| 162 | } |
| 163 | |
| 164 | return sg; |
| 165 | } |
| 166 | |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 167 | static void mmc_queue_setup_discard(struct request_queue *q, |
| 168 | struct mmc_card *card) |
| 169 | { |
| 170 | unsigned max_discard; |
| 171 | |
| 172 | max_discard = mmc_calc_max_discard(card); |
| 173 | if (!max_discard) |
| 174 | return; |
| 175 | |
| 176 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
Jens Axboe | 2bb4cd5 | 2015-07-14 08:15:12 -0600 | [diff] [blame] | 177 | blk_queue_max_discard_sectors(q, max_discard); |
Adrian Hunter | 7194efb | 2012-04-05 14:45:47 +0300 | [diff] [blame] | 178 | if (card->erased_byte == 0 && !mmc_can_discard(card)) |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 179 | q->limits.discard_zeroes_data = 1; |
| 180 | q->limits.discard_granularity = card->pref_erase << 9; |
| 181 | /* granularity must not be greater than max. discard */ |
| 182 | if (card->pref_erase > max_discard) |
| 183 | q->limits.discard_granularity = 0; |
Maya Erez | 775a936 | 2013-04-18 15:41:55 +0300 | [diff] [blame] | 184 | if (mmc_can_secure_erase_trim(card)) |
Christoph Hellwig | 288dab8 | 2016-06-09 16:00:36 +0200 | [diff] [blame] | 185 | queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, q); |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 186 | } |
| 187 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | /** |
| 189 | * mmc_init_queue - initialise a queue structure. |
| 190 | * @mq: mmc queue |
| 191 | * @card: mmc card to attach this queue |
| 192 | * @lock: queue lock |
Adrian Hunter | d09408a | 2011-06-23 13:40:28 +0300 | [diff] [blame] | 193 | * @subname: partition subname |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | * |
| 195 | * Initialise a MMC card request queue. |
| 196 | */ |
Adrian Hunter | d09408a | 2011-06-23 13:40:28 +0300 | [diff] [blame] | 197 | int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, |
| 198 | spinlock_t *lock, const char *subname) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | { |
| 200 | struct mmc_host *host = card->host; |
| 201 | u64 limit = BLK_BOUNCE_HIGH; |
| 202 | int ret; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 203 | struct mmc_queue_req *mqrq_cur = &mq->mqrq[0]; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 204 | struct mmc_queue_req *mqrq_prev = &mq->mqrq[1]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | |
Greg Kroah-Hartman | fcaf71f | 2006-09-12 17:00:10 +0200 | [diff] [blame] | 206 | if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask) |
Russell King | e83b366 | 2014-02-11 17:11:04 +0000 | [diff] [blame] | 207 | limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 208 | |
| 209 | mq->card = card; |
Venkatraman S | 1b50f5f | 2012-04-13 17:54:11 +0530 | [diff] [blame] | 210 | mq->queue = blk_init_queue(mmc_request_fn, lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | if (!mq->queue) |
| 212 | return -ENOMEM; |
| 213 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 214 | mq->mqrq_cur = mqrq_cur; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 215 | mq->mqrq_prev = mqrq_prev; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 216 | mq->queue->queuedata = mq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 217 | |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 218 | blk_queue_prep_rq(mq->queue, mmc_prep_request); |
Pierre Ossman | 8dddfe1 | 2008-10-14 20:04:46 +0200 | [diff] [blame] | 219 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue); |
Mike Snitzer | b277da0 | 2014-10-04 10:55:32 -0600 | [diff] [blame] | 220 | queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, mq->queue); |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 221 | if (mmc_can_erase(card)) |
| 222 | mmc_queue_setup_discard(mq->queue, card); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 223 | |
| 224 | #ifdef CONFIG_MMC_BLOCK_BOUNCE |
Martin K. Petersen | a36274e | 2010-09-10 01:33:59 -0400 | [diff] [blame] | 225 | if (host->max_segs == 1) { |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 226 | unsigned int bouncesz; |
| 227 | |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 228 | bouncesz = MMC_QUEUE_BOUNCESZ; |
| 229 | |
| 230 | if (bouncesz > host->max_req_size) |
| 231 | bouncesz = host->max_req_size; |
| 232 | if (bouncesz > host->max_seg_size) |
| 233 | bouncesz = host->max_seg_size; |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 234 | if (bouncesz > (host->max_blk_count * 512)) |
| 235 | bouncesz = host->max_blk_count * 512; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 236 | |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 237 | if (bouncesz > 512) { |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 238 | mqrq_cur->bounce_buf = kmalloc(bouncesz, GFP_KERNEL); |
| 239 | if (!mqrq_cur->bounce_buf) { |
Joe Perches | 6606110 | 2014-09-12 14:56:56 -0700 | [diff] [blame] | 240 | pr_warn("%s: unable to allocate bounce cur buffer\n", |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 241 | mmc_card_name(card)); |
Bhuvanesh Surachari | fdb409f | 2014-12-01 02:23:02 -0500 | [diff] [blame] | 242 | } else { |
| 243 | mqrq_prev->bounce_buf = |
| 244 | kmalloc(bouncesz, GFP_KERNEL); |
| 245 | if (!mqrq_prev->bounce_buf) { |
| 246 | pr_warn("%s: unable to allocate bounce prev buffer\n", |
| 247 | mmc_card_name(card)); |
| 248 | kfree(mqrq_cur->bounce_buf); |
| 249 | mqrq_cur->bounce_buf = NULL; |
| 250 | } |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 251 | } |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 252 | } |
| 253 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 254 | if (mqrq_cur->bounce_buf && mqrq_prev->bounce_buf) { |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 255 | blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY); |
Martin K. Petersen | 086fa5f | 2010-02-26 00:20:38 -0500 | [diff] [blame] | 256 | blk_queue_max_hw_sectors(mq->queue, bouncesz / 512); |
Martin K. Petersen | 8a78362 | 2010-02-26 00:20:39 -0500 | [diff] [blame] | 257 | blk_queue_max_segments(mq->queue, bouncesz / 512); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 258 | blk_queue_max_segment_size(mq->queue, bouncesz); |
| 259 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 260 | mqrq_cur->sg = mmc_alloc_sg(1, &ret); |
| 261 | if (ret) |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 262 | goto cleanup_queue; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 263 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 264 | mqrq_cur->bounce_sg = |
| 265 | mmc_alloc_sg(bouncesz / 512, &ret); |
| 266 | if (ret) |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 267 | goto cleanup_queue; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 268 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 269 | mqrq_prev->sg = mmc_alloc_sg(1, &ret); |
| 270 | if (ret) |
| 271 | goto cleanup_queue; |
| 272 | |
| 273 | mqrq_prev->bounce_sg = |
| 274 | mmc_alloc_sg(bouncesz / 512, &ret); |
| 275 | if (ret) |
| 276 | goto cleanup_queue; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 277 | } |
| 278 | } |
| 279 | #endif |
| 280 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 281 | if (!mqrq_cur->bounce_buf && !mqrq_prev->bounce_buf) { |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 282 | blk_queue_bounce_limit(mq->queue, limit); |
Martin K. Petersen | 086fa5f | 2010-02-26 00:20:38 -0500 | [diff] [blame] | 283 | blk_queue_max_hw_sectors(mq->queue, |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 284 | min(host->max_blk_count, host->max_req_size / 512)); |
Martin K. Petersen | a36274e | 2010-09-10 01:33:59 -0400 | [diff] [blame] | 285 | blk_queue_max_segments(mq->queue, host->max_segs); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 286 | blk_queue_max_segment_size(mq->queue, host->max_seg_size); |
| 287 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 288 | mqrq_cur->sg = mmc_alloc_sg(host->max_segs, &ret); |
| 289 | if (ret) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 290 | goto cleanup_queue; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 291 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 292 | |
| 293 | mqrq_prev->sg = mmc_alloc_sg(host->max_segs, &ret); |
| 294 | if (ret) |
| 295 | goto cleanup_queue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | } |
| 297 | |
Thomas Gleixner | 632cf92 | 2010-09-14 07:12:35 -0400 | [diff] [blame] | 298 | sema_init(&mq->thread_sem, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | |
Adrian Hunter | d09408a | 2011-06-23 13:40:28 +0300 | [diff] [blame] | 300 | mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s", |
| 301 | host->index, subname ? subname : ""); |
Ethan Du | de528fa | 2010-09-30 18:40:27 -0400 | [diff] [blame] | 302 | |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 303 | if (IS_ERR(mq->thread)) { |
| 304 | ret = PTR_ERR(mq->thread); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 305 | goto free_bounce_sg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 306 | } |
| 307 | |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 308 | return 0; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 309 | free_bounce_sg: |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 310 | kfree(mqrq_cur->bounce_sg); |
| 311 | mqrq_cur->bounce_sg = NULL; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 312 | kfree(mqrq_prev->bounce_sg); |
| 313 | mqrq_prev->bounce_sg = NULL; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 314 | |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 315 | cleanup_queue: |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 316 | kfree(mqrq_cur->sg); |
| 317 | mqrq_cur->sg = NULL; |
| 318 | kfree(mqrq_cur->bounce_buf); |
| 319 | mqrq_cur->bounce_buf = NULL; |
| 320 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 321 | kfree(mqrq_prev->sg); |
| 322 | mqrq_prev->sg = NULL; |
| 323 | kfree(mqrq_prev->bounce_buf); |
| 324 | mqrq_prev->bounce_buf = NULL; |
| 325 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 326 | blk_cleanup_queue(mq->queue); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 327 | return ret; |
| 328 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 329 | |
| 330 | void mmc_cleanup_queue(struct mmc_queue *mq) |
| 331 | { |
Jens Axboe | 165125e | 2007-07-24 09:28:11 +0200 | [diff] [blame] | 332 | struct request_queue *q = mq->queue; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 333 | unsigned long flags; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 334 | struct mmc_queue_req *mqrq_cur = mq->mqrq_cur; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 335 | struct mmc_queue_req *mqrq_prev = mq->mqrq_prev; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 336 | |
Pierre Ossman | d2b46f6 | 2007-04-28 16:52:12 +0200 | [diff] [blame] | 337 | /* Make sure the queue isn't suspended, as that will deadlock */ |
| 338 | mmc_queue_resume(mq); |
| 339 | |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 340 | /* Then terminate our worker thread */ |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 341 | kthread_stop(mq->thread); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 342 | |
Adrian Hunter | 5fa83ce | 2010-01-08 14:43:00 -0800 | [diff] [blame] | 343 | /* Empty the queue */ |
| 344 | spin_lock_irqsave(q->queue_lock, flags); |
| 345 | q->queuedata = NULL; |
| 346 | blk_start_queue(q); |
| 347 | spin_unlock_irqrestore(q->queue_lock, flags); |
| 348 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 349 | kfree(mqrq_cur->bounce_sg); |
| 350 | mqrq_cur->bounce_sg = NULL; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 351 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 352 | kfree(mqrq_cur->sg); |
| 353 | mqrq_cur->sg = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 354 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 355 | kfree(mqrq_cur->bounce_buf); |
| 356 | mqrq_cur->bounce_buf = NULL; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 357 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 358 | kfree(mqrq_prev->bounce_sg); |
| 359 | mqrq_prev->bounce_sg = NULL; |
| 360 | |
| 361 | kfree(mqrq_prev->sg); |
| 362 | mqrq_prev->sg = NULL; |
| 363 | |
| 364 | kfree(mqrq_prev->bounce_buf); |
| 365 | mqrq_prev->bounce_buf = NULL; |
| 366 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 367 | mq->card = NULL; |
| 368 | } |
| 369 | EXPORT_SYMBOL(mmc_cleanup_queue); |
| 370 | |
Seungwon Jeon | ce39f9d | 2013-02-06 17:02:46 +0900 | [diff] [blame] | 371 | int mmc_packed_init(struct mmc_queue *mq, struct mmc_card *card) |
| 372 | { |
| 373 | struct mmc_queue_req *mqrq_cur = &mq->mqrq[0]; |
| 374 | struct mmc_queue_req *mqrq_prev = &mq->mqrq[1]; |
| 375 | int ret = 0; |
| 376 | |
| 377 | |
| 378 | mqrq_cur->packed = kzalloc(sizeof(struct mmc_packed), GFP_KERNEL); |
| 379 | if (!mqrq_cur->packed) { |
| 380 | pr_warn("%s: unable to allocate packed cmd for mqrq_cur\n", |
| 381 | mmc_card_name(card)); |
| 382 | ret = -ENOMEM; |
| 383 | goto out; |
| 384 | } |
| 385 | |
| 386 | mqrq_prev->packed = kzalloc(sizeof(struct mmc_packed), GFP_KERNEL); |
| 387 | if (!mqrq_prev->packed) { |
| 388 | pr_warn("%s: unable to allocate packed cmd for mqrq_prev\n", |
| 389 | mmc_card_name(card)); |
| 390 | kfree(mqrq_cur->packed); |
| 391 | mqrq_cur->packed = NULL; |
| 392 | ret = -ENOMEM; |
| 393 | goto out; |
| 394 | } |
| 395 | |
| 396 | INIT_LIST_HEAD(&mqrq_cur->packed->list); |
| 397 | INIT_LIST_HEAD(&mqrq_prev->packed->list); |
| 398 | |
| 399 | out: |
| 400 | return ret; |
| 401 | } |
| 402 | |
| 403 | void mmc_packed_clean(struct mmc_queue *mq) |
| 404 | { |
| 405 | struct mmc_queue_req *mqrq_cur = &mq->mqrq[0]; |
| 406 | struct mmc_queue_req *mqrq_prev = &mq->mqrq[1]; |
| 407 | |
| 408 | kfree(mqrq_cur->packed); |
| 409 | mqrq_cur->packed = NULL; |
| 410 | kfree(mqrq_prev->packed); |
| 411 | mqrq_prev->packed = NULL; |
| 412 | } |
| 413 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 414 | /** |
| 415 | * mmc_queue_suspend - suspend a MMC request queue |
| 416 | * @mq: MMC queue to suspend |
| 417 | * |
| 418 | * Stop the block request queue, and wait for our thread to |
| 419 | * complete any outstanding requests. This ensures that we |
| 420 | * won't suspend while a request is being processed. |
| 421 | */ |
| 422 | void mmc_queue_suspend(struct mmc_queue *mq) |
| 423 | { |
Jens Axboe | 165125e | 2007-07-24 09:28:11 +0200 | [diff] [blame] | 424 | struct request_queue *q = mq->queue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 425 | unsigned long flags; |
| 426 | |
| 427 | if (!(mq->flags & MMC_QUEUE_SUSPENDED)) { |
| 428 | mq->flags |= MMC_QUEUE_SUSPENDED; |
| 429 | |
| 430 | spin_lock_irqsave(q->queue_lock, flags); |
| 431 | blk_stop_queue(q); |
| 432 | spin_unlock_irqrestore(q->queue_lock, flags); |
| 433 | |
| 434 | down(&mq->thread_sem); |
| 435 | } |
| 436 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 437 | |
| 438 | /** |
| 439 | * mmc_queue_resume - resume a previously suspended MMC request queue |
| 440 | * @mq: MMC queue to resume |
| 441 | */ |
| 442 | void mmc_queue_resume(struct mmc_queue *mq) |
| 443 | { |
Jens Axboe | 165125e | 2007-07-24 09:28:11 +0200 | [diff] [blame] | 444 | struct request_queue *q = mq->queue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 445 | unsigned long flags; |
| 446 | |
| 447 | if (mq->flags & MMC_QUEUE_SUSPENDED) { |
| 448 | mq->flags &= ~MMC_QUEUE_SUSPENDED; |
| 449 | |
| 450 | up(&mq->thread_sem); |
| 451 | |
| 452 | spin_lock_irqsave(q->queue_lock, flags); |
| 453 | blk_start_queue(q); |
| 454 | spin_unlock_irqrestore(q->queue_lock, flags); |
| 455 | } |
| 456 | } |
Pierre Ossman | 98ac216 | 2006-12-23 20:03:02 +0100 | [diff] [blame] | 457 | |
Seungwon Jeon | ce39f9d | 2013-02-06 17:02:46 +0900 | [diff] [blame] | 458 | static unsigned int mmc_queue_packed_map_sg(struct mmc_queue *mq, |
| 459 | struct mmc_packed *packed, |
| 460 | struct scatterlist *sg, |
| 461 | enum mmc_packed_type cmd_type) |
| 462 | { |
| 463 | struct scatterlist *__sg = sg; |
| 464 | unsigned int sg_len = 0; |
| 465 | struct request *req; |
| 466 | |
| 467 | if (mmc_packed_wr(cmd_type)) { |
| 468 | unsigned int hdr_sz = mmc_large_sector(mq->card) ? 4096 : 512; |
| 469 | unsigned int max_seg_sz = queue_max_segment_size(mq->queue); |
| 470 | unsigned int len, remain, offset = 0; |
| 471 | u8 *buf = (u8 *)packed->cmd_hdr; |
| 472 | |
| 473 | remain = hdr_sz; |
| 474 | do { |
| 475 | len = min(remain, max_seg_sz); |
| 476 | sg_set_buf(__sg, buf + offset, len); |
| 477 | offset += len; |
| 478 | remain -= len; |
Dan Williams | da81ed1 | 2015-08-07 18:15:14 +0200 | [diff] [blame] | 479 | sg_unmark_end(__sg++); |
Seungwon Jeon | ce39f9d | 2013-02-06 17:02:46 +0900 | [diff] [blame] | 480 | sg_len++; |
| 481 | } while (remain); |
| 482 | } |
| 483 | |
| 484 | list_for_each_entry(req, &packed->list, queuelist) { |
| 485 | sg_len += blk_rq_map_sg(mq->queue, req, __sg); |
| 486 | __sg = sg + (sg_len - 1); |
Dan Williams | da81ed1 | 2015-08-07 18:15:14 +0200 | [diff] [blame] | 487 | sg_unmark_end(__sg++); |
Seungwon Jeon | ce39f9d | 2013-02-06 17:02:46 +0900 | [diff] [blame] | 488 | } |
| 489 | sg_mark_end(sg + (sg_len - 1)); |
| 490 | return sg_len; |
| 491 | } |
| 492 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 493 | /* |
| 494 | * Prepare the sg list(s) to be handed of to the host driver |
| 495 | */ |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 496 | unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 497 | { |
| 498 | unsigned int sg_len; |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 499 | size_t buflen; |
| 500 | struct scatterlist *sg; |
Seungwon Jeon | ce39f9d | 2013-02-06 17:02:46 +0900 | [diff] [blame] | 501 | enum mmc_packed_type cmd_type; |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 502 | int i; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 503 | |
Seungwon Jeon | ce39f9d | 2013-02-06 17:02:46 +0900 | [diff] [blame] | 504 | cmd_type = mqrq->cmd_type; |
| 505 | |
| 506 | if (!mqrq->bounce_buf) { |
| 507 | if (mmc_packed_cmd(cmd_type)) |
| 508 | return mmc_queue_packed_map_sg(mq, mqrq->packed, |
| 509 | mqrq->sg, cmd_type); |
| 510 | else |
| 511 | return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg); |
| 512 | } |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 513 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 514 | BUG_ON(!mqrq->bounce_sg); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 515 | |
Seungwon Jeon | ce39f9d | 2013-02-06 17:02:46 +0900 | [diff] [blame] | 516 | if (mmc_packed_cmd(cmd_type)) |
| 517 | sg_len = mmc_queue_packed_map_sg(mq, mqrq->packed, |
| 518 | mqrq->bounce_sg, cmd_type); |
| 519 | else |
| 520 | sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 521 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 522 | mqrq->bounce_sg_len = sg_len; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 523 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 524 | buflen = 0; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 525 | for_each_sg(mqrq->bounce_sg, sg, sg_len, i) |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 526 | buflen += sg->length; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 527 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 528 | sg_init_one(mqrq->sg, mqrq->bounce_buf, buflen); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 529 | |
| 530 | return 1; |
| 531 | } |
| 532 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 533 | /* |
| 534 | * If writing, bounce the data to the buffer before the request |
| 535 | * is sent to the host driver |
| 536 | */ |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 537 | void mmc_queue_bounce_pre(struct mmc_queue_req *mqrq) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 538 | { |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 539 | if (!mqrq->bounce_buf) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 540 | return; |
| 541 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 542 | if (rq_data_dir(mqrq->req) != WRITE) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 543 | return; |
| 544 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 545 | sg_copy_to_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len, |
| 546 | mqrq->bounce_buf, mqrq->sg[0].length); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 547 | } |
| 548 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 549 | /* |
| 550 | * If reading, bounce the data from the buffer after the request |
| 551 | * has been handled by the host driver |
| 552 | */ |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 553 | void mmc_queue_bounce_post(struct mmc_queue_req *mqrq) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 554 | { |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 555 | if (!mqrq->bounce_buf) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 556 | return; |
| 557 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 558 | if (rq_data_dir(mqrq->req) != READ) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 559 | return; |
| 560 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 561 | sg_copy_from_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len, |
| 562 | mqrq->bounce_buf, mqrq->sg[0].length); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 563 | } |