Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Pierre Ossman | 70f1048 | 2007-07-11 20:04:50 +0200 | [diff] [blame] | 2 | * linux/drivers/mmc/card/queue.c |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2003 Russell King, All Rights Reserved. |
Pierre Ossman | 98ac216 | 2006-12-23 20:03:02 +0100 | [diff] [blame] | 5 | * Copyright 2006-2007 Pierre Ossman |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License version 2 as |
| 9 | * published by the Free Software Foundation. |
| 10 | * |
| 11 | */ |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 12 | #include <linux/slab.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <linux/module.h> |
| 14 | #include <linux/blkdev.h> |
Rafael J. Wysocki | 8314418 | 2007-07-17 04:03:35 -0700 | [diff] [blame] | 15 | #include <linux/freezer.h> |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 16 | #include <linux/kthread.h> |
Jens Axboe | 45711f1 | 2007-10-22 21:19:53 +0200 | [diff] [blame] | 17 | #include <linux/scatterlist.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | |
| 19 | #include <linux/mmc/card.h> |
| 20 | #include <linux/mmc/host.h> |
Pierre Ossman | 98ac216 | 2006-12-23 20:03:02 +0100 | [diff] [blame] | 21 | #include "queue.h" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 22 | |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 23 | #define MMC_QUEUE_BOUNCESZ 65536 |
| 24 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
| 26 | /* |
Tatyana Brokhman | 0cc7640 | 2012-10-07 09:52:16 +0200 | [diff] [blame] | 27 | * Based on benchmark tests the default num of requests to trigger the write |
| 28 | * packing was determined, to keep the read latency as low as possible and |
| 29 | * manage to keep the high write throughput. |
| 30 | */ |
| 31 | #define DEFAULT_NUM_REQS_TO_START_PACK 17 |
| 32 | |
| 33 | /* |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 34 | * Prepare a MMC request. This just filters out odd stuff. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | */ |
| 36 | static int mmc_prep_request(struct request_queue *q, struct request *req) |
| 37 | { |
Sujit Reddy Thumma | a8ad82c | 2011-12-08 14:05:50 +0530 | [diff] [blame] | 38 | struct mmc_queue *mq = q->queuedata; |
| 39 | |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 40 | /* |
Adrian Hunter | bd788c9 | 2010-08-11 14:17:47 -0700 | [diff] [blame] | 41 | * We only like normal block requests and discards. |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 42 | */ |
Adrian Hunter | bd788c9 | 2010-08-11 14:17:47 -0700 | [diff] [blame] | 43 | if (req->cmd_type != REQ_TYPE_FS && !(req->cmd_flags & REQ_DISCARD)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | blk_dump_rq_flags(req, "MMC bad request"); |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 45 | return BLKPREP_KILL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 46 | } |
| 47 | |
Sujit Reddy Thumma | a8ad82c | 2011-12-08 14:05:50 +0530 | [diff] [blame] | 48 | if (mq && mmc_card_removed(mq->card)) |
| 49 | return BLKPREP_KILL; |
| 50 | |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 51 | req->cmd_flags |= REQ_DONTPREP; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 52 | |
Pierre Ossman | 9c9f2d6 | 2007-05-16 17:29:21 +0200 | [diff] [blame] | 53 | return BLKPREP_OK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 54 | } |
| 55 | |
| 56 | static int mmc_queue_thread(void *d) |
| 57 | { |
| 58 | struct mmc_queue *mq = d; |
| 59 | struct request_queue *q = mq->queue; |
Bryan Huntsman | 3f2bc4d | 2011-08-16 17:27:22 -0700 | [diff] [blame] | 60 | struct request *req; |
Maya Erez | 5f36069 | 2012-10-10 03:47:54 +0200 | [diff] [blame] | 61 | struct mmc_card *card = mq->card; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | |
Rafael J. Wysocki | 8314418 | 2007-07-17 04:03:35 -0700 | [diff] [blame] | 63 | current->flags |= PF_MEMALLOC; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | down(&mq->thread_sem); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 66 | do { |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 67 | struct mmc_queue_req *tmp; |
Bryan Huntsman | 3f2bc4d | 2011-08-16 17:27:22 -0700 | [diff] [blame] | 68 | req = NULL; /* Must be set to NULL at each iteration */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | |
| 70 | spin_lock_irq(q->queue_lock); |
| 71 | set_current_state(TASK_INTERRUPTIBLE); |
Jens Axboe | 7eaceac | 2011-03-10 08:52:07 +0100 | [diff] [blame] | 72 | req = blk_fetch_request(q); |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 73 | mq->mqrq_cur->req = req; |
Konstantin Dorfman | 69bd0fb | 2012-12-12 16:12:08 +0200 | [diff] [blame^] | 74 | if (!req && mq->mqrq_prev->req && |
| 75 | !(mq->mqrq_prev->req->cmd_flags & REQ_SANITIZE) && |
| 76 | !(mq->mqrq_prev->req->cmd_flags & REQ_FLUSH) && |
| 77 | !(mq->mqrq_prev->req->cmd_flags & REQ_DISCARD)) |
| 78 | card->host->context_info.is_waiting_last_req = true; |
| 79 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 80 | spin_unlock_irq(q->queue_lock); |
| 81 | |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 82 | if (req || mq->mqrq_prev->req) { |
Maya Erez | 5f36069 | 2012-10-10 03:47:54 +0200 | [diff] [blame] | 83 | /* |
| 84 | * If this is the first request, BKOPs might be in |
| 85 | * progress and needs to be stopped before issuing the |
| 86 | * request |
| 87 | */ |
| 88 | if (card->ext_csd.bkops_en && |
| 89 | card->bkops_info.started_delayed_bkops) { |
| 90 | card->bkops_info.started_delayed_bkops = false; |
| 91 | mmc_stop_bkops(card); |
| 92 | } |
| 93 | |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 94 | set_current_state(TASK_RUNNING); |
| 95 | mq->issue_fn(mq, req); |
Konstantin Dorfman | 69bd0fb | 2012-12-12 16:12:08 +0200 | [diff] [blame^] | 96 | if (mq->flags & MMC_QUEUE_NEW_REQUEST) { |
| 97 | mq->flags &= ~MMC_QUEUE_NEW_REQUEST; |
| 98 | continue; /* fetch again */ |
| 99 | } |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 100 | } else { |
Vitaly Wool | 7b30d28 | 2006-12-07 20:08:02 +0100 | [diff] [blame] | 101 | if (kthread_should_stop()) { |
| 102 | set_current_state(TASK_RUNNING); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 103 | break; |
Vitaly Wool | 7b30d28 | 2006-12-07 20:08:02 +0100 | [diff] [blame] | 104 | } |
Maya Erez | 5f36069 | 2012-10-10 03:47:54 +0200 | [diff] [blame] | 105 | mmc_start_delayed_bkops(card); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | up(&mq->thread_sem); |
| 107 | schedule(); |
| 108 | down(&mq->thread_sem); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | |
Per Forlin | ee8a43a | 2011-07-01 18:55:33 +0200 | [diff] [blame] | 111 | /* Current request becomes previous request and vice versa. */ |
| 112 | mq->mqrq_prev->brq.mrq.data = NULL; |
| 113 | mq->mqrq_prev->req = NULL; |
| 114 | tmp = mq->mqrq_prev; |
| 115 | mq->mqrq_prev = mq->mqrq_cur; |
| 116 | mq->mqrq_cur = tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 117 | } while (1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | up(&mq->thread_sem); |
| 119 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 120 | return 0; |
| 121 | } |
| 122 | |
| 123 | /* |
| 124 | * Generic MMC request handler. This is called for any queue on a |
| 125 | * particular host. When the host is not busy, we look for a request |
| 126 | * on any queue on this host, and attempt to issue it. This may |
| 127 | * not be the queue we were asked to process. |
| 128 | */ |
Jens Axboe | 165125e | 2007-07-24 09:28:11 +0200 | [diff] [blame] | 129 | static void mmc_request(struct request_queue *q) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | { |
| 131 | struct mmc_queue *mq = q->queuedata; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 132 | struct request *req; |
Konstantin Dorfman | 69bd0fb | 2012-12-12 16:12:08 +0200 | [diff] [blame^] | 133 | unsigned long flags; |
| 134 | struct mmc_context_info *cntx; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 135 | |
| 136 | if (!mq) { |
Adrian Hunter | 5fa83ce | 2010-01-08 14:43:00 -0800 | [diff] [blame] | 137 | while ((req = blk_fetch_request(q)) != NULL) { |
| 138 | req->cmd_flags |= REQ_QUIET; |
Tejun Heo | 296b2f6 | 2009-05-08 11:54:15 +0900 | [diff] [blame] | 139 | __blk_end_request_all(req, -EIO); |
Adrian Hunter | 5fa83ce | 2010-01-08 14:43:00 -0800 | [diff] [blame] | 140 | } |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 141 | return; |
| 142 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | |
Konstantin Dorfman | 69bd0fb | 2012-12-12 16:12:08 +0200 | [diff] [blame^] | 144 | cntx = &mq->card->host->context_info; |
| 145 | if (!mq->mqrq_cur->req && mq->mqrq_prev->req) { |
| 146 | /* |
| 147 | * New MMC request arrived when MMC thread may be |
| 148 | * blocked on the previous request to be complete |
| 149 | * with no current request fetched |
| 150 | */ |
| 151 | spin_lock_irqsave(&cntx->lock, flags); |
| 152 | if (cntx->is_waiting_last_req) { |
| 153 | cntx->is_new_req = true; |
| 154 | wake_up_interruptible(&cntx->wait); |
| 155 | } |
| 156 | spin_unlock_irqrestore(&cntx->lock, flags); |
| 157 | } else if (!mq->mqrq_cur->req && !mq->mqrq_prev->req) |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 158 | wake_up_process(mq->thread); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 159 | } |
| 160 | |
Venkatraman S | 7513cd7 | 2011-08-23 21:16:02 +0530 | [diff] [blame] | 161 | static struct scatterlist *mmc_alloc_sg(int sg_len, int *err) |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 162 | { |
| 163 | struct scatterlist *sg; |
| 164 | |
| 165 | sg = kmalloc(sizeof(struct scatterlist)*sg_len, GFP_KERNEL); |
| 166 | if (!sg) |
| 167 | *err = -ENOMEM; |
| 168 | else { |
| 169 | *err = 0; |
| 170 | sg_init_table(sg, sg_len); |
| 171 | } |
| 172 | |
| 173 | return sg; |
| 174 | } |
| 175 | |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 176 | static void mmc_queue_setup_discard(struct request_queue *q, |
| 177 | struct mmc_card *card) |
| 178 | { |
| 179 | unsigned max_discard; |
| 180 | |
| 181 | max_discard = mmc_calc_max_discard(card); |
| 182 | if (!max_discard) |
| 183 | return; |
| 184 | |
| 185 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
| 186 | q->limits.max_discard_sectors = max_discard; |
Adrian Hunter | 7194efb | 2012-04-05 14:45:47 +0300 | [diff] [blame] | 187 | if (card->erased_byte == 0 && !mmc_can_discard(card)) |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 188 | q->limits.discard_zeroes_data = 1; |
| 189 | q->limits.discard_granularity = card->pref_erase << 9; |
| 190 | /* granularity must not be greater than max. discard */ |
| 191 | if (card->pref_erase > max_discard) |
| 192 | q->limits.discard_granularity = 0; |
Maya Erez | 463bb95 | 2012-05-24 23:46:29 +0300 | [diff] [blame] | 193 | if (mmc_can_secure_erase_trim(card)) |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 194 | queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q); |
| 195 | } |
| 196 | |
Maya Erez | 463bb95 | 2012-05-24 23:46:29 +0300 | [diff] [blame] | 197 | static void mmc_queue_setup_sanitize(struct request_queue *q) |
| 198 | { |
| 199 | queue_flag_set_unlocked(QUEUE_FLAG_SANITIZE, q); |
| 200 | } |
| 201 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 202 | /** |
| 203 | * mmc_init_queue - initialise a queue structure. |
| 204 | * @mq: mmc queue |
| 205 | * @card: mmc card to attach this queue |
| 206 | * @lock: queue lock |
Adrian Hunter | d09408a | 2011-06-23 13:40:28 +0300 | [diff] [blame] | 207 | * @subname: partition subname |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 208 | * |
| 209 | * Initialise a MMC card request queue. |
| 210 | */ |
Adrian Hunter | d09408a | 2011-06-23 13:40:28 +0300 | [diff] [blame] | 211 | int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, |
| 212 | spinlock_t *lock, const char *subname) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 213 | { |
| 214 | struct mmc_host *host = card->host; |
| 215 | u64 limit = BLK_BOUNCE_HIGH; |
| 216 | int ret; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 217 | struct mmc_queue_req *mqrq_cur = &mq->mqrq[0]; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 218 | struct mmc_queue_req *mqrq_prev = &mq->mqrq[1]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 219 | |
Greg Kroah-Hartman | fcaf71f | 2006-09-12 17:00:10 +0200 | [diff] [blame] | 220 | if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask) |
| 221 | limit = *mmc_dev(host)->dma_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | |
| 223 | mq->card = card; |
| 224 | mq->queue = blk_init_queue(mmc_request, lock); |
| 225 | if (!mq->queue) |
| 226 | return -ENOMEM; |
| 227 | |
Maya Erez | 47b3792 | 2012-10-29 20:19:01 +0200 | [diff] [blame] | 228 | memset(&mq->mqrq_cur, 0, sizeof(mq->mqrq_cur)); |
| 229 | memset(&mq->mqrq_prev, 0, sizeof(mq->mqrq_prev)); |
| 230 | |
Seungwon Jeon | 53f8f57 | 2012-09-27 15:00:26 +0200 | [diff] [blame] | 231 | INIT_LIST_HEAD(&mqrq_cur->packed_list); |
| 232 | INIT_LIST_HEAD(&mqrq_prev->packed_list); |
| 233 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 234 | mq->mqrq_cur = mqrq_cur; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 235 | mq->mqrq_prev = mqrq_prev; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 236 | mq->queue->queuedata = mq; |
Yaniv Gardi | ec7a9ac | 2012-11-28 14:52:52 +0200 | [diff] [blame] | 237 | mq->num_wr_reqs_to_start_packing = |
| 238 | min_t(int, (int)card->ext_csd.max_packed_writes, |
| 239 | DEFAULT_NUM_REQS_TO_START_PACK); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 240 | |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 241 | blk_queue_prep_rq(mq->queue, mmc_prep_request); |
Pierre Ossman | 8dddfe1 | 2008-10-14 20:04:46 +0200 | [diff] [blame] | 242 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue); |
Adrian Hunter | e056a1b | 2011-06-28 17:16:02 +0300 | [diff] [blame] | 243 | if (mmc_can_erase(card)) |
| 244 | mmc_queue_setup_discard(mq->queue, card); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 245 | |
Maya Erez | 463bb95 | 2012-05-24 23:46:29 +0300 | [diff] [blame] | 246 | if ((mmc_can_sanitize(card) && (host->caps2 & MMC_CAP2_SANITIZE))) |
| 247 | mmc_queue_setup_sanitize(mq->queue); |
| 248 | |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 249 | #ifdef CONFIG_MMC_BLOCK_BOUNCE |
Martin K. Petersen | a36274e | 2010-09-10 01:33:59 -0400 | [diff] [blame] | 250 | if (host->max_segs == 1) { |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 251 | unsigned int bouncesz; |
| 252 | |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 253 | bouncesz = MMC_QUEUE_BOUNCESZ; |
| 254 | |
| 255 | if (bouncesz > host->max_req_size) |
| 256 | bouncesz = host->max_req_size; |
| 257 | if (bouncesz > host->max_seg_size) |
| 258 | bouncesz = host->max_seg_size; |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 259 | if (bouncesz > (host->max_blk_count * 512)) |
| 260 | bouncesz = host->max_blk_count * 512; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 261 | |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 262 | if (bouncesz > 512) { |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 263 | mqrq_cur->bounce_buf = kmalloc(bouncesz, GFP_KERNEL); |
| 264 | if (!mqrq_cur->bounce_buf) { |
Girish K S | a3c76eb | 2011-10-11 11:44:09 +0530 | [diff] [blame] | 265 | pr_warning("%s: unable to " |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 266 | "allocate bounce cur buffer\n", |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 267 | mmc_card_name(card)); |
| 268 | } |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 269 | mqrq_prev->bounce_buf = kmalloc(bouncesz, GFP_KERNEL); |
| 270 | if (!mqrq_prev->bounce_buf) { |
Girish K S | a3c76eb | 2011-10-11 11:44:09 +0530 | [diff] [blame] | 271 | pr_warning("%s: unable to " |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 272 | "allocate bounce prev buffer\n", |
| 273 | mmc_card_name(card)); |
| 274 | kfree(mqrq_cur->bounce_buf); |
| 275 | mqrq_cur->bounce_buf = NULL; |
| 276 | } |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 277 | } |
| 278 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 279 | if (mqrq_cur->bounce_buf && mqrq_prev->bounce_buf) { |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 280 | blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY); |
Martin K. Petersen | 086fa5f | 2010-02-26 00:20:38 -0500 | [diff] [blame] | 281 | blk_queue_max_hw_sectors(mq->queue, bouncesz / 512); |
Martin K. Petersen | 8a78362 | 2010-02-26 00:20:39 -0500 | [diff] [blame] | 282 | blk_queue_max_segments(mq->queue, bouncesz / 512); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 283 | blk_queue_max_segment_size(mq->queue, bouncesz); |
| 284 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 285 | mqrq_cur->sg = mmc_alloc_sg(1, &ret); |
| 286 | if (ret) |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 287 | goto cleanup_queue; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 288 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 289 | mqrq_cur->bounce_sg = |
| 290 | mmc_alloc_sg(bouncesz / 512, &ret); |
| 291 | if (ret) |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 292 | goto cleanup_queue; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 293 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 294 | mqrq_prev->sg = mmc_alloc_sg(1, &ret); |
| 295 | if (ret) |
| 296 | goto cleanup_queue; |
| 297 | |
| 298 | mqrq_prev->bounce_sg = |
| 299 | mmc_alloc_sg(bouncesz / 512, &ret); |
| 300 | if (ret) |
| 301 | goto cleanup_queue; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 302 | } |
| 303 | } |
| 304 | #endif |
| 305 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 306 | if (!mqrq_cur->bounce_buf && !mqrq_prev->bounce_buf) { |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 307 | blk_queue_bounce_limit(mq->queue, limit); |
Martin K. Petersen | 086fa5f | 2010-02-26 00:20:38 -0500 | [diff] [blame] | 308 | blk_queue_max_hw_sectors(mq->queue, |
Pierre Ossman | f3eb0aa | 2008-08-16 21:34:02 +0200 | [diff] [blame] | 309 | min(host->max_blk_count, host->max_req_size / 512)); |
Martin K. Petersen | a36274e | 2010-09-10 01:33:59 -0400 | [diff] [blame] | 310 | blk_queue_max_segments(mq->queue, host->max_segs); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 311 | blk_queue_max_segment_size(mq->queue, host->max_seg_size); |
| 312 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 313 | mqrq_cur->sg = mmc_alloc_sg(host->max_segs, &ret); |
| 314 | if (ret) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 315 | goto cleanup_queue; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 316 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 317 | |
| 318 | mqrq_prev->sg = mmc_alloc_sg(host->max_segs, &ret); |
| 319 | if (ret) |
| 320 | goto cleanup_queue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 321 | } |
| 322 | |
Thomas Gleixner | 632cf92 | 2010-09-14 07:12:35 -0400 | [diff] [blame] | 323 | sema_init(&mq->thread_sem, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 324 | |
Adrian Hunter | d09408a | 2011-06-23 13:40:28 +0300 | [diff] [blame] | 325 | mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s", |
| 326 | host->index, subname ? subname : ""); |
Ethan Du | de528fa | 2010-09-30 18:40:27 -0400 | [diff] [blame] | 327 | |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 328 | if (IS_ERR(mq->thread)) { |
| 329 | ret = PTR_ERR(mq->thread); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 330 | goto free_bounce_sg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | } |
| 332 | |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 333 | return 0; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 334 | free_bounce_sg: |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 335 | kfree(mqrq_cur->bounce_sg); |
| 336 | mqrq_cur->bounce_sg = NULL; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 337 | kfree(mqrq_prev->bounce_sg); |
| 338 | mqrq_prev->bounce_sg = NULL; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 339 | |
Pierre Ossman | aafabfa | 2007-08-09 14:28:02 +0200 | [diff] [blame] | 340 | cleanup_queue: |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 341 | kfree(mqrq_cur->sg); |
| 342 | mqrq_cur->sg = NULL; |
| 343 | kfree(mqrq_cur->bounce_buf); |
| 344 | mqrq_cur->bounce_buf = NULL; |
| 345 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 346 | kfree(mqrq_prev->sg); |
| 347 | mqrq_prev->sg = NULL; |
| 348 | kfree(mqrq_prev->bounce_buf); |
| 349 | mqrq_prev->bounce_buf = NULL; |
| 350 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 351 | blk_cleanup_queue(mq->queue); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 352 | return ret; |
| 353 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 354 | |
| 355 | void mmc_cleanup_queue(struct mmc_queue *mq) |
| 356 | { |
Jens Axboe | 165125e | 2007-07-24 09:28:11 +0200 | [diff] [blame] | 357 | struct request_queue *q = mq->queue; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 358 | unsigned long flags; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 359 | struct mmc_queue_req *mqrq_cur = mq->mqrq_cur; |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 360 | struct mmc_queue_req *mqrq_prev = mq->mqrq_prev; |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 361 | |
Pierre Ossman | d2b46f6 | 2007-04-28 16:52:12 +0200 | [diff] [blame] | 362 | /* Make sure the queue isn't suspended, as that will deadlock */ |
| 363 | mmc_queue_resume(mq); |
| 364 | |
Pierre Ossman | 89b4e13 | 2006-11-14 22:08:16 +0100 | [diff] [blame] | 365 | /* Then terminate our worker thread */ |
Christoph Hellwig | 87598a2 | 2006-11-13 20:23:52 +0100 | [diff] [blame] | 366 | kthread_stop(mq->thread); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 367 | |
Adrian Hunter | 5fa83ce | 2010-01-08 14:43:00 -0800 | [diff] [blame] | 368 | /* Empty the queue */ |
| 369 | spin_lock_irqsave(q->queue_lock, flags); |
| 370 | q->queuedata = NULL; |
| 371 | blk_start_queue(q); |
| 372 | spin_unlock_irqrestore(q->queue_lock, flags); |
| 373 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 374 | kfree(mqrq_cur->bounce_sg); |
| 375 | mqrq_cur->bounce_sg = NULL; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 376 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 377 | kfree(mqrq_cur->sg); |
| 378 | mqrq_cur->sg = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 379 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 380 | kfree(mqrq_cur->bounce_buf); |
| 381 | mqrq_cur->bounce_buf = NULL; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 382 | |
Per Forlin | 04296b7 | 2011-07-01 18:55:31 +0200 | [diff] [blame] | 383 | kfree(mqrq_prev->bounce_sg); |
| 384 | mqrq_prev->bounce_sg = NULL; |
| 385 | |
| 386 | kfree(mqrq_prev->sg); |
| 387 | mqrq_prev->sg = NULL; |
| 388 | |
| 389 | kfree(mqrq_prev->bounce_buf); |
| 390 | mqrq_prev->bounce_buf = NULL; |
| 391 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 392 | mq->card = NULL; |
| 393 | } |
| 394 | EXPORT_SYMBOL(mmc_cleanup_queue); |
| 395 | |
| 396 | /** |
| 397 | * mmc_queue_suspend - suspend a MMC request queue |
| 398 | * @mq: MMC queue to suspend |
| 399 | * |
| 400 | * Stop the block request queue, and wait for our thread to |
| 401 | * complete any outstanding requests. This ensures that we |
| 402 | * won't suspend while a request is being processed. |
| 403 | */ |
| 404 | void mmc_queue_suspend(struct mmc_queue *mq) |
| 405 | { |
Jens Axboe | 165125e | 2007-07-24 09:28:11 +0200 | [diff] [blame] | 406 | struct request_queue *q = mq->queue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 407 | unsigned long flags; |
| 408 | |
| 409 | if (!(mq->flags & MMC_QUEUE_SUSPENDED)) { |
| 410 | mq->flags |= MMC_QUEUE_SUSPENDED; |
| 411 | |
| 412 | spin_lock_irqsave(q->queue_lock, flags); |
| 413 | blk_stop_queue(q); |
| 414 | spin_unlock_irqrestore(q->queue_lock, flags); |
| 415 | |
| 416 | down(&mq->thread_sem); |
| 417 | } |
| 418 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 419 | |
| 420 | /** |
| 421 | * mmc_queue_resume - resume a previously suspended MMC request queue |
| 422 | * @mq: MMC queue to resume |
| 423 | */ |
| 424 | void mmc_queue_resume(struct mmc_queue *mq) |
| 425 | { |
Jens Axboe | 165125e | 2007-07-24 09:28:11 +0200 | [diff] [blame] | 426 | struct request_queue *q = mq->queue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 427 | unsigned long flags; |
| 428 | |
| 429 | if (mq->flags & MMC_QUEUE_SUSPENDED) { |
| 430 | mq->flags &= ~MMC_QUEUE_SUSPENDED; |
| 431 | |
| 432 | up(&mq->thread_sem); |
| 433 | |
| 434 | spin_lock_irqsave(q->queue_lock, flags); |
| 435 | blk_start_queue(q); |
| 436 | spin_unlock_irqrestore(q->queue_lock, flags); |
| 437 | } |
| 438 | } |
Pierre Ossman | 98ac216 | 2006-12-23 20:03:02 +0100 | [diff] [blame] | 439 | |
Seungwon Jeon | 53f8f57 | 2012-09-27 15:00:26 +0200 | [diff] [blame] | 440 | static unsigned int mmc_queue_packed_map_sg(struct mmc_queue *mq, |
| 441 | struct mmc_queue_req *mqrq, |
| 442 | struct scatterlist *sg) |
| 443 | { |
| 444 | struct scatterlist *__sg; |
| 445 | unsigned int sg_len = 0; |
| 446 | struct request *req; |
| 447 | enum mmc_packed_cmd cmd; |
| 448 | |
| 449 | cmd = mqrq->packed_cmd; |
| 450 | |
| 451 | if (cmd == MMC_PACKED_WRITE) { |
| 452 | __sg = sg; |
| 453 | sg_set_buf(__sg, mqrq->packed_cmd_hdr, |
| 454 | sizeof(mqrq->packed_cmd_hdr)); |
| 455 | sg_len++; |
| 456 | __sg->page_link &= ~0x02; |
| 457 | } |
| 458 | |
| 459 | __sg = sg + sg_len; |
| 460 | list_for_each_entry(req, &mqrq->packed_list, queuelist) { |
| 461 | sg_len += blk_rq_map_sg(mq->queue, req, __sg); |
| 462 | __sg = sg + (sg_len - 1); |
| 463 | (__sg++)->page_link &= ~0x02; |
| 464 | } |
| 465 | sg_mark_end(sg + (sg_len - 1)); |
| 466 | return sg_len; |
| 467 | } |
| 468 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 469 | /* |
| 470 | * Prepare the sg list(s) to be handed of to the host driver |
| 471 | */ |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 472 | unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 473 | { |
| 474 | unsigned int sg_len; |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 475 | size_t buflen; |
| 476 | struct scatterlist *sg; |
| 477 | int i; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 478 | |
Seungwon Jeon | 53f8f57 | 2012-09-27 15:00:26 +0200 | [diff] [blame] | 479 | if (!mqrq->bounce_buf) { |
| 480 | if (!list_empty(&mqrq->packed_list)) |
| 481 | return mmc_queue_packed_map_sg(mq, mqrq, mqrq->sg); |
| 482 | else |
| 483 | return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg); |
| 484 | } |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 485 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 486 | BUG_ON(!mqrq->bounce_sg); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 487 | |
Seungwon Jeon | 53f8f57 | 2012-09-27 15:00:26 +0200 | [diff] [blame] | 488 | if (!list_empty(&mqrq->packed_list)) |
| 489 | sg_len = mmc_queue_packed_map_sg(mq, mqrq, mqrq->bounce_sg); |
| 490 | else |
| 491 | sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 492 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 493 | mqrq->bounce_sg_len = sg_len; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 494 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 495 | buflen = 0; |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 496 | for_each_sg(mqrq->bounce_sg, sg, sg_len, i) |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 497 | buflen += sg->length; |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 498 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 499 | sg_init_one(mqrq->sg, mqrq->bounce_buf, buflen); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 500 | |
| 501 | return 1; |
| 502 | } |
| 503 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 504 | /* |
| 505 | * If writing, bounce the data to the buffer before the request |
| 506 | * is sent to the host driver |
| 507 | */ |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 508 | void mmc_queue_bounce_pre(struct mmc_queue_req *mqrq) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 509 | { |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 510 | if (!mqrq->bounce_buf) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 511 | return; |
| 512 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 513 | if (rq_data_dir(mqrq->req) != WRITE) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 514 | return; |
| 515 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 516 | sg_copy_to_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len, |
| 517 | mqrq->bounce_buf, mqrq->sg[0].length); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 518 | } |
| 519 | |
Pierre Ossman | 2ff1fa6 | 2008-07-22 14:35:42 +0200 | [diff] [blame] | 520 | /* |
| 521 | * If reading, bounce the data from the buffer after the request |
| 522 | * has been handled by the host driver |
| 523 | */ |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 524 | void mmc_queue_bounce_post(struct mmc_queue_req *mqrq) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 525 | { |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 526 | if (!mqrq->bounce_buf) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 527 | return; |
| 528 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 529 | if (rq_data_dir(mqrq->req) != READ) |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 530 | return; |
| 531 | |
Per Forlin | 97868a2 | 2011-07-09 17:12:36 -0400 | [diff] [blame] | 532 | sg_copy_from_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len, |
| 533 | mqrq->bounce_buf, mqrq->sg[0].length); |
Pierre Ossman | 98ccf14 | 2007-05-12 00:26:16 +0200 | [diff] [blame] | 534 | } |