mmc: fix async request mechanism for sequential read scenarios
When current request is running on the bus and if next request fetched
by mmcqd is NULL, mmc context (mmcqd thread) gets blocked until the
current request completes. This means that if new request comes in while
the mmcqd thread is blocked, this new request can not be prepared in
parallel to current ongoing request. This may result in delaying the new
request execution and increase it's latency.
This change allows to wake up the MMC thread on new request arrival.
Now once the MMC thread is woken up, a new request can be fetched and
prepared in parallel to the current running request which means this new
request can be started immediately after the current running request
completes.
With this change read throughput is improved by 16%.
Change-Id: I8d74fda719c89a710330807c9d8994d40c885aa0
Signed-off-by: Konstantin Dorfman <kdorfman@codeaurora.org>
diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c
index 0b5449e..73c47cd 100644
--- a/drivers/mmc/card/block.c
+++ b/drivers/mmc/card/block.c
@@ -1917,8 +1917,11 @@
} else
areq = NULL;
areq = mmc_start_req(card->host, areq, (int *) &status);
- if (!areq)
+ if (!areq) {
+ if (status == MMC_BLK_NEW_REQUEST)
+ mq->flags |= MMC_QUEUE_NEW_REQUEST;
return 0;
+ }
mq_rq = container_of(areq, struct mmc_queue_req, mmc_active);
brq = &mq_rq->brq;
@@ -1927,6 +1930,8 @@
mmc_queue_bounce_post(mq_rq);
switch (status) {
+ case MMC_BLK_NEW_REQUEST:
+ BUG(); /* should never get here */
case MMC_BLK_SUCCESS:
case MMC_BLK_PARTIAL:
/*
@@ -2077,6 +2082,7 @@
mmc_blk_write_packing_control(mq, req);
+ mq->flags &= ~MMC_QUEUE_NEW_REQUEST;
if (req && req->cmd_flags & REQ_SANITIZE) {
/* complete ongoing async transfer before issuing sanitize */
if (card->host && card->host->areq)
@@ -2101,7 +2107,7 @@
}
out:
- if (!req)
+ if (!req && !(mq->flags & MMC_QUEUE_NEW_REQUEST))
/* release host only when there are no more requests */
mmc_release_host(card->host);
return ret;
diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c
index 8eb787d..c762ed1 100644
--- a/drivers/mmc/card/queue.c
+++ b/drivers/mmc/card/queue.c
@@ -22,7 +22,6 @@
#define MMC_QUEUE_BOUNCESZ 65536
-#define MMC_QUEUE_SUSPENDED (1 << 0)
/*
* Based on benchmark tests the default num of requests to trigger the write
@@ -72,6 +71,12 @@
set_current_state(TASK_INTERRUPTIBLE);
req = blk_fetch_request(q);
mq->mqrq_cur->req = req;
+ if (!req && mq->mqrq_prev->req &&
+ !(mq->mqrq_prev->req->cmd_flags & REQ_SANITIZE) &&
+ !(mq->mqrq_prev->req->cmd_flags & REQ_FLUSH) &&
+ !(mq->mqrq_prev->req->cmd_flags & REQ_DISCARD))
+ card->host->context_info.is_waiting_last_req = true;
+
spin_unlock_irq(q->queue_lock);
if (req || mq->mqrq_prev->req) {
@@ -88,6 +93,10 @@
set_current_state(TASK_RUNNING);
mq->issue_fn(mq, req);
+ if (mq->flags & MMC_QUEUE_NEW_REQUEST) {
+ mq->flags &= ~MMC_QUEUE_NEW_REQUEST;
+ continue; /* fetch again */
+ }
} else {
if (kthread_should_stop()) {
set_current_state(TASK_RUNNING);
@@ -121,6 +130,8 @@
{
struct mmc_queue *mq = q->queuedata;
struct request *req;
+ unsigned long flags;
+ struct mmc_context_info *cntx;
if (!mq) {
while ((req = blk_fetch_request(q)) != NULL) {
@@ -130,7 +141,20 @@
return;
}
- if (!mq->mqrq_cur->req && !mq->mqrq_prev->req)
+ cntx = &mq->card->host->context_info;
+ if (!mq->mqrq_cur->req && mq->mqrq_prev->req) {
+ /*
+ * New MMC request arrived when MMC thread may be
+ * blocked on the previous request to be complete
+ * with no current request fetched
+ */
+ spin_lock_irqsave(&cntx->lock, flags);
+ if (cntx->is_waiting_last_req) {
+ cntx->is_new_req = true;
+ wake_up_interruptible(&cntx->wait);
+ }
+ spin_unlock_irqrestore(&cntx->lock, flags);
+ } else if (!mq->mqrq_cur->req && !mq->mqrq_prev->req)
wake_up_process(mq->thread);
}
diff --git a/drivers/mmc/card/queue.h b/drivers/mmc/card/queue.h
index a8c104e..0a72372 100644
--- a/drivers/mmc/card/queue.h
+++ b/drivers/mmc/card/queue.h
@@ -12,17 +12,6 @@
struct mmc_data data;
};
-enum mmc_blk_status {
- MMC_BLK_SUCCESS = 0,
- MMC_BLK_PARTIAL,
- MMC_BLK_CMD_ERR,
- MMC_BLK_RETRY,
- MMC_BLK_ABORT,
- MMC_BLK_DATA_ERR,
- MMC_BLK_ECC_ERR,
- MMC_BLK_NOMEDIUM,
-};
-
enum mmc_packed_cmd {
MMC_PACKED_NONE = 0,
MMC_PACKED_WRITE,
@@ -50,6 +39,9 @@
struct task_struct *thread;
struct semaphore thread_sem;
unsigned int flags;
+#define MMC_QUEUE_SUSPENDED (1 << 0)
+#define MMC_QUEUE_NEW_REQUEST (1 << 1)
+
int (*issue_fn)(struct mmc_queue *, struct request *);
void *data;
struct request_queue *queue;
diff --git a/drivers/mmc/core/bus.c b/drivers/mmc/core/bus.c
index b24620b..2f27407 100644
--- a/drivers/mmc/core/bus.c
+++ b/drivers/mmc/core/bus.c
@@ -328,6 +328,7 @@
#ifdef CONFIG_DEBUG_FS
mmc_add_card_debugfs(card);
#endif
+ mmc_init_context_info(card->host);
ret = device_add(&card->dev);
if (ret)
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 89f834a..ce86105 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -564,12 +564,44 @@
mmc_start_bkops(card, false);
}
EXPORT_SYMBOL(mmc_start_idle_time_bkops);
+/*
+ * mmc_wait_data_done() - done callback for data request
+ * @mrq: done data request
+ *
+ * Wakes up mmc context, passed as a callback to host controller driver
+ */
+static void mmc_wait_data_done(struct mmc_request *mrq)
+{
+ mrq->host->context_info.is_done_rcv = true;
+ wake_up_interruptible(&mrq->host->context_info.wait);
+}
static void mmc_wait_done(struct mmc_request *mrq)
{
complete(&mrq->completion);
}
+/*
+ *__mmc_start_data_req() - starts data request
+ * @host: MMC host to start the request
+ * @mrq: data request to start
+ *
+ * Sets the done callback to be called when request is completed by the card.
+ * Starts data mmc request execution
+ */
+static int __mmc_start_data_req(struct mmc_host *host, struct mmc_request *mrq)
+{
+ mrq->done = mmc_wait_data_done;
+ mrq->host = host;
+ if (mmc_card_removed(host->card)) {
+ mrq->cmd->error = -ENOMEDIUM;
+ return -ENOMEDIUM;
+ }
+ mmc_start_request(host, mrq);
+
+ return 0;
+}
+
static int __mmc_start_req(struct mmc_host *host, struct mmc_request *mrq)
{
init_completion(&mrq->completion);
@@ -583,6 +615,59 @@
return 0;
}
+/*
+ * mmc_wait_for_data_req_done() - wait for request completed
+ * @host: MMC host to prepare the command.
+ * @mrq: MMC request to wait for
+ *
+ * Blocks MMC context till host controller will ack end of data request
+ * execution or new request notification arrives from the block layer.
+ * Handles command retries.
+ *
+ * Returns enum mmc_blk_status after checking errors.
+ */
+static int mmc_wait_for_data_req_done(struct mmc_host *host,
+ struct mmc_request *mrq)
+{
+ struct mmc_command *cmd;
+ struct mmc_context_info *context_info = &host->context_info;
+ int err;
+ unsigned long flags;
+
+ while (1) {
+ wait_io_event_interruptible(context_info->wait,
+ (context_info->is_done_rcv ||
+ context_info->is_new_req));
+ spin_lock_irqsave(&context_info->lock, flags);
+ context_info->is_waiting_last_req = false;
+ spin_unlock_irqrestore(&context_info->lock, flags);
+ if (context_info->is_done_rcv) {
+ context_info->is_done_rcv = false;
+ context_info->is_new_req = false;
+ cmd = mrq->cmd;
+ if (!cmd->error || !cmd->retries ||
+ mmc_card_removed(host->card)) {
+ err = host->areq->err_check(host->card,
+ host->areq);
+ break; /* return err */
+ } else {
+ pr_info("%s: req failed (CMD%u): %d, retrying...\n",
+ mmc_hostname(host),
+ cmd->opcode, cmd->error);
+ cmd->retries--;
+ cmd->error = 0;
+ host->ops->request(host, mrq);
+ continue; /* wait for done/new event again */
+ }
+ } else if (context_info->is_new_req) {
+ context_info->is_new_req = false;
+ err = MMC_BLK_NEW_REQUEST;
+ break; /* return err */
+ }
+ } /* while */
+ return err;
+}
+
static void mmc_wait_for_req_done(struct mmc_host *host,
struct mmc_request *mrq)
{
@@ -672,8 +757,21 @@
mmc_pre_req(host, areq->mrq, !host->areq);
if (host->areq) {
- mmc_wait_for_req_done(host, host->areq->mrq);
- err = host->areq->err_check(host->card, host->areq);
+ err = mmc_wait_for_data_req_done(host, host->areq->mrq);
+ if (err == MMC_BLK_NEW_REQUEST) {
+ if (areq) {
+ pr_err("%s: new request while areq = %p",
+ mmc_hostname(host), areq);
+ BUG_ON(1);
+ }
+ if (error)
+ *error = err;
+ /*
+ * The previous request was not completed,
+ * nothing to return
+ */
+ return NULL;
+ }
/*
* Check BKOPS urgency for each R1 response
*/
@@ -688,7 +786,7 @@
}
if (!err && areq)
- start_err = __mmc_start_req(host, areq->mrq);
+ start_err = __mmc_start_data_req(host, areq->mrq);
if (host->areq)
mmc_post_req(host, host->areq->mrq, 0);
@@ -3199,6 +3297,15 @@
EXPORT_SYMBOL(mmc_set_embedded_sdio_data);
#endif
+void mmc_init_context_info(struct mmc_host *host)
+{
+ spin_lock_init(&host->context_info.lock);
+ host->context_info.is_new_req = false;
+ host->context_info.is_done_rcv = false;
+ host->context_info.is_waiting_last_req = false;
+ init_waitqueue_head(&host->context_info.wait);
+}
+
static int __init mmc_init(void)
{
int ret;
diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h
index c85f5aa..6fa51e0 100644
--- a/drivers/mmc/core/core.h
+++ b/drivers/mmc/core/core.h
@@ -87,5 +87,5 @@
extern void mmc_exit_clk_scaling(struct mmc_host *host);
extern void mmc_reset_clk_scale_stats(struct mmc_host *host);
extern unsigned long mmc_get_max_frequency(struct mmc_host *host);
+void mmc_init_context_info(struct mmc_host *host);
#endif
-
diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h
index ec1d619..2895d51 100644
--- a/include/linux/mmc/card.h
+++ b/include/linux/mmc/card.h
@@ -201,6 +201,18 @@
MAX_REASONS,
};
+enum mmc_blk_status {
+ MMC_BLK_SUCCESS = 0,
+ MMC_BLK_PARTIAL,
+ MMC_BLK_CMD_ERR,
+ MMC_BLK_RETRY,
+ MMC_BLK_ABORT,
+ MMC_BLK_DATA_ERR,
+ MMC_BLK_ECC_ERR,
+ MMC_BLK_NOMEDIUM,
+ MMC_BLK_NEW_REQUEST,
+};
+
struct mmc_wr_pack_stats {
u32 *packing_events;
u32 pack_stop_reason[MAX_REASONS];
diff --git a/include/linux/mmc/core.h b/include/linux/mmc/core.h
index 24b9790..83cc723 100644
--- a/include/linux/mmc/core.h
+++ b/include/linux/mmc/core.h
@@ -124,6 +124,7 @@
bool fault_injected; /* fault injected */
};
+struct mmc_host;
struct mmc_request {
struct mmc_command *sbc; /* SET_BLOCK_COUNT for multiblock */
struct mmc_command *cmd;
@@ -132,9 +133,9 @@
struct completion completion;
void (*done)(struct mmc_request *);/* completion function */
+ struct mmc_host *host;
};
-struct mmc_host;
struct mmc_card;
struct mmc_async_req;
diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h
index 6c43ec7..6982c45 100644
--- a/include/linux/mmc/host.h
+++ b/include/linux/mmc/host.h
@@ -155,6 +155,22 @@
int (*err_check) (struct mmc_card *, struct mmc_async_req *);
};
+/**
+ * mmc_context_info - synchronization details for mmc context
+ * @is_done_rcv wake up reason was done request
+ * @is_new_req wake up reason was new request
+ * @is_waiting_last_req mmc context waiting for single running request
+ * @wait wait queue
+ * @lock lock to protect data fields
+ */
+struct mmc_context_info {
+ bool is_done_rcv;
+ bool is_new_req;
+ bool is_waiting_last_req;
+ wait_queue_head_t wait;
+ spinlock_t lock;
+};
+
struct mmc_hotplug {
unsigned int irq;
void *handler_priv;
@@ -327,6 +343,7 @@
struct dentry *debugfs_root;
struct mmc_async_req *areq; /* active async req */
+ struct mmc_context_info context_info; /* async synchronization info */
#ifdef CONFIG_FAIL_MMC_REQUEST
struct fault_attr fail_mmc_request;